mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-08 07:35:50 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9860ae582b | ||
|
|
c6f454fb9b | ||
|
|
abbd0207ba |
@@ -9,96 +9,17 @@ permissions:
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -109,40 +30,40 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v7
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -58,80 +58,8 @@ jobs:
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
needs: [setup]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -166,32 +94,11 @@ jobs:
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
@@ -220,13 +127,13 @@ jobs:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -259,7 +166,7 @@ jobs:
|
||||
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
needs: [setup, build]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
@@ -274,7 +181,7 @@ jobs:
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -284,29 +191,6 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
@@ -429,12 +313,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -74,8 +74,6 @@ jobs:
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
@@ -105,7 +103,7 @@ jobs:
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
@@ -168,7 +166,7 @@ jobs:
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
@@ -236,14 +234,14 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -299,14 +297,14 @@ jobs:
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
@@ -169,7 +169,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
@@ -245,7 +245,7 @@ jobs:
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "steward",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "steward-key",
|
||||
"secretKey": "steward-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "le001",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "le001-key",
|
||||
"secretKey": "le001-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -34,7 +34,7 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
@@ -86,7 +86,7 @@ require (
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.etcd.io/etcd/client/v3 v3.6.9
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.45.0
|
||||
@@ -95,21 +95,22 @@ require (
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
|
||||
golang.org/x/image v0.38.0
|
||||
golang.org/x/net v0.52.0
|
||||
golang.org/x/net v0.51.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.274.0
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/api v0.267.0
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.46.1
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -125,8 +126,8 @@ require (
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.13
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
@@ -135,9 +136,10 @@ require (
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.28.0
|
||||
@@ -155,7 +157,7 @@ require (
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.9
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
@@ -289,7 +291,7 @@ require (
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth v0.18.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
@@ -321,21 +323,21 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
@@ -385,7 +387,7 @@ require (
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
@@ -484,22 +486,22 @@ require (
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.9 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
|
||||
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
|
||||
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
||||
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
|
||||
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA=
|
||||
cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
@@ -716,12 +716,12 @@ github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
@@ -732,28 +732,28 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgq
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
|
||||
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
|
||||
@@ -1383,8 +1383,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
|
||||
github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
|
||||
github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
|
||||
github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
|
||||
github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
@@ -2101,12 +2101,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
|
||||
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
|
||||
go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
|
||||
go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
|
||||
go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
|
||||
go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
@@ -2128,8 +2128,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
|
||||
@@ -2144,14 +2144,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
@@ -2344,8 +2344,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -2552,8 +2552,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -2703,8 +2703,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
||||
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -2838,12 +2838,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
|
||||
Generated
+321
-150
File diff suppressed because it is too large
Load Diff
@@ -20,7 +20,7 @@ default = ["5bytes"]
|
||||
[dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
tokio-stream = "0.1"
|
||||
tokio-io-timeout = "1"
|
||||
|
||||
# gRPC + protobuf
|
||||
@@ -82,7 +82,7 @@ memmap2 = "0.9"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
# HTTP client (for proxying, remote fetch)
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
reqwest = { version = "0.12", features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
|
||||
# Content hashing
|
||||
md-5 = "0.10"
|
||||
|
||||
+40
-68
@@ -600,35 +600,35 @@ async fn run(
|
||||
})
|
||||
};
|
||||
|
||||
// Bind the gRPC listener before spawning to propagate bind errors at startup.
|
||||
let grpc_listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let grpc_local_addr = grpc_listener
|
||||
.local_addr()
|
||||
.unwrap_or_else(|e| panic!("Failed to get gRPC local addr: {}", e));
|
||||
|
||||
let grpc_handle = {
|
||||
let grpc_state = state.clone();
|
||||
let grpc_addr = grpc_addr.clone();
|
||||
let grpc_tls_acceptor = grpc_tls_acceptor.clone();
|
||||
let mut shutdown_rx = shutdown_tx.subscribe();
|
||||
let shutdown_tx_grpc = shutdown_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let addr = tokio::net::lookup_host(&grpc_addr)
|
||||
.await
|
||||
.expect("Failed to resolve gRPC address")
|
||||
.next()
|
||||
.expect("No addresses found for gRPC bind address");
|
||||
let grpc_service = VolumeGrpcService {
|
||||
state: grpc_state.clone(),
|
||||
};
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
let result = if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let incoming = grpc_tls_incoming(grpc_listener, tls_acceptor);
|
||||
info!("gRPC server listening on {} (TLS enabled)", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let incoming = grpc_tls_incoming(listener, tls_acceptor);
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {} (TLS enabled)", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
@@ -637,25 +637,32 @@ async fn run(
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
} else {
|
||||
let incoming =
|
||||
tokio_stream::wrappers::TcpListenerStream::new(grpc_listener);
|
||||
info!("gRPC server listening on {}", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {}", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
.add_service(build_volume_grpc_service(grpc_service))
|
||||
.serve_with_incoming_shutdown(incoming, async move {
|
||||
.serve_with_shutdown(addr, async move {
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
};
|
||||
if let Err(ref e) = result {
|
||||
error!("gRPC server error: {}", e);
|
||||
let _ = shutdown_tx_grpc.send(());
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
}
|
||||
result
|
||||
})
|
||||
};
|
||||
|
||||
@@ -764,40 +771,9 @@ async fn run(
|
||||
}))
|
||||
};
|
||||
|
||||
// Wait for servers. Use select! with &mut so the losing handle is not
|
||||
// dropped, then await it explicitly afterward.
|
||||
let mut server_err: Option<String> = None;
|
||||
let mut http_handle = http_handle;
|
||||
let mut grpc_handle = grpc_handle;
|
||||
let grpc_finished_first = tokio::select! {
|
||||
_ = &mut http_handle => false,
|
||||
_ = &mut grpc_handle => true,
|
||||
};
|
||||
// Inspect the gRPC result (already resolved if it finished first,
|
||||
// otherwise await it now).
|
||||
let grpc_result = if grpc_finished_first {
|
||||
grpc_handle.await
|
||||
} else {
|
||||
// HTTP finished first; gRPC is still running. Await it.
|
||||
grpc_handle.await
|
||||
};
|
||||
match grpc_result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(e)) => {
|
||||
let msg = format!("gRPC server exited with error: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
// serve error already sent shutdown inside the task
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("gRPC task panicked: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
let _ = shutdown_tx.send(());
|
||||
}
|
||||
}
|
||||
// Ensure the HTTP handle completes too.
|
||||
// Wait for all servers
|
||||
let _ = http_handle.await;
|
||||
let _ = grpc_handle.await;
|
||||
if let Some(h) = public_handle {
|
||||
let _ = h.await;
|
||||
}
|
||||
@@ -822,10 +798,6 @@ async fn run(
|
||||
cpu_profile.finish().map_err(std::io::Error::other)?;
|
||||
}
|
||||
|
||||
if let Some(err_msg) = server_err {
|
||||
return Err(std::io::Error::other(err_msg).into());
|
||||
}
|
||||
|
||||
info!("Volume server stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1867,7 +1867,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
{
|
||||
let needle_header = resp.needle_header;
|
||||
let mut needle_body = resp.needle_body;
|
||||
let resp_version = resp.version;
|
||||
|
||||
if needle_header.is_empty() {
|
||||
continue;
|
||||
@@ -1892,36 +1891,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Parse needle from header + body
|
||||
let mut n = Needle::default();
|
||||
n.read_header(&needle_header);
|
||||
|
||||
if n.size.0 < 0 {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"unexpected negative needle size {} for needle {}",
|
||||
n.size.0, n.id.0
|
||||
)));
|
||||
} else if n.size.0 > 0 {
|
||||
// Normal needle: parse the body fields (DataSize, Data, flags, etc.)
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
} else {
|
||||
// Delete tombstone (size == 0): body is checksum + timestamp
|
||||
// (V3) or checksum only (V2) + padding. Validate minimum
|
||||
// footer length for the protocol version.
|
||||
use crate::storage::types::{
|
||||
NEEDLE_CHECKSUM_SIZE, TIMESTAMP_SIZE, VERSION_3, Version,
|
||||
};
|
||||
let version = Version(resp_version as u8);
|
||||
let min_footer = if version >= VERSION_3 {
|
||||
NEEDLE_CHECKSUM_SIZE + TIMESTAMP_SIZE
|
||||
} else {
|
||||
NEEDLE_CHECKSUM_SIZE
|
||||
};
|
||||
if needle_body.len() < min_footer {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"tombstone needle {} body too short: got {} bytes, need >= {} for version {}",
|
||||
n.id.0, needle_body.len(), min_footer, resp_version
|
||||
)));
|
||||
}
|
||||
}
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
|
||||
// Write needle to local volume
|
||||
let mut store = state.store.write().unwrap();
|
||||
@@ -4076,12 +4047,11 @@ fn find_last_append_at_ns(idx_path: &str, dat_path: &str, version: u32) -> Optio
|
||||
let mut header = [0u8; 16];
|
||||
dat_file.read_exact(&mut header).ok()?;
|
||||
let needle_size = i32::from_be_bytes([header[12], header[13], header[14], header[15]]);
|
||||
if needle_size < 0 {
|
||||
if needle_size <= 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Seek to tail: offset + 16 (header) + size -> checksum (4) + timestamp (8)
|
||||
// For delete needles (size == 0), the tail is right after the header.
|
||||
let tail_offset = actual_offset as u64 + 16 + needle_size as u64;
|
||||
dat_file.seek(SeekFrom::Start(tail_offset)).ok()?;
|
||||
|
||||
|
||||
@@ -1,368 +0,0 @@
|
||||
package framework
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
)
|
||||
|
||||
// MixedVolumeCluster wraps a Go master + a mix of Go and Rust volume servers
|
||||
// for integration testing. Go servers occupy indices [0, goCount) and Rust
|
||||
// servers occupy indices [goCount, goCount+rustCount).
|
||||
type MixedVolumeCluster struct {
|
||||
testingTB testing.TB
|
||||
profile matrix.Profile
|
||||
|
||||
weedBinary string // Go weed binary (master + Go volume servers)
|
||||
rustVolumeBinary string // Rust volume binary
|
||||
|
||||
baseDir string
|
||||
configDir string
|
||||
logsDir string
|
||||
keepLogs bool
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
|
||||
volumePorts []int
|
||||
volumeGrpcPorts []int
|
||||
volumePubPorts []int
|
||||
isRust []bool // which servers are Rust
|
||||
|
||||
masterCmd *exec.Cmd
|
||||
volumeCmds []*exec.Cmd
|
||||
|
||||
cleanupOnce sync.Once
|
||||
}
|
||||
|
||||
// StartMixedVolumeCluster starts a cluster with 1 Go master, goCount Go volume
|
||||
// servers, and rustCount Rust volume servers. Go servers come first in the index.
|
||||
func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rustCount int) *MixedVolumeCluster {
|
||||
t.Helper()
|
||||
|
||||
if goCount < 0 || rustCount < 0 {
|
||||
t.Fatalf("goCount and rustCount must be non-negative, got go=%d rust=%d", goCount, rustCount)
|
||||
}
|
||||
total := goCount + rustCount
|
||||
if total < 2 {
|
||||
t.Fatalf("need at least 2 volume servers, got %d", total)
|
||||
}
|
||||
|
||||
weedBinary, err := FindOrBuildWeedBinary()
|
||||
if err != nil {
|
||||
t.Fatalf("resolve weed binary: %v", err)
|
||||
}
|
||||
|
||||
// Only build the Rust binary when Rust servers are requested.
|
||||
var rustBinary string
|
||||
if rustCount > 0 {
|
||||
rustBinary, err = FindOrBuildRustBinary()
|
||||
if err != nil {
|
||||
t.Skipf("skipping mixed cluster test: rust binary unavailable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
baseDir, keepLogs, err := newWorkDir()
|
||||
if err != nil {
|
||||
t.Fatalf("create temp test directory: %v", err)
|
||||
}
|
||||
|
||||
configDir := filepath.Join(baseDir, "config")
|
||||
logsDir := filepath.Join(baseDir, "logs")
|
||||
masterDataDir := filepath.Join(baseDir, "master")
|
||||
|
||||
dirs := []string{configDir, logsDir, masterDataDir}
|
||||
for i := 0; i < total; i++ {
|
||||
dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
|
||||
t.Fatalf("create %s: %v", dir, mkErr)
|
||||
}
|
||||
}
|
||||
|
||||
if err = writeSecurityConfig(configDir, profile); err != nil {
|
||||
t.Fatalf("write security config: %v", err)
|
||||
}
|
||||
|
||||
masterPort, masterGrpcPort, err := allocateMasterPortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("allocate master port pair: %v", err)
|
||||
}
|
||||
|
||||
// 2 ports per server (admin, grpc); add 1 more when public port is split out.
|
||||
portsPerServer := 2
|
||||
if profile.SplitPublicPort {
|
||||
portsPerServer = 3
|
||||
}
|
||||
ports, err := allocatePorts(total * portsPerServer)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate volume ports: %v", err)
|
||||
}
|
||||
|
||||
isRust := make([]bool, total)
|
||||
for i := goCount; i < total; i++ {
|
||||
isRust[i] = true
|
||||
}
|
||||
|
||||
c := &MixedVolumeCluster{
|
||||
testingTB: t,
|
||||
profile: profile,
|
||||
weedBinary: weedBinary,
|
||||
rustVolumeBinary: rustBinary,
|
||||
baseDir: baseDir,
|
||||
configDir: configDir,
|
||||
logsDir: logsDir,
|
||||
keepLogs: keepLogs,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
volumePorts: make([]int, total),
|
||||
volumeGrpcPorts: make([]int, total),
|
||||
volumePubPorts: make([]int, total),
|
||||
isRust: isRust,
|
||||
volumeCmds: make([]*exec.Cmd, total),
|
||||
}
|
||||
|
||||
for i := 0; i < total; i++ {
|
||||
baseIdx := i * portsPerServer
|
||||
c.volumePorts[i] = ports[baseIdx]
|
||||
c.volumeGrpcPorts[i] = ports[baseIdx+1]
|
||||
if profile.SplitPublicPort {
|
||||
c.volumePubPorts[i] = ports[baseIdx+2]
|
||||
} else {
|
||||
c.volumePubPorts[i] = c.volumePorts[i] // reuse admin port
|
||||
}
|
||||
}
|
||||
|
||||
// Start master
|
||||
if err = c.startMaster(masterDataDir); err != nil {
|
||||
c.Stop()
|
||||
t.Fatalf("start master: %v", err)
|
||||
}
|
||||
helper := &Cluster{logsDir: logsDir}
|
||||
if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
|
||||
masterLog := helper.tailLog("master.log")
|
||||
c.Stop()
|
||||
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
|
||||
}
|
||||
|
||||
// Start volume servers
|
||||
for i := 0; i < total; i++ {
|
||||
volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
|
||||
if isRust[i] {
|
||||
err = c.startRustVolume(i, volumeDataDir)
|
||||
} else {
|
||||
err = c.startGoVolume(i, volumeDataDir)
|
||||
}
|
||||
if err != nil {
|
||||
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
|
||||
c.Stop()
|
||||
t.Fatalf("start volume server %d (rust=%v): %v\nlog tail:\n%s", i, isRust[i], err, logTail)
|
||||
}
|
||||
|
||||
// Rust uses /healthz, Go uses /status
|
||||
healthURL := c.VolumeAdminURL(i) + "/status"
|
||||
if isRust[i] {
|
||||
healthURL = c.VolumeAdminURL(i) + "/healthz"
|
||||
}
|
||||
if err = helper.waitForHTTP(healthURL); err != nil {
|
||||
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
|
||||
c.Stop()
|
||||
t.Fatalf("wait for volume server %d readiness: %v\nlog tail:\n%s", i, err, logTail)
|
||||
}
|
||||
if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
|
||||
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
|
||||
c.Stop()
|
||||
t.Fatalf("wait for volume server %d grpc readiness: %v\nlog tail:\n%s", i, err, logTail)
|
||||
}
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
c.Stop()
|
||||
})
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) Stop() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.cleanupOnce.Do(func() {
|
||||
for i := len(c.volumeCmds) - 1; i >= 0; i-- {
|
||||
stopProcess(c.volumeCmds[i])
|
||||
}
|
||||
stopProcess(c.masterCmd)
|
||||
if !c.keepLogs && !c.testingTB.Failed() {
|
||||
_ = os.RemoveAll(c.baseDir)
|
||||
} else if c.baseDir != "" {
|
||||
c.testingTB.Logf("mixed volume server integration logs kept at %s", c.baseDir)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) startMaster(dataDir string) error {
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-config_dir=" + c.configDir,
|
||||
"master",
|
||||
"-ip=127.0.0.1",
|
||||
"-port=" + strconv.Itoa(c.masterPort),
|
||||
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
|
||||
"-mdir=" + dataDir,
|
||||
"-peers=none",
|
||||
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
|
||||
"-defaultReplication=000",
|
||||
}
|
||||
|
||||
c.masterCmd = exec.Command(c.weedBinary, args...)
|
||||
c.masterCmd.Dir = c.baseDir
|
||||
c.masterCmd.Stdout = logFile
|
||||
c.masterCmd.Stderr = logFile
|
||||
if err = c.masterCmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return err
|
||||
}
|
||||
logFile.Close() // child inherited the fd
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
|
||||
logName := fmt.Sprintf("volume%d.log", index)
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-config_dir=" + c.configDir,
|
||||
"volume",
|
||||
"-ip=127.0.0.1",
|
||||
"-port=" + strconv.Itoa(c.volumePorts[index]),
|
||||
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
|
||||
"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
|
||||
"-dir=" + dataDir,
|
||||
"-max=16",
|
||||
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
|
||||
"-readMode=" + c.profile.ReadMode,
|
||||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
|
||||
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
|
||||
}
|
||||
if c.profile.InflightUploadTimeout > 0 {
|
||||
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
|
||||
}
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
|
||||
cmd := exec.Command(c.weedBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
if err = cmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return err
|
||||
}
|
||||
logFile.Close() // child inherited the fd
|
||||
c.volumeCmds[index] = cmd
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) startRustVolume(index int, dataDir string) error {
|
||||
logName := fmt.Sprintf("volume%d.log", index)
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := rustVolumeArgs(
|
||||
c.profile,
|
||||
c.configDir,
|
||||
c.masterPort,
|
||||
c.volumePorts[index],
|
||||
c.volumeGrpcPorts[index],
|
||||
c.volumePubPorts[index],
|
||||
dataDir,
|
||||
)
|
||||
|
||||
cmd := exec.Command(c.rustVolumeBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
if err = cmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return err
|
||||
}
|
||||
logFile.Close() // child inherited the fd
|
||||
c.volumeCmds[index] = cmd
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- accessor methods (mirror MultiVolumeCluster) ---
|
||||
|
||||
func (c *MixedVolumeCluster) MasterAddress() string {
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) MasterURL() string {
|
||||
return "http://" + c.MasterAddress()
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumeAdminAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumePorts) {
|
||||
return ""
|
||||
}
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumePublicAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumePubPorts) {
|
||||
return ""
|
||||
}
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumeGRPCAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumeGrpcPorts) {
|
||||
return ""
|
||||
}
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumeAdminURL(index int) string {
|
||||
return "http://" + c.VolumeAdminAddress(index)
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumePublicURL(index int) string {
|
||||
return "http://" + c.VolumePublicAddress(index)
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) BaseDir() string {
|
||||
return c.baseDir
|
||||
}
|
||||
|
||||
// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
|
||||
func (c *MixedVolumeCluster) VolumeServerAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumePorts) {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(index), c.volumeGrpcPorts[index])
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) IsRust(index int) bool {
|
||||
if index < 0 || index >= len(c.isRust) {
|
||||
return false
|
||||
}
|
||||
return c.isRust[index]
|
||||
}
|
||||
@@ -1,567 +0,0 @@
|
||||
package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
)
|
||||
|
||||
// deleteAndWaitForTombstone issues an HTTP DELETE for the given fid on
|
||||
// volumeURL, asserts a successful response, then polls with GET until
|
||||
// the file returns 404 (tombstone visible) or the timeout elapses.
|
||||
func deleteAndWaitForTombstone(t *testing.T, httpClient *http.Client, volumeURL, fid string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("build delete request for %s: %v", fid, err)
|
||||
}
|
||||
resp := framework.DoRequest(t, httpClient, req)
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("delete %s: expected 200 or 202, got %d", fid, resp.StatusCode)
|
||||
}
|
||||
// Poll until GET returns 404 (tombstone flushed to disk).
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
getResp := framework.ReadBytes(t, httpClient, volumeURL, fid)
|
||||
status := getResp.StatusCode
|
||||
framework.ReadAllAndClose(t, getResp)
|
||||
if status == http.StatusNotFound {
|
||||
return
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("delete %s: tombstone not visible after 5s", fid)
|
||||
}
|
||||
|
||||
// TestMixedBalanceCopyGoToRust verifies that VolumeCopy works from a Go volume
|
||||
// server to a Rust volume server. This is the core operation behind volume
|
||||
// balancing in a mixed Go+Rust cluster.
|
||||
func TestMixedBalanceCopyGoToRust(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
// server 0 = Go, server 1 = Rust
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(50)
|
||||
|
||||
// Allocate volume on Go server and upload test data
|
||||
framework.AllocateVolume(t, goClient, volumeID, "")
|
||||
|
||||
testFiles := []struct {
|
||||
key uint64
|
||||
cookie uint32
|
||||
data []byte
|
||||
}{
|
||||
{1, 0xAABBCCDD, []byte("hello from Go server")},
|
||||
{2, 0x11223344, []byte("second file for balance test")},
|
||||
{3, 0xDEADBEEF, make([]byte, 4096)}, // larger file
|
||||
}
|
||||
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, f.data)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Read source volume status before copy
|
||||
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
|
||||
}
|
||||
t.Logf("Source: dat=%d idx=%d files=%d version=%d",
|
||||
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
|
||||
sourceStatus.GetFileCount(), sourceStatus.GetVersion())
|
||||
|
||||
// Copy volume from Go (server 0) to Rust (server 1)
|
||||
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy call failed: %v", err)
|
||||
}
|
||||
|
||||
var lastAppendAtNs uint64
|
||||
for {
|
||||
resp, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv error: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
if resp.GetLastAppendAtNs() != 0 {
|
||||
lastAppendAtNs = resp.GetLastAppendAtNs()
|
||||
}
|
||||
}
|
||||
t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
|
||||
|
||||
// Verify: read volume status from Rust server
|
||||
targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
|
||||
}
|
||||
t.Logf("Target: dat=%d idx=%d files=%d version=%d",
|
||||
targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
|
||||
targetStatus.GetFileCount(), targetStatus.GetVersion())
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
|
||||
t.Fatalf("dat file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
|
||||
t.Fatalf("idx file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
|
||||
}
|
||||
if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
|
||||
t.Fatalf("file count mismatch: source=%d target=%d",
|
||||
sourceStatus.GetFileCount(), targetStatus.GetFileCount())
|
||||
}
|
||||
|
||||
// Verify data can be read from Rust server
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read %s from Rust server: expected 200, got %d", fid, resp.StatusCode)
|
||||
}
|
||||
if !bytes.Equal(body, f.data) {
|
||||
t.Fatalf("read %s from Rust server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMixedBalanceCopyRustToGo verifies that VolumeCopy works from a Rust
|
||||
// volume server to a Go volume server.
|
||||
func TestMixedBalanceCopyRustToGo(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(51)
|
||||
|
||||
// Allocate volume on Rust server and upload test data
|
||||
framework.AllocateVolume(t, rustClient, volumeID, "")
|
||||
|
||||
testFiles := []struct {
|
||||
key uint64
|
||||
cookie uint32
|
||||
data []byte
|
||||
}{
|
||||
{1, 0xAABBCCDD, []byte("hello from Rust server")},
|
||||
{2, 0x11223344, []byte("second file for reverse balance")},
|
||||
}
|
||||
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid, f.data)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
sourceStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
|
||||
}
|
||||
|
||||
// Copy volume from Rust (server 1) to Go (server 0)
|
||||
copyStream, err := goClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy call failed: %v", err)
|
||||
}
|
||||
for {
|
||||
_, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv error: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
targetStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
|
||||
}
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
|
||||
t.Fatalf("dat file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
|
||||
t.Fatalf("idx file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
|
||||
}
|
||||
if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
|
||||
t.Fatalf("file count mismatch: source=%d target=%d",
|
||||
sourceStatus.GetFileCount(), targetStatus.GetFileCount())
|
||||
}
|
||||
|
||||
// Verify data can be read from Go server
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read %s from Go server: expected 200, got %d", fid, resp.StatusCode)
|
||||
}
|
||||
if !bytes.Equal(body, f.data) {
|
||||
t.Fatalf("read %s from Go server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMixedBalanceCopyWithDeletes verifies that VolumeCopy correctly handles
|
||||
// volumes that have both active and deleted needles.
|
||||
func TestMixedBalanceCopyWithDeletes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(52)
|
||||
framework.AllocateVolume(t, goClient, volumeID, "")
|
||||
|
||||
// Upload files
|
||||
for i := uint64(1); i <= 5; i++ {
|
||||
fid := framework.NewFileID(volumeID, i, 0x12340000+uint32(i))
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("file-%d", i)))
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete some files and wait for tombstones to be visible
|
||||
for _, key := range []uint64{2, 4} {
|
||||
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
|
||||
deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0), fid)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus: %v", err)
|
||||
}
|
||||
t.Logf("Source after deletes: dat=%d idx=%d files=%d",
|
||||
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
|
||||
sourceStatus.GetFileCount())
|
||||
|
||||
// Copy volume from Go to Rust
|
||||
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy: %v", err)
|
||||
}
|
||||
var lastAppendAtNs uint64
|
||||
for {
|
||||
resp, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv error: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
if resp.GetLastAppendAtNs() != 0 {
|
||||
lastAppendAtNs = resp.GetLastAppendAtNs()
|
||||
}
|
||||
}
|
||||
if lastAppendAtNs == 0 {
|
||||
t.Fatalf("VolumeCopy did not return a lastAppendAtNs timestamp")
|
||||
}
|
||||
t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
|
||||
|
||||
targetStatusAfterCopy, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Rust: %v", err)
|
||||
}
|
||||
t.Logf("Target after copy: dat=%d idx=%d files=%d",
|
||||
targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterCopy.GetIdxFileSize(),
|
||||
targetStatusAfterCopy.GetFileCount())
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
|
||||
t.Fatalf("dat file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatusAfterCopy.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
|
||||
t.Fatalf("idx file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatusAfterCopy.GetIdxFileSize())
|
||||
}
|
||||
|
||||
// Tail from the copy checkpoint — source is unchanged (deletes happened
|
||||
// before copy), so tailing should not append any data.
|
||||
_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
|
||||
VolumeId: volumeID,
|
||||
SinceNs: lastAppendAtNs,
|
||||
IdleTimeoutSeconds: 3,
|
||||
SourceVolumeServer: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeTailReceiver: %v", err)
|
||||
}
|
||||
|
||||
targetStatusAfterTail, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus after tail: %v", err)
|
||||
}
|
||||
if targetStatusAfterTail.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
|
||||
t.Fatalf("dat grew after tail: before=%d after=%d",
|
||||
targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterTail.GetDatFileSize())
|
||||
}
|
||||
if targetStatusAfterTail.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
|
||||
t.Fatalf("idx grew after tail: before=%d after=%d",
|
||||
targetStatusAfterCopy.GetIdxFileSize(), targetStatusAfterTail.GetIdxFileSize())
|
||||
}
|
||||
|
||||
// Verify surviving files are readable from Rust
|
||||
for _, key := range []uint64{1, 3, 5} {
|
||||
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read surviving file %d from Rust: expected 200, got %d", key, resp.StatusCode)
|
||||
}
|
||||
expected := fmt.Sprintf("file-%d", key)
|
||||
if string(body) != expected {
|
||||
t.Fatalf("data mismatch for file %d: got %q want %q", key, body, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify deleted files return 404
|
||||
for _, key := range []uint64{2, 4} {
|
||||
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("read deleted file %d from Rust: expected 404, got %d", key, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMixedBalanceFullMoveGoToRust exercises the complete volume balance move
|
||||
// flow: mark readonly → copy → tail → verify sizes → delete source.
|
||||
// This mirrors the steps in balance_task.go Execute().
|
||||
func TestMixedBalanceFullMoveGoToRust(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(53)
|
||||
framework.AllocateVolume(t, goClient, volumeID, "")
|
||||
|
||||
// Upload test data
|
||||
for i := uint64(1); i <= 5; i++ {
|
||||
fid := framework.NewFileID(volumeID, i, 0xABCD0000+uint32(i))
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("balance-move-file-%d", i)))
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Step 1: Copy volume to Rust server (source stays writable so we can
|
||||
// delete after copy to exercise the tail tombstone path)
|
||||
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy: %v", err)
|
||||
}
|
||||
var lastAppendAtNs uint64
|
||||
for {
|
||||
resp, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
if resp.GetLastAppendAtNs() != 0 {
|
||||
lastAppendAtNs = resp.GetLastAppendAtNs()
|
||||
}
|
||||
}
|
||||
t.Logf("Copy done, lastAppendAtNs=%d", lastAppendAtNs)
|
||||
|
||||
// Step 2: Delete file 3 on the source AFTER copy. This creates a
|
||||
// tombstone needle that the tail step must propagate to the Rust server.
|
||||
deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0),
|
||||
framework.NewFileID(volumeID, 3, 0xABCD0003))
|
||||
|
||||
// Step 3: Mark source readonly so no further writes arrive during tail
|
||||
_, err = goClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("mark readonly: %v", err)
|
||||
}
|
||||
|
||||
// Read source status (the reference for post-tail verification)
|
||||
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read source status: %v", err)
|
||||
}
|
||||
t.Logf("Source: dat=%d idx=%d files=%d",
|
||||
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
|
||||
sourceStatus.GetFileCount())
|
||||
|
||||
// Step 4: Tail for updates — this must pick up the delete tombstone
|
||||
_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
|
||||
VolumeId: volumeID,
|
||||
SinceNs: lastAppendAtNs,
|
||||
IdleTimeoutSeconds: 5,
|
||||
SourceVolumeServer: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeTailReceiver: %v", err)
|
||||
}
|
||||
|
||||
// Step 5: Verify dat/idx sizes match after tail.
|
||||
// We compare file sizes (byte-level correctness) rather than file_count
|
||||
// because the tail writes the tombstone via the write path (which
|
||||
// increments file_count) while the source used the delete path (which
|
||||
// only increments deletion_count). This is consistent with Go behavior.
|
||||
targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read target status: %v", err)
|
||||
}
|
||||
t.Logf("Target: dat=%d idx=%d files=%d",
|
||||
targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
|
||||
targetStatus.GetFileCount())
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
|
||||
t.Fatalf("dat size mismatch after tail: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
|
||||
t.Fatalf("idx size mismatch after tail: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
|
||||
}
|
||||
|
||||
// Step 6: Delete volume from source
|
||||
_, err = goClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("delete source volume: %v", err)
|
||||
}
|
||||
|
||||
// Verify source volume is gone
|
||||
_, err = goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error reading deleted source volume, got nil")
|
||||
}
|
||||
t.Logf("Source volume deleted successfully (error as expected: %v)", err)
|
||||
|
||||
// Verify all surviving data is readable from Rust (the new home)
|
||||
for _, key := range []uint64{1, 2, 4, 5} {
|
||||
fid := framework.NewFileID(volumeID, key, 0xABCD0000+uint32(key))
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read file %d from Rust after move: expected 200, got %d", key, resp.StatusCode)
|
||||
}
|
||||
expected := fmt.Sprintf("balance-move-file-%d", key)
|
||||
if string(body) != expected {
|
||||
t.Fatalf("data mismatch for file %d after move: got %q want %q", key, body, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify deleted file is 404 on Rust (tombstone propagated via tail)
|
||||
{
|
||||
fid := framework.NewFileID(volumeID, 3, 0xABCD0003)
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("deleted file 3 on Rust: expected 404, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Full balance move completed: volume %d moved from Go to Rust, source purged", volumeID)
|
||||
}
|
||||
@@ -45,7 +45,6 @@ type ObjectStoreUser struct {
|
||||
SecretKey string `json:"secret_key"`
|
||||
Permissions []string `json:"permissions"`
|
||||
PolicyNames []string `json:"policy_names"`
|
||||
IsStatic bool `json:"is_static"` // loaded from static config file, not editable
|
||||
}
|
||||
|
||||
type ObjectStoreUsersData struct {
|
||||
|
||||
@@ -867,24 +867,6 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
|
||||
})
|
||||
}
|
||||
|
||||
// IsStaticUser checks if a user is a static identity by loading the
|
||||
// configuration from the credential manager and checking the IsStatic flag.
|
||||
func (s *AdminServer) IsStaticUser(username string) bool {
|
||||
if s.credentialManager == nil {
|
||||
return false
|
||||
}
|
||||
s3cfg, err := s.credentialManager.LoadConfiguration(context.Background())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, ident := range s3cfg.Identities {
|
||||
if ident.Name == username {
|
||||
return ident.IsStatic
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetObjectStoreUsers retrieves object store users from identity.json
|
||||
func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) {
|
||||
if s.credentialManager == nil {
|
||||
@@ -908,7 +890,6 @@ func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUse
|
||||
user := ObjectStoreUser{
|
||||
Username: identity.Name,
|
||||
Permissions: identity.Actions,
|
||||
IsStatic: identity.IsStatic,
|
||||
}
|
||||
|
||||
// Set email from account if available
|
||||
|
||||
@@ -1091,7 +1091,7 @@ func (cp *ConfigPersistence) loadTaskStateLocked(taskID string) (*maintenance.Ma
|
||||
// Convert protobuf to maintenance task
|
||||
task := cp.protobufToMaintenanceTask(taskStateFile.Task)
|
||||
|
||||
glog.V(3).Infof("Loaded task state for task %s from %s", taskID, taskFilePath)
|
||||
glog.V(2).Infof("Loaded task state for task %s from %s", taskID, taskFilePath)
|
||||
return task, nil
|
||||
}
|
||||
|
||||
@@ -1135,43 +1135,6 @@ func (cp *ConfigPersistence) loadAllTaskStatesLocked() ([]*maintenance.Maintenan
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// DeleteAllTaskStates removes all task state .pb files from disk without reading them.
|
||||
// Used at startup to clean up stale files from previous runs — the scanner will
|
||||
// re-detect any tasks that are still needed from live cluster state.
|
||||
func (cp *ConfigPersistence) DeleteAllTaskStates() error {
|
||||
cp.tasksMu.Lock()
|
||||
defer cp.tasksMu.Unlock()
|
||||
|
||||
if cp.dataDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
tasksDir := filepath.Join(cp.dataDir, TasksSubdir)
|
||||
entries, err := os.ReadDir(tasksDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to read tasks directory: %w", err)
|
||||
}
|
||||
|
||||
var removed int
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && filepath.Ext(entry.Name()) == ".pb" {
|
||||
if err := os.Remove(filepath.Join(tasksDir, entry.Name())); err != nil && !os.IsNotExist(err) {
|
||||
glog.Warningf("Failed to delete task file %s: %v", entry.Name(), err)
|
||||
} else {
|
||||
removed++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if removed > 0 {
|
||||
glog.Infof("Cleaned up %d stale task files from disk", removed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteTaskState removes a task state file from disk
|
||||
func (cp *ConfigPersistence) DeleteTaskState(taskID string) error {
|
||||
cp.tasksMu.Lock()
|
||||
|
||||
@@ -175,7 +175,7 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails,
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Get user using credential manager (resolves static users via filer gRPC)
|
||||
// Get user using credential manager
|
||||
identity, err := s.credentialManager.GetUser(ctx, username)
|
||||
if err != nil {
|
||||
if err == credential.ErrUserNotFound {
|
||||
|
||||
@@ -93,11 +93,6 @@ func (h *UserHandlers) UpdateUser(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if h.adminServer.IsStaticUser(username) {
|
||||
writeJSONError(w, http.StatusForbidden, "Cannot modify static user "+username+" (loaded from config file)")
|
||||
return
|
||||
}
|
||||
|
||||
var req dash.UpdateUserRequest
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
@@ -125,11 +120,6 @@ func (h *UserHandlers) DeleteUser(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if h.adminServer.IsStaticUser(username) {
|
||||
writeJSONError(w, http.StatusForbidden, "Cannot delete static user "+username+" (loaded from config file)")
|
||||
return
|
||||
}
|
||||
|
||||
err := h.adminServer.DeleteObjectStoreUser(username)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to delete user %s: %v", username, err)
|
||||
|
||||
@@ -33,16 +33,80 @@ func (mq *MaintenanceQueue) SetPersistence(persistence TaskPersistence) {
|
||||
glog.V(1).Infof("Maintenance queue configured with task persistence")
|
||||
}
|
||||
|
||||
// LoadTasksFromPersistence is called on startup. Previous task states are NOT loaded
|
||||
// into memory — the maintenance scanner will re-detect current needs from the live
|
||||
// cluster state. Stale task files from previous runs are deleted from disk.
|
||||
// LoadTasksFromPersistence loads tasks from persistent storage on startup
|
||||
func (mq *MaintenanceQueue) LoadTasksFromPersistence() error {
|
||||
if mq.persistence != nil {
|
||||
if err := mq.persistence.DeleteAllTaskStates(); err != nil {
|
||||
glog.Warningf("Failed to clean up old task files: %v", err)
|
||||
if mq.persistence == nil {
|
||||
glog.V(1).Infof("No task persistence configured, skipping task loading")
|
||||
return nil
|
||||
}
|
||||
|
||||
mq.mutex.Lock()
|
||||
defer mq.mutex.Unlock()
|
||||
|
||||
glog.Infof("Loading tasks from persistence...")
|
||||
|
||||
tasks, err := mq.persistence.LoadAllTaskStates()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load task states: %w", err)
|
||||
}
|
||||
|
||||
glog.Infof("DEBUG LoadTasksFromPersistence: Found %d tasks in persistence", len(tasks))
|
||||
|
||||
// Reset task maps
|
||||
mq.tasks = make(map[string]*MaintenanceTask)
|
||||
mq.pendingTasks = make([]*MaintenanceTask, 0)
|
||||
|
||||
// Load tasks by status
|
||||
for _, task := range tasks {
|
||||
glog.Infof("DEBUG LoadTasksFromPersistence: Loading task %s (type: %s, status: %s, scheduled: %v)", task.ID, task.Type, task.Status, task.ScheduledAt)
|
||||
mq.tasks[task.ID] = task
|
||||
|
||||
switch task.Status {
|
||||
case TaskStatusPending:
|
||||
glog.Infof("DEBUG LoadTasksFromPersistence: Adding task %s to pending queue", task.ID)
|
||||
mq.pendingTasks = append(mq.pendingTasks, task)
|
||||
case TaskStatusAssigned, TaskStatusInProgress:
|
||||
// For assigned/in-progress tasks, we need to check if the worker is still available
|
||||
// If not, we should fail them and make them eligible for retry
|
||||
if task.WorkerID != "" {
|
||||
if _, exists := mq.workers[task.WorkerID]; !exists {
|
||||
glog.Warningf("Task %s was assigned to unavailable worker %s, marking as failed", task.ID, task.WorkerID)
|
||||
task.Status = TaskStatusFailed
|
||||
task.Error = "Worker unavailable after restart"
|
||||
completedTime := time.Now()
|
||||
task.CompletedAt = &completedTime
|
||||
|
||||
// Check if it should be retried
|
||||
if task.RetryCount < task.MaxRetries {
|
||||
task.RetryCount++
|
||||
task.Status = TaskStatusPending
|
||||
task.WorkerID = ""
|
||||
task.StartedAt = nil
|
||||
task.CompletedAt = nil
|
||||
task.Error = ""
|
||||
task.ScheduledAt = time.Now().Add(1 * time.Minute) // Retry after restart delay
|
||||
glog.Infof("DEBUG LoadTasksFromPersistence: Retrying task %s, adding to pending queue", task.ID)
|
||||
mq.pendingTasks = append(mq.pendingTasks, task)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sync task with ActiveTopology for capacity tracking
|
||||
if mq.integration != nil {
|
||||
mq.integration.SyncTask(task)
|
||||
}
|
||||
}
|
||||
glog.Infof("Task queue initialized (previous tasks will be re-detected by scanner)")
|
||||
|
||||
// Sort pending tasks by priority and schedule time
|
||||
sort.Slice(mq.pendingTasks, func(i, j int) bool {
|
||||
if mq.pendingTasks[i].Priority != mq.pendingTasks[j].Priority {
|
||||
return mq.pendingTasks[i].Priority > mq.pendingTasks[j].Priority
|
||||
}
|
||||
return mq.pendingTasks[i].ScheduledAt.Before(mq.pendingTasks[j].ScheduledAt)
|
||||
})
|
||||
|
||||
glog.Infof("Loaded %d tasks from persistence (%d pending)", len(tasks), len(mq.pendingTasks))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -55,14 +119,6 @@ func (mq *MaintenanceQueue) saveTaskState(task *MaintenanceTask) {
|
||||
}
|
||||
}
|
||||
|
||||
func (mq *MaintenanceQueue) deleteTaskState(taskID string) {
|
||||
if mq.persistence != nil {
|
||||
if err := mq.persistence.DeleteTaskState(taskID); err != nil {
|
||||
glog.V(2).Infof("Failed to delete task state for %s: %v", taskID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupCompletedTasks removes old completed tasks beyond the retention limit
|
||||
func (mq *MaintenanceQueue) cleanupCompletedTasks() {
|
||||
if mq.persistence != nil {
|
||||
@@ -72,24 +128,10 @@ func (mq *MaintenanceQueue) cleanupCompletedTasks() {
|
||||
}
|
||||
}
|
||||
|
||||
const MaxTasksPerType = 100
|
||||
|
||||
// AddTask adds a new maintenance task to the queue with deduplication
|
||||
func (mq *MaintenanceQueue) AddTask(task *MaintenanceTask) {
|
||||
mq.mutex.Lock()
|
||||
|
||||
// Enforce per-type capacity limit (only counting active tasks)
|
||||
if mq.countActiveTasksByType(task.Type) >= MaxTasksPerType {
|
||||
// Purge terminal tasks first, then recheck
|
||||
mq.purgeTerminalTasksLocked()
|
||||
if mq.countActiveTasksByType(task.Type) >= MaxTasksPerType {
|
||||
mq.mutex.Unlock()
|
||||
glog.V(1).Infof("Task skipped (type %s at capacity %d): volume %d on %s",
|
||||
task.Type, MaxTasksPerType, task.VolumeID, task.Server)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Enforce one queued/active task per volume (across all task types).
|
||||
if mq.hasQueuedOrActiveTaskForVolume(task.VolumeID) {
|
||||
mq.mutex.Unlock()
|
||||
@@ -155,30 +197,6 @@ func (mq *MaintenanceQueue) AddTask(task *MaintenanceTask) {
|
||||
taskSnapshot.ID, taskSnapshot.Type, taskSnapshot.VolumeID, taskSnapshot.Server, taskSnapshot.Priority, scheduleInfo, taskSnapshot.Reason)
|
||||
}
|
||||
|
||||
// countActiveTasksByType returns the number of active (non-terminal) tasks of a given type. Caller must hold mq.mutex.
|
||||
func (mq *MaintenanceQueue) countActiveTasksByType(taskType MaintenanceTaskType) int {
|
||||
count := 0
|
||||
for _, t := range mq.tasks {
|
||||
if t.Type == taskType {
|
||||
switch t.Status {
|
||||
case TaskStatusPending, TaskStatusAssigned, TaskStatusInProgress:
|
||||
count++
|
||||
}
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// purgeTerminalTasksLocked removes terminal tasks from the in-memory map. Caller must hold mq.mutex.
|
||||
func (mq *MaintenanceQueue) purgeTerminalTasksLocked() {
|
||||
for id, task := range mq.tasks {
|
||||
switch task.Status {
|
||||
case TaskStatusCompleted, TaskStatusFailed, TaskStatusCancelled:
|
||||
delete(mq.tasks, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hasQueuedOrActiveTaskForVolume checks if any pending/assigned/in-progress task already exists for this volume.
|
||||
// Caller must hold mq.mutex.
|
||||
func (mq *MaintenanceQueue) hasQueuedOrActiveTaskForVolume(volumeID uint32) bool {
|
||||
@@ -255,9 +273,6 @@ func (mq *MaintenanceQueue) CancelPendingTasksByType(taskType MaintenanceTaskTyp
|
||||
|
||||
// AddTasksFromResults converts detection results to tasks and adds them to the queue
|
||||
func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult) {
|
||||
// Purge terminal tasks from memory before adding new ones
|
||||
mq.purgeTerminalTasks()
|
||||
|
||||
for _, result := range results {
|
||||
// Validate that task has proper typed parameters
|
||||
if result.TypedParams == nil {
|
||||
@@ -282,21 +297,6 @@ func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult)
|
||||
}
|
||||
}
|
||||
|
||||
// purgeTerminalTasks removes completed/failed/cancelled tasks from memory.
|
||||
// Terminal tasks are already deleted from disk by CompleteTask, so this
|
||||
// only needs to clean up the in-memory map.
|
||||
func (mq *MaintenanceQueue) purgeTerminalTasks() {
|
||||
mq.mutex.Lock()
|
||||
before := len(mq.tasks)
|
||||
mq.purgeTerminalTasksLocked()
|
||||
purged := before - len(mq.tasks)
|
||||
mq.mutex.Unlock()
|
||||
|
||||
if purged > 0 {
|
||||
glog.V(1).Infof("Purged %d terminal tasks from memory", purged)
|
||||
}
|
||||
}
|
||||
|
||||
// GetNextTask returns the next available task for a worker
|
||||
func (mq *MaintenanceQueue) GetNextTask(workerID string, capabilities []MaintenanceTaskType) *MaintenanceTask {
|
||||
// Use read lock for initial checks and search
|
||||
@@ -570,6 +570,7 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
|
||||
}
|
||||
}
|
||||
taskStatus := task.Status
|
||||
taskCount := len(mq.tasks)
|
||||
// Snapshot task state while lock is still held to avoid data race
|
||||
var taskToSaveSnapshot *MaintenanceTask
|
||||
if taskToSave != nil {
|
||||
@@ -577,18 +578,9 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
|
||||
}
|
||||
mq.mutex.Unlock()
|
||||
|
||||
// Only persist non-terminal tasks (retries). Completed/failed tasks stay
|
||||
// in memory for the UI but are not written to disk — they would just
|
||||
// accumulate and slow down future startups.
|
||||
// Save task state to persistence outside the lock
|
||||
if taskToSaveSnapshot != nil {
|
||||
switch taskStatus {
|
||||
case TaskStatusPending:
|
||||
// Retry — save so the task survives a restart
|
||||
mq.saveTaskState(taskToSaveSnapshot)
|
||||
case TaskStatusCompleted, TaskStatusFailed, TaskStatusCancelled:
|
||||
// Terminal — delete the file if one exists from a previous state
|
||||
mq.deleteTaskState(taskToSaveSnapshot.ID)
|
||||
}
|
||||
mq.saveTaskState(taskToSaveSnapshot)
|
||||
}
|
||||
|
||||
if logFn != nil {
|
||||
@@ -599,6 +591,13 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
|
||||
if taskStatus != TaskStatusPending {
|
||||
mq.removePendingOperation(taskID)
|
||||
}
|
||||
|
||||
// Periodically cleanup old completed tasks (when total task count is a multiple of 10)
|
||||
if taskStatus == TaskStatusCompleted {
|
||||
if taskCount%10 == 0 {
|
||||
go mq.cleanupCompletedTasks()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// isNonRetriableError returns true for errors that will never succeed on retry,
|
||||
|
||||
@@ -693,11 +693,10 @@ func (m *MockPersistence) SaveTaskState(task *MaintenanceTask) error
|
||||
func (m *MockPersistence) LoadTaskState(taskID string) (*MaintenanceTask, error) { return nil, nil }
|
||||
func (m *MockPersistence) LoadAllTaskStates() ([]*MaintenanceTask, error) { return m.tasks, nil }
|
||||
func (m *MockPersistence) DeleteTaskState(taskID string) error { return nil }
|
||||
func (m *MockPersistence) DeleteAllTaskStates() error { return nil }
|
||||
func (m *MockPersistence) CleanupCompletedTasks() error { return nil }
|
||||
func (m *MockPersistence) SaveTaskPolicy(taskType string, policy *TaskPolicy) error { return nil }
|
||||
|
||||
func TestMaintenanceQueue_LoadTasksStartsEmpty(t *testing.T) {
|
||||
func TestMaintenanceQueue_LoadTasksCapacitySync(t *testing.T) {
|
||||
// Setup
|
||||
policy := &MaintenancePolicy{
|
||||
TaskPolicies: map[string]*worker_pb.TaskPolicy{
|
||||
@@ -705,25 +704,56 @@ func TestMaintenanceQueue_LoadTasksStartsEmpty(t *testing.T) {
|
||||
},
|
||||
}
|
||||
mq := NewMaintenanceQueue(policy)
|
||||
integration := NewMaintenanceIntegration(mq, policy)
|
||||
mq.SetIntegration(integration)
|
||||
at := integration.GetActiveTopology()
|
||||
|
||||
// Setup mock persistence with tasks — these should NOT be loaded
|
||||
topologyInfo := &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{
|
||||
{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{
|
||||
{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: []*master_pb.DataNodeInfo{
|
||||
{
|
||||
Id: "server1",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd1": {DiskId: 1, VolumeCount: 1, MaxVolumeCount: 10},
|
||||
"hdd2": {DiskId: 2, VolumeCount: 0, MaxVolumeCount: 10},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
at.UpdateTopology(topologyInfo)
|
||||
|
||||
// Setup mock persistence with a pending task
|
||||
taskID := "load_test_123"
|
||||
mockTask := &MaintenanceTask{
|
||||
ID: "old_task_123",
|
||||
ID: taskID,
|
||||
Type: "balance",
|
||||
Status: TaskStatusPending,
|
||||
TypedParams: &worker_pb.TaskParams{
|
||||
TaskId: taskID,
|
||||
Sources: []*worker_pb.TaskSource{{Node: "server1", DiskId: 1}},
|
||||
Targets: []*worker_pb.TaskTarget{{Node: "server1", DiskId: 2}},
|
||||
},
|
||||
}
|
||||
mq.SetPersistence(&MockPersistence{tasks: []*MaintenanceTask{mockTask}})
|
||||
|
||||
// LoadTasksFromPersistence should be a no-op — scanner will re-detect
|
||||
// Load tasks
|
||||
err := mq.LoadTasksFromPersistence()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTasksFromPersistence failed: %v", err)
|
||||
t.Fatalf("Failed to load tasks: %v", err)
|
||||
}
|
||||
|
||||
// Queue should be empty — tasks will be re-detected by scanner
|
||||
stats := mq.GetStats()
|
||||
if stats.TotalTasks != 0 {
|
||||
t.Errorf("Expected 0 tasks after startup, got %d", stats.TotalTasks)
|
||||
// Verify capacity is reserved in ActiveTopology after loading (9 left)
|
||||
if at.GetEffectiveAvailableCapacity("server1", 2) != 9 {
|
||||
t.Errorf("Expected capacity 9 after loading tasks, got %d", at.GetEffectiveAvailableCapacity("server1", 2))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,18 +32,21 @@ func NewMaintenanceScanner(adminClient AdminClient, policy *MaintenancePolicy, q
|
||||
|
||||
// ScanForMaintenanceTasks analyzes the cluster and generates maintenance tasks
|
||||
func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult, error) {
|
||||
// Get volume health metrics directly in task-system format, along with topology info
|
||||
taskMetrics, topologyInfo, err := ms.getVolumeHealthMetrics()
|
||||
// Get volume health metrics
|
||||
volumeMetrics, err := ms.getVolumeHealthMetrics()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get volume health metrics: %w", err)
|
||||
}
|
||||
|
||||
// Use task system for all task types
|
||||
if ms.integration != nil {
|
||||
// Convert metrics to task system format
|
||||
taskMetrics := ms.convertToTaskMetrics(volumeMetrics)
|
||||
|
||||
// Update topology information for complete cluster view (including empty servers)
|
||||
// This must happen before task detection to ensure EC placement can consider all servers
|
||||
if topologyInfo != nil {
|
||||
if err := ms.integration.UpdateTopologyInfo(topologyInfo); err != nil {
|
||||
if ms.lastTopologyInfo != nil {
|
||||
if err := ms.integration.UpdateTopologyInfo(ms.lastTopologyInfo); err != nil {
|
||||
glog.Errorf("Failed to update topology info for empty servers: %v", err)
|
||||
// Don't fail the scan - continue with just volume-bearing servers
|
||||
} else {
|
||||
@@ -67,12 +70,9 @@ func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult,
|
||||
return []*TaskDetectionResult{}, nil
|
||||
}
|
||||
|
||||
// getVolumeHealthMetrics collects health information for all volumes.
|
||||
// Returns metrics in task-system format directly (no intermediate copy) and
|
||||
// the topology info for updating the active topology.
|
||||
func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMetrics, *master_pb.TopologyInfo, error) {
|
||||
var metrics []*types.VolumeHealthMetrics
|
||||
var topologyInfo *master_pb.TopologyInfo
|
||||
// getVolumeHealthMetrics collects health information for all volumes
|
||||
func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics, error) {
|
||||
var metrics []*VolumeHealthMetrics
|
||||
|
||||
glog.V(1).Infof("Collecting volume health metrics from master")
|
||||
err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
|
||||
@@ -89,28 +89,30 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMet
|
||||
|
||||
volumeSizeLimitBytes := uint64(resp.VolumeSizeLimitMb) * 1024 * 1024 // Convert MB to bytes
|
||||
|
||||
// Track node counts for summary logging (avoid accumulating full ID slices)
|
||||
var totalNodes, nodesWithVolumes, nodesWithoutVolumes int
|
||||
// Track all nodes discovered in topology
|
||||
var allNodesInTopology []string
|
||||
var nodesWithVolumes []string
|
||||
var nodesWithoutVolumes []string
|
||||
|
||||
for _, dc := range resp.TopologyInfo.DataCenterInfos {
|
||||
glog.V(3).Infof("Processing datacenter: %s", dc.Id)
|
||||
glog.V(2).Infof("Processing datacenter: %s", dc.Id)
|
||||
for _, rack := range dc.RackInfos {
|
||||
glog.V(3).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
|
||||
glog.V(2).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
|
||||
for _, node := range rack.DataNodeInfos {
|
||||
totalNodes++
|
||||
glog.V(3).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
|
||||
allNodesInTopology = append(allNodesInTopology, node.Id)
|
||||
glog.V(2).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
|
||||
|
||||
hasVolumes := false
|
||||
// Process each disk on this node
|
||||
for diskType, diskInfo := range node.DiskInfos {
|
||||
if len(diskInfo.VolumeInfos) > 0 {
|
||||
hasVolumes = true
|
||||
glog.V(3).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
|
||||
glog.V(2).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
|
||||
}
|
||||
|
||||
// Process volumes on this specific disk
|
||||
for _, volInfo := range diskInfo.VolumeInfos {
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
metric := &VolumeHealthMetrics{
|
||||
VolumeID: volInfo.Id,
|
||||
Server: node.Id,
|
||||
ServerAddress: node.Address,
|
||||
@@ -136,7 +138,7 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMet
|
||||
}
|
||||
metric.Age = time.Since(metric.LastModified)
|
||||
|
||||
glog.V(4).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
|
||||
glog.V(3).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
|
||||
metric.VolumeID, metric.Server, metric.DiskType, metric.DiskId, metric.Size, volumeSizeLimitBytes, metric.FullnessRatio)
|
||||
|
||||
metrics = append(metrics, metric)
|
||||
@@ -144,27 +146,29 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMet
|
||||
}
|
||||
|
||||
if hasVolumes {
|
||||
nodesWithVolumes++
|
||||
nodesWithVolumes = append(nodesWithVolumes, node.Id)
|
||||
} else {
|
||||
nodesWithoutVolumes++
|
||||
nodesWithoutVolumes = append(nodesWithoutVolumes, node.Id)
|
||||
glog.V(1).Infof("Volume server %s found in topology but has no volumes", node.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
glog.Infof("Topology discovery: %d volume servers (%d with volumes, %d without)",
|
||||
totalNodes, nodesWithVolumes, nodesWithoutVolumes)
|
||||
glog.Infof("Topology discovery complete:")
|
||||
glog.Infof(" - Total volume servers in topology: %d (%v)", len(allNodesInTopology), allNodesInTopology)
|
||||
glog.Infof(" - Volume servers with volumes: %d (%v)", len(nodesWithVolumes), nodesWithVolumes)
|
||||
glog.Infof(" - Volume servers without volumes: %d (%v)", len(nodesWithoutVolumes), nodesWithoutVolumes)
|
||||
|
||||
// Return topology info as a local value (not retained on the scanner struct)
|
||||
topologyInfo = resp.TopologyInfo
|
||||
// Store topology info for volume shard tracker
|
||||
ms.lastTopologyInfo = resp.TopologyInfo
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get volume health metrics: %v", err)
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Successfully collected metrics for %d actual volumes with disk ID information", len(metrics))
|
||||
@@ -172,13 +176,13 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMet
|
||||
// Count actual replicas and identify EC volumes
|
||||
ms.enrichVolumeMetrics(metrics)
|
||||
|
||||
return metrics, topologyInfo, nil
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
// enrichVolumeMetrics adds additional information like replica counts
|
||||
func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*types.VolumeHealthMetrics) {
|
||||
func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*VolumeHealthMetrics) {
|
||||
// Group volumes by ID to count replicas
|
||||
volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
|
||||
volumeGroups := make(map[uint32][]*VolumeHealthMetrics)
|
||||
for _, metric := range metrics {
|
||||
volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
|
||||
}
|
||||
@@ -189,9 +193,41 @@ func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*types.VolumeHealthM
|
||||
for _, replica := range replicas {
|
||||
replica.ReplicaCount = replicaCount
|
||||
}
|
||||
glog.V(4).Infof("Volume %d has %d replicas", volumeID, replicaCount)
|
||||
glog.V(3).Infof("Volume %d has %d replicas", volumeID, replicaCount)
|
||||
}
|
||||
|
||||
// TODO: Identify EC volumes by checking volume structure
|
||||
// This would require querying volume servers for EC shard information
|
||||
}
|
||||
|
||||
// convertToTaskMetrics converts existing volume metrics to task system format
|
||||
func (ms *MaintenanceScanner) convertToTaskMetrics(metrics []*VolumeHealthMetrics) []*types.VolumeHealthMetrics {
|
||||
var simplified []*types.VolumeHealthMetrics
|
||||
|
||||
for _, metric := range metrics {
|
||||
simplified = append(simplified, &types.VolumeHealthMetrics{
|
||||
VolumeID: metric.VolumeID,
|
||||
Server: metric.Server,
|
||||
ServerAddress: metric.ServerAddress,
|
||||
DiskType: metric.DiskType,
|
||||
DiskId: metric.DiskId,
|
||||
DataCenter: metric.DataCenter,
|
||||
Rack: metric.Rack,
|
||||
Collection: metric.Collection,
|
||||
Size: metric.Size,
|
||||
DeletedBytes: metric.DeletedBytes,
|
||||
GarbageRatio: metric.GarbageRatio,
|
||||
LastModified: metric.LastModified,
|
||||
Age: metric.Age,
|
||||
ReplicaCount: metric.ReplicaCount,
|
||||
ExpectedReplicas: metric.ExpectedReplicas,
|
||||
IsReadOnly: metric.IsReadOnly,
|
||||
HasRemoteCopy: metric.HasRemoteCopy,
|
||||
IsECVolume: metric.IsECVolume,
|
||||
FullnessRatio: metric.FullnessRatio,
|
||||
})
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Converted %d volume metrics with disk ID information for task detection", len(simplified))
|
||||
return simplified
|
||||
}
|
||||
|
||||
@@ -130,7 +130,6 @@ type TaskPersistence interface {
|
||||
LoadTaskState(taskID string) (*MaintenanceTask, error)
|
||||
LoadAllTaskStates() ([]*MaintenanceTask, error)
|
||||
DeleteTaskState(taskID string) error
|
||||
DeleteAllTaskStates() error
|
||||
CleanupCompletedTasks() error
|
||||
|
||||
// Policy persistence
|
||||
@@ -207,11 +206,12 @@ type MaintenanceQueue struct {
|
||||
|
||||
// MaintenanceScanner analyzes the cluster and generates maintenance tasks
|
||||
type MaintenanceScanner struct {
|
||||
adminClient AdminClient
|
||||
policy *MaintenancePolicy
|
||||
queue *MaintenanceQueue
|
||||
lastScan map[MaintenanceTaskType]time.Time
|
||||
integration *MaintenanceIntegration
|
||||
adminClient AdminClient
|
||||
policy *MaintenancePolicy
|
||||
queue *MaintenanceQueue
|
||||
lastScan map[MaintenanceTaskType]time.Time
|
||||
integration *MaintenanceIntegration
|
||||
lastTopologyInfo *master_pb.TopologyInfo
|
||||
}
|
||||
|
||||
// TaskDetectionResult represents the result of scanning for maintenance needs
|
||||
|
||||
@@ -87,7 +87,7 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
|
||||
}
|
||||
|
||||
diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
|
||||
glog.V(3).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
|
||||
glog.V(2).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
|
||||
diskKey, nodeInfo.Id, diskInfo.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, diskInfo.VolumeCount, diskInfo.MaxVolumeCount)
|
||||
node.disks[diskInfo.DiskId] = disk
|
||||
at.disks[diskKey] = disk
|
||||
|
||||
@@ -125,9 +125,6 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
<div class="d-flex align-items-center">
|
||||
<i class="fas fa-user me-2 text-muted"></i>
|
||||
<strong>{user.Username}</strong>
|
||||
if user.IsStatic {
|
||||
<span class="badge bg-secondary ms-2" title="Loaded from config file (read-only)">static</span>
|
||||
}
|
||||
</div>
|
||||
</td>
|
||||
<td>{user.Email}</td>
|
||||
@@ -136,28 +133,24 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
</td>
|
||||
<td>
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
<button type="button" class="btn btn-outline-info"
|
||||
<button type="button" class="btn btn-outline-info"
|
||||
data-action="show-user-details" data-username={ user.Username }>
|
||||
<i class="fas fa-info-circle"></i>
|
||||
</button>
|
||||
if !user.IsStatic {
|
||||
<button type="button" class="btn btn-outline-primary"
|
||||
data-action="edit-user" data-username={ user.Username }>
|
||||
<i class="fas fa-edit"></i>
|
||||
</button>
|
||||
}
|
||||
if user.Username != "anonymous" && !user.IsStatic {
|
||||
<button type="button" class="btn btn-outline-primary"
|
||||
data-action="edit-user" data-username={ user.Username }>
|
||||
<i class="fas fa-edit"></i>
|
||||
</button>
|
||||
if user.Username != "anonymous" {
|
||||
<button type="button" class="btn btn-outline-secondary"
|
||||
data-action="manage-access-keys" data-username={ user.Username }>
|
||||
<i class="fas fa-key"></i>
|
||||
</button>
|
||||
}
|
||||
if !user.IsStatic {
|
||||
<button type="button" class="btn btn-outline-danger"
|
||||
data-action="delete-user" data-username={ user.Username }>
|
||||
<i class="fas fa-trash"></i>
|
||||
</button>
|
||||
}
|
||||
<button type="button" class="btn btn-outline-danger"
|
||||
data-action="delete-user" data-username={ user.Username }>
|
||||
<i class="fas fa-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -49,10 +48,6 @@ type AdminOptions struct {
|
||||
dataDir *string
|
||||
icebergPort *int
|
||||
urlPrefix *string
|
||||
debug *bool
|
||||
debugPort *int
|
||||
cpuProfile *string
|
||||
memProfile *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -69,10 +64,6 @@ func init() {
|
||||
a.readOnlyPassword = cmdAdmin.Flag.String("readOnlyPassword", "", "read-only user password (optional, for view-only access; requires adminPassword to be set)")
|
||||
a.icebergPort = cmdAdmin.Flag.Int("iceberg.port", 8181, "Iceberg REST Catalog port (0 to hide in UI)")
|
||||
a.urlPrefix = cmdAdmin.Flag.String("urlPrefix", "", "URL path prefix when running behind a reverse proxy under a subdirectory (e.g. /seaweedfs)")
|
||||
a.debug = cmdAdmin.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
|
||||
a.debugPort = cmdAdmin.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
a.cpuProfile = cmdAdmin.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
a.memProfile = cmdAdmin.Flag.String("memprofile", "", "memory profile output file")
|
||||
}
|
||||
|
||||
var cmdAdmin = &Command{
|
||||
@@ -149,16 +140,6 @@ var cmdAdmin = &Command{
|
||||
- All static assets, API endpoints, and navigation links will use the prefix
|
||||
- Session cookies are scoped to the prefix path
|
||||
|
||||
Debugging and Profiling:
|
||||
- Use -debug to start a pprof HTTP server for live profiling (localhost only)
|
||||
- Set -debug.port to choose the pprof port (default 6060)
|
||||
- Profiles are accessible at http://127.0.0.1:<debug.port>/debug/pprof/
|
||||
- Use -cpuprofile and -memprofile to write profiles to files on shutdown
|
||||
- WARNING: -debug exposes runtime internals; use only in trusted environments
|
||||
- Examples:
|
||||
weed admin -debug -debug.port=6060 -master="localhost:9333"
|
||||
weed admin -cpuprofile=cpu.prof -memprofile=mem.prof -master="localhost:9333"
|
||||
|
||||
Configuration File:
|
||||
- The security.toml file is read from ".", "$HOME/.seaweedfs/",
|
||||
"/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order
|
||||
@@ -168,12 +149,6 @@ var cmdAdmin = &Command{
|
||||
}
|
||||
|
||||
func runAdmin(cmd *Command, args []string) bool {
|
||||
if *a.debug {
|
||||
grace.StartDebugServer(*a.debugPort)
|
||||
}
|
||||
|
||||
grace.SetupProfiling(*a.cpuProfile, *a.memProfile)
|
||||
|
||||
// Load security configuration
|
||||
util.LoadSecurityConfiguration()
|
||||
|
||||
|
||||
@@ -81,7 +81,6 @@ type FilerOptions struct {
|
||||
exposeDirectoryData *bool
|
||||
tusBasePath *string
|
||||
certProvider certprovider.Provider
|
||||
s3ConfigFile *string // optional path to static S3 identity config
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -344,15 +343,6 @@ func (fo *FilerOptions) startFiler() {
|
||||
glog.V(0).Infof("Initialized credential manager: %s", credentialManager.GetStoreName())
|
||||
}
|
||||
|
||||
// Load static S3 identities from config file if specified
|
||||
if fo.s3ConfigFile != nil && *fo.s3ConfigFile != "" {
|
||||
if credentialManager != nil {
|
||||
if err := credentialManager.LoadS3ConfigFile(*fo.s3ConfigFile); err != nil {
|
||||
glog.Warningf("Failed to load S3 config file for static identities: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
|
||||
Masters: fo.masters,
|
||||
FilerGroup: *fo.filerGroup,
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
@@ -144,10 +143,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
||||
sourceFiler.ToGrpcAddress(),
|
||||
sourcePath,
|
||||
*backupOption.proxyByFiler)
|
||||
|
||||
if err := repl_util.InitializeSSEForReplication(filerSource); err != nil {
|
||||
return fmt.Errorf("SSE initialization failed: %v", err)
|
||||
}
|
||||
dataSink.SetSourceFiler(filerSource)
|
||||
|
||||
var processEventFn func(*filer_pb.SubscribeMetadataResponse) error
|
||||
|
||||
@@ -96,7 +96,7 @@ func init() {
|
||||
m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
m.metricsHttpIp = cmdMaster.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
m.raftResumeState = cmdMaster.Flag.Bool("resumeState", true, "resume previous state on start master server")
|
||||
m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
|
||||
m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft")
|
||||
@@ -208,7 +208,6 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
DataDir: util.ResolvePath(metaDir),
|
||||
Topo: ms.Topo,
|
||||
RaftResumeState: *masterOption.raftResumeState,
|
||||
SingleMaster: isSingleMaster,
|
||||
HeartbeatInterval: *masterOption.heartbeatInterval,
|
||||
ElectionTimeout: *masterOption.electionTimeout,
|
||||
RaftBootstrap: *masterOption.raftBootstrap,
|
||||
|
||||
@@ -162,7 +162,7 @@ func initMiniMasterFlags() {
|
||||
miniMasterOptions.garbageThreshold = cmdMini.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
|
||||
miniMasterOptions.metricsAddress = cmdMini.Flag.String("master.metrics.address", "", "Prometheus gateway address")
|
||||
miniMasterOptions.metricsIntervalSec = cmdMini.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
miniMasterOptions.raftResumeState = cmdMini.Flag.Bool("master.resumeState", true, "resume previous state on start master server")
|
||||
miniMasterOptions.raftResumeState = cmdMini.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
|
||||
miniMasterOptions.heartbeatInterval = cmdMini.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
miniMasterOptions.electionTimeout = cmdMini.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
miniMasterOptions.raftHashicorp = cmdMini.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
@@ -820,10 +820,6 @@ func runMini(cmd *Command, args []string) bool {
|
||||
miniFilerOptions.disableHttp = miniDisableHttp
|
||||
miniMasterOptions.disableHttp = miniDisableHttp
|
||||
|
||||
// Share the S3 static identity config file with the filer so its
|
||||
// credential manager can also serve static users.
|
||||
miniFilerOptions.s3ConfigFile = miniS3Config
|
||||
|
||||
filerAddress := string(pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc))
|
||||
miniS3Options.filer = &filerAddress
|
||||
miniWebDavOptions.filer = &filerAddress
|
||||
|
||||
@@ -102,7 +102,7 @@ func init() {
|
||||
masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
|
||||
masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", true, "resume previous state on start master server")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
masterOptions.raftBootstrap = cmdServer.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
|
||||
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
@@ -372,8 +372,6 @@ func runServer(cmd *Command, args []string) bool {
|
||||
} else if *serverIamConfig != "" && *s3Options.iamConfig != *serverIamConfig {
|
||||
glog.V(0).Infof("both -s3.iam.config(%s) and -iam.config(%s) provided; using -s3.iam.config", *s3Options.iamConfig, *serverIamConfig)
|
||||
}
|
||||
// Share the S3 static identity config file with the filer
|
||||
filerOptions.s3ConfigFile = s3Options.config
|
||||
go func() {
|
||||
time.Sleep(2 * time.Second)
|
||||
s3Options.localFilerSocket = filerOptions.localSocket
|
||||
|
||||
@@ -3,18 +3,14 @@ package credential
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
)
|
||||
|
||||
// FilerAddressSetter is an interface for credential stores that need a dynamic filer address
|
||||
@@ -25,15 +21,6 @@ type FilerAddressSetter interface {
|
||||
// CredentialManager manages user credentials using a configurable store
|
||||
type CredentialManager struct {
|
||||
Store CredentialStore
|
||||
// staticMu protects staticIdentities and staticNames, which are written
|
||||
// by SetStaticIdentities (startup + config reload) and read concurrently
|
||||
// by LoadConfiguration, SaveConfiguration, GetStaticUsernames, and IsStaticIdentity.
|
||||
staticMu sync.RWMutex
|
||||
// staticIdentities holds identities loaded from a static config file (-s3.config).
|
||||
// These are included in LoadConfiguration so that listing operations
|
||||
// return all configured identities, not just dynamic ones from the store.
|
||||
staticIdentities []*iam_pb.Identity
|
||||
staticNames map[string]bool
|
||||
}
|
||||
|
||||
// NewCredentialManager creates a new credential manager with the specified store
|
||||
@@ -87,97 +74,13 @@ func (cm *CredentialManager) GetStoreName() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// SetStaticIdentities registers identities loaded from a static config file.
|
||||
// These identities are included in LoadConfiguration and ListUsers results
|
||||
// but are never persisted to the dynamic store.
|
||||
func (cm *CredentialManager) SetStaticIdentities(identities []*iam_pb.Identity) {
|
||||
filtered := make([]*iam_pb.Identity, 0, len(identities))
|
||||
names := make(map[string]bool, len(identities))
|
||||
for _, ident := range identities {
|
||||
if ident != nil {
|
||||
filtered = append(filtered, ident)
|
||||
names[ident.Name] = true
|
||||
}
|
||||
}
|
||||
cm.staticMu.Lock()
|
||||
cm.staticIdentities = filtered
|
||||
cm.staticNames = names
|
||||
cm.staticMu.Unlock()
|
||||
}
|
||||
|
||||
// IsStaticIdentity returns true if the named identity was loaded from static config.
|
||||
func (cm *CredentialManager) IsStaticIdentity(name string) bool {
|
||||
cm.staticMu.RLock()
|
||||
defer cm.staticMu.RUnlock()
|
||||
return cm.staticNames[name]
|
||||
}
|
||||
|
||||
// GetStaticIdentity returns the protobuf identity for a static user, or nil.
|
||||
func (cm *CredentialManager) GetStaticIdentity(name string) *iam_pb.Identity {
|
||||
cm.staticMu.RLock()
|
||||
defer cm.staticMu.RUnlock()
|
||||
for _, ident := range cm.staticIdentities {
|
||||
if ident.Name == name {
|
||||
return ident
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStaticUsernames returns the names of all static identities.
|
||||
func (cm *CredentialManager) GetStaticUsernames() []string {
|
||||
cm.staticMu.RLock()
|
||||
defer cm.staticMu.RUnlock()
|
||||
names := make([]string, 0, len(cm.staticIdentities))
|
||||
for _, ident := range cm.staticIdentities {
|
||||
names = append(names, ident.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// LoadConfiguration loads the S3 API configuration from the store and merges
|
||||
// in any static identities so that listing operations show all users.
|
||||
// LoadConfiguration loads the S3 API configuration
|
||||
func (cm *CredentialManager) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) {
|
||||
config, err := cm.Store.LoadConfiguration(ctx)
|
||||
if err != nil {
|
||||
return config, err
|
||||
}
|
||||
// Merge static identities that are not already in the dynamic config
|
||||
cm.staticMu.RLock()
|
||||
staticIdents := cm.staticIdentities
|
||||
cm.staticMu.RUnlock()
|
||||
if len(staticIdents) > 0 {
|
||||
dynamicNames := make(map[string]bool, len(config.Identities))
|
||||
for _, ident := range config.Identities {
|
||||
dynamicNames[ident.Name] = true
|
||||
}
|
||||
for _, si := range staticIdents {
|
||||
if !dynamicNames[si.Name] {
|
||||
config.Identities = append(config.Identities, si)
|
||||
}
|
||||
}
|
||||
}
|
||||
return config, nil
|
||||
return cm.Store.LoadConfiguration(ctx)
|
||||
}
|
||||
|
||||
// SaveConfiguration saves the S3 API configuration.
|
||||
// Static identities are filtered out before saving to the store.
|
||||
// The caller's config is not mutated.
|
||||
// SaveConfiguration saves the S3 API configuration
|
||||
func (cm *CredentialManager) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {
|
||||
cm.staticMu.RLock()
|
||||
staticNames := cm.staticNames
|
||||
cm.staticMu.RUnlock()
|
||||
if len(staticNames) > 0 {
|
||||
var dynamicOnly []*iam_pb.Identity
|
||||
for _, ident := range config.Identities {
|
||||
if !staticNames[ident.Name] {
|
||||
dynamicOnly = append(dynamicOnly, ident)
|
||||
}
|
||||
}
|
||||
configCopy := *config
|
||||
configCopy.Identities = dynamicOnly
|
||||
return cm.Store.SaveConfiguration(ctx, &configCopy)
|
||||
}
|
||||
return cm.Store.SaveConfiguration(ctx, config)
|
||||
}
|
||||
|
||||
@@ -201,12 +104,7 @@ func (cm *CredentialManager) DeleteUser(ctx context.Context, username string) er
|
||||
return cm.Store.DeleteUser(ctx, username)
|
||||
}
|
||||
|
||||
// ListUsers returns usernames from the dynamic store via cm.Store.ListUsers.
|
||||
// On store error the error is returned directly without merging static entries.
|
||||
// Static identities (cm.staticIdentities) are NOT included here because
|
||||
// internal callers (e.g. DeletePolicy) look up each user in the store and
|
||||
// would fail on non-existent static entries. External callers that need the
|
||||
// full list should merge GetStaticUsernames separately.
|
||||
// ListUsers returns all usernames
|
||||
func (cm *CredentialManager) ListUsers(ctx context.Context) ([]string, error) {
|
||||
return cm.Store.ListUsers(ctx)
|
||||
}
|
||||
@@ -271,26 +169,6 @@ func (cm *CredentialManager) UpdatePolicy(ctx context.Context, name string, docu
|
||||
return cm.Store.PutPolicy(ctx, name, document)
|
||||
}
|
||||
|
||||
// LoadS3ConfigFile reads a static S3 identity config file and registers
|
||||
// the identities so they appear in LoadConfiguration and listing results.
|
||||
func (cm *CredentialManager) LoadS3ConfigFile(path string) error {
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
config := &iam_pb.S3ApiConfiguration{}
|
||||
opts := protojson.UnmarshalOptions{DiscardUnknown: true, AllowPartial: true}
|
||||
if err := opts.Unmarshal(content, config); err != nil {
|
||||
return fmt.Errorf("parse %s: %w", path, err)
|
||||
}
|
||||
for _, ident := range config.Identities {
|
||||
ident.IsStatic = true
|
||||
}
|
||||
cm.SetStaticIdentities(config.Identities)
|
||||
glog.V(1).Infof("Loaded %d static identities from %s", len(config.Identities), path)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown performs cleanup
|
||||
func (cm *CredentialManager) Shutdown() {
|
||||
if cm.Store != nil {
|
||||
|
||||
@@ -128,6 +128,8 @@ func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration, prefi
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {
|
||||
// Fail fast if the context is already cancelled to prevent orphaned metadata
|
||||
// when the originating request has been abandoned.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -245,7 +247,7 @@ func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp util.FullPath) (
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
@@ -276,7 +278,7 @@ func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath)
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
@@ -303,7 +305,7 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
@@ -437,7 +439,7 @@ func (fsw *FilerStoreWrapper) Shutdown() {
|
||||
}
|
||||
|
||||
func (fsw *FilerStoreWrapper) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
@@ -448,7 +450,7 @@ func (fsw *FilerStoreWrapper) KvGet(ctx context.Context, key []byte) (value []by
|
||||
return fsw.getDefaultStore().KvGet(ctx, key)
|
||||
}
|
||||
func (fsw *FilerStoreWrapper) KvDelete(ctx context.Context, key []byte) (err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
|
||||
@@ -82,7 +82,7 @@ func cancelledCtx() context.Context {
|
||||
// expiredCtx returns a context whose deadline has already passed.
|
||||
func expiredCtx() context.Context {
|
||||
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||
cancel() // release resources immediately as it's already expired
|
||||
_ = cancel // already expired, but keep the cancel func from leaking
|
||||
return ctx
|
||||
}
|
||||
|
||||
@@ -198,10 +198,9 @@ func TestFilerStoreWrapperReadOpsSucceedWithCancelledContext(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// RollbackTransaction must succeed even when the context is cancelled or
|
||||
// expired, because it is a cleanup operation called after failures.
|
||||
// RollbackTransaction must succeed even when the context is cancelled,
|
||||
// because it is a cleanup operation called after failures.
|
||||
func TestFilerStoreWrapperRollbackSucceedsWithCancelledContext(t *testing.T) {
|
||||
wrapper := NewFilerStoreWrapper(newStubFilerStore())
|
||||
assert.NoError(t, wrapper.RollbackTransaction(cancelledCtx()))
|
||||
assert.NoError(t, wrapper.RollbackTransaction(expiredCtx()))
|
||||
}
|
||||
|
||||
@@ -245,66 +245,6 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PrepareStreamContentWithPrefetch is like PrepareStreamContentWithThrottler but uses
|
||||
// concurrent chunk prefetching to overlap network I/O. When prefetchAhead > 1, fetch
|
||||
// goroutines establish HTTP connections to volume servers ahead of time, streaming data
|
||||
// through io.Pipe with minimal memory overhead.
|
||||
//
|
||||
// prefetchAhead controls the number of chunks fetched concurrently:
|
||||
// - 0 or 1: falls back to sequential fetching (same as PrepareStreamContentWithThrottler)
|
||||
// - 2+: uses pipe-based prefetch pipeline with that many concurrent fetches
|
||||
func PrepareStreamContentWithPrefetch(ctx context.Context, masterClient wdclient.HasLookupFileIdFunction, jwtFunc VolumeServerJwtFunction, chunks []*filer_pb.FileChunk, offset int64, size int64, downloadMaxBytesPs int64, prefetchAhead int) (DoStreamContent, error) {
|
||||
if prefetchAhead <= 1 {
|
||||
return PrepareStreamContentWithThrottler(ctx, masterClient, jwtFunc, chunks, offset, size, downloadMaxBytesPs)
|
||||
}
|
||||
|
||||
glog.V(4).InfofCtx(ctx, "prepare to stream content with prefetch=%d for chunks: %d", prefetchAhead, len(chunks))
|
||||
chunkViews := ViewFromChunks(ctx, masterClient.GetLookupFileIdFunction(), chunks, offset, size)
|
||||
|
||||
fileId2Url := make(map[string][]string)
|
||||
|
||||
for x := chunkViews.Front(); x != nil; x = x.Next {
|
||||
chunkView := x.Value
|
||||
var urlStrings []string
|
||||
var err error
|
||||
for _, backoff := range getLookupFileIdBackoffSchedule {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
urlStrings, err = masterClient.GetLookupFileIdFunction()(ctx, chunkView.FileId)
|
||||
if err == nil && len(urlStrings) > 0 {
|
||||
break
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
glog.V(4).InfofCtx(ctx, "waiting for chunk: %s", chunkView.FileId)
|
||||
timer := time.NewTimer(backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
glog.V(1).InfofCtx(ctx, "operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
|
||||
return nil, err
|
||||
} else if len(urlStrings) == 0 {
|
||||
errUrlNotFound := fmt.Errorf("operation LookupFileId %s failed, err: urls not found", chunkView.FileId)
|
||||
glog.ErrorCtx(ctx, errUrlNotFound)
|
||||
return nil, errUrlNotFound
|
||||
}
|
||||
fileId2Url[chunkView.FileId] = urlStrings
|
||||
}
|
||||
|
||||
return func(writer io.Writer) error {
|
||||
return streamChunksPrefetched(ctx, writer, chunkViews, fileId2Url, jwtFunc, masterClient, offset, size, downloadMaxBytesPs, prefetchAhead)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func StreamContent(masterClient wdclient.HasLookupFileIdFunction, writer io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int64) error {
|
||||
streamFn, err := PrepareStreamContent(masterClient, JwtForVolumeServer, chunks, offset, size)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,317 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Initialize the global HTTP client required by ReadUrlAsStream
|
||||
util_http.InitGlobalHttpClient()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// mockMasterClientForBenchmark implements HasLookupFileIdFunction and CacheInvalidator
|
||||
type mockMasterClientForBenchmark struct {
|
||||
urls map[string][]string
|
||||
}
|
||||
|
||||
func (m *mockMasterClientForBenchmark) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
|
||||
return func(ctx context.Context, fileId string) ([]string, error) {
|
||||
if urls, ok := m.urls[fileId]; ok {
|
||||
return urls, nil
|
||||
}
|
||||
return nil, fmt.Errorf("fileId %s not found", fileId)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockMasterClientForBenchmark) InvalidateCache(fileId string) {}
|
||||
|
||||
// noopJwtFunc returns empty JWT for testing
|
||||
func noopJwtFunc(fileId string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// createMockVolumeServer creates an httptest server that serves chunk data
|
||||
// with configurable per-request latency to simulate network conditions.
|
||||
// The latency is applied once per request (simulating RTT), not per byte.
|
||||
func createMockVolumeServer(chunkData map[string][]byte, latency time.Duration) *httptest.Server {
|
||||
var mu sync.RWMutex
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Simulate network latency (connection setup + RTT)
|
||||
if latency > 0 {
|
||||
time.Sleep(latency)
|
||||
}
|
||||
|
||||
// Extract fileId from path (e.g., "/1,abc123")
|
||||
path := r.URL.Path
|
||||
if strings.HasPrefix(path, "/") {
|
||||
path = path[1:]
|
||||
}
|
||||
|
||||
mu.RLock()
|
||||
data, ok := chunkData[path]
|
||||
mu.RUnlock()
|
||||
if !ok {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Handle Range header
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
var start, end int64
|
||||
fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end)
|
||||
if start >= 0 && end < int64(len(data)) && start <= end {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", end-start+1))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(data[start : end+1])
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(data)
|
||||
}))
|
||||
}
|
||||
|
||||
// benchmarkConfig holds parameters for a single benchmark scenario
|
||||
type benchmarkConfig struct {
|
||||
numChunks int
|
||||
chunkSize int
|
||||
latency time.Duration
|
||||
prefetch int // 0 = sequential
|
||||
}
|
||||
|
||||
func (c benchmarkConfig) name() string {
|
||||
name := fmt.Sprintf("chunks=%d/size=%dKB/latency=%dms",
|
||||
c.numChunks, c.chunkSize/1024, c.latency.Milliseconds())
|
||||
if c.prefetch > 0 {
|
||||
name += fmt.Sprintf("/prefetch=%d", c.prefetch)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// setupBenchmark creates mock infrastructure and returns chunks, master client, and cleanup func
|
||||
func setupBenchmark(b *testing.B, cfg benchmarkConfig) ([]*filer_pb.FileChunk, *mockMasterClientForBenchmark, func()) {
|
||||
b.Helper()
|
||||
|
||||
// Generate random chunk data
|
||||
chunkData := make(map[string][]byte, cfg.numChunks)
|
||||
chunks := make([]*filer_pb.FileChunk, cfg.numChunks)
|
||||
|
||||
for i := 0; i < cfg.numChunks; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
data := make([]byte, cfg.chunkSize)
|
||||
rand.Read(data)
|
||||
chunkData[fileId] = data
|
||||
|
||||
chunks[i] = &filer_pb.FileChunk{
|
||||
FileId: fileId,
|
||||
Offset: int64(i * cfg.chunkSize),
|
||||
Size: uint64(cfg.chunkSize),
|
||||
ModifiedTsNs: int64(i),
|
||||
Fid: &filer_pb.FileId{FileKey: uint64(i)},
|
||||
}
|
||||
}
|
||||
|
||||
// Start mock volume server
|
||||
server := createMockVolumeServer(chunkData, cfg.latency)
|
||||
|
||||
// Build URL map
|
||||
urls := make(map[string][]string, cfg.numChunks)
|
||||
for i := 0; i < cfg.numChunks; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
urls[fileId] = []string{server.URL + "/" + fileId}
|
||||
}
|
||||
|
||||
masterClient := &mockMasterClientForBenchmark{urls: urls}
|
||||
cleanup := func() { server.Close() }
|
||||
|
||||
return chunks, masterClient, cleanup
|
||||
}
|
||||
|
||||
// runSequentialBenchmark runs the current sequential streaming path
|
||||
func runSequentialBenchmark(b *testing.B, cfg benchmarkConfig) {
|
||||
chunks, masterClient, cleanup := setupBenchmark(b, cfg)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(cfg.numChunks * cfg.chunkSize)
|
||||
|
||||
b.ResetTimer()
|
||||
b.SetBytes(totalSize)
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
streamFn, err := PrepareStreamContentWithThrottler(
|
||||
context.Background(),
|
||||
masterClient,
|
||||
noopJwtFunc,
|
||||
chunks,
|
||||
0,
|
||||
totalSize,
|
||||
0, // no throttle
|
||||
)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if err := streamFn(io.Discard); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkStreamSequential benchmarks the current sequential streaming path.
|
||||
// This provides the BEFORE baseline for comparison.
|
||||
func BenchmarkStreamSequential(b *testing.B) {
|
||||
configs := []benchmarkConfig{
|
||||
// Pure throughput (no latency)
|
||||
{numChunks: 16, chunkSize: 64 * 1024, latency: 0},
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 0},
|
||||
// Moderate latency — shows RTT gap overhead
|
||||
{numChunks: 16, chunkSize: 64 * 1024, latency: 5 * time.Millisecond},
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond},
|
||||
// High latency — significant RTT overhead
|
||||
{numChunks: 16, chunkSize: 64 * 1024, latency: 20 * time.Millisecond},
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 10 * time.Millisecond},
|
||||
}
|
||||
|
||||
for _, cfg := range configs {
|
||||
b.Run(cfg.name(), func(b *testing.B) {
|
||||
runSequentialBenchmark(b, cfg)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkStreamSequentialVerify is a quick functional test that the benchmark
|
||||
// infrastructure works correctly — ensures data integrity through the pipeline.
|
||||
func BenchmarkStreamSequentialVerify(b *testing.B) {
|
||||
cfg := benchmarkConfig{numChunks: 4, chunkSize: 1024, latency: 0}
|
||||
chunks, masterClient, cleanup := setupBenchmark(b, cfg)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(cfg.numChunks * cfg.chunkSize)
|
||||
|
||||
streamFn, err := PrepareStreamContentWithThrottler(
|
||||
context.Background(),
|
||||
masterClient,
|
||||
noopJwtFunc,
|
||||
chunks,
|
||||
0,
|
||||
totalSize,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := streamFn(&buf); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
if buf.Len() != int(totalSize) {
|
||||
b.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
|
||||
}
|
||||
}
|
||||
|
||||
// runPrefetchBenchmark runs the new prefetch streaming path
|
||||
func runPrefetchBenchmark(b *testing.B, cfg benchmarkConfig) {
|
||||
chunks, masterClient, cleanup := setupBenchmark(b, cfg)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(cfg.numChunks * cfg.chunkSize)
|
||||
|
||||
b.ResetTimer()
|
||||
b.SetBytes(totalSize)
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(),
|
||||
masterClient,
|
||||
noopJwtFunc,
|
||||
chunks,
|
||||
0,
|
||||
totalSize,
|
||||
0, // no throttle
|
||||
cfg.prefetch,
|
||||
)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if err := streamFn(io.Discard); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkStreamPrefetch benchmarks the new prefetch streaming path.
|
||||
// Compare against BenchmarkStreamSequential for the AFTER measurement.
|
||||
func BenchmarkStreamPrefetch(b *testing.B) {
|
||||
configs := []benchmarkConfig{
|
||||
// Pure throughput (no latency) — should be similar to sequential
|
||||
{numChunks: 16, chunkSize: 64 * 1024, latency: 0, prefetch: 4},
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 0, prefetch: 4},
|
||||
// Moderate latency — prefetch should eliminate most RTT overhead
|
||||
{numChunks: 16, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 4},
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 4},
|
||||
// High latency — most benefit from prefetch
|
||||
{numChunks: 16, chunkSize: 64 * 1024, latency: 20 * time.Millisecond, prefetch: 4},
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 10 * time.Millisecond, prefetch: 4},
|
||||
// Vary prefetch count with moderate latency
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 2},
|
||||
{numChunks: 64, chunkSize: 64 * 1024, latency: 5 * time.Millisecond, prefetch: 8},
|
||||
}
|
||||
|
||||
for _, cfg := range configs {
|
||||
b.Run(cfg.name(), func(b *testing.B) {
|
||||
runPrefetchBenchmark(b, cfg)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// BenchmarkStreamPrefetchVerify verifies data integrity through the prefetch pipeline.
|
||||
func BenchmarkStreamPrefetchVerify(b *testing.B) {
|
||||
cfg := benchmarkConfig{numChunks: 4, chunkSize: 1024, latency: 0, prefetch: 4}
|
||||
chunks, masterClient, cleanup := setupBenchmark(b, cfg)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(cfg.numChunks * cfg.chunkSize)
|
||||
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(),
|
||||
masterClient,
|
||||
noopJwtFunc,
|
||||
chunks,
|
||||
0,
|
||||
totalSize,
|
||||
0,
|
||||
cfg.prefetch,
|
||||
)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := streamFn(&buf); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
if buf.Len() != int(totalSize) {
|
||||
b.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
|
||||
}
|
||||
}
|
||||
@@ -1,274 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/mem"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
// chunkPipeResult represents a prefetched chunk streaming through a pipe.
|
||||
// The fetch goroutine writes data into the pipeWriter; the consumer reads from pipeReader.
|
||||
type chunkPipeResult struct {
|
||||
chunkView *ChunkView
|
||||
reader *io.PipeReader
|
||||
fetchErr error // final error from fetch goroutine
|
||||
written int64 // bytes written by fetch goroutine
|
||||
done chan struct{} // closed when fetch goroutine finishes
|
||||
urlStrings []string // snapshot of URLs at dispatch time (for retry logic)
|
||||
}
|
||||
|
||||
// streamChunksPrefetched streams chunks with concurrent prefetch using io.Pipe.
|
||||
//
|
||||
// For each chunk in file order, a goroutine is launched (bounded by a semaphore)
|
||||
// that establishes an HTTP connection to the volume server and streams data through
|
||||
// an io.Pipe. The consumer reads from pipes in order, writing to the response.
|
||||
//
|
||||
// Memory usage is minimal: pipes are synchronous (no buffering), and only one
|
||||
// reusable copy buffer is allocated for the consumer.
|
||||
func streamChunksPrefetched(
|
||||
ctx context.Context,
|
||||
writer io.Writer,
|
||||
chunkViews *IntervalList[*ChunkView],
|
||||
fileId2Url map[string][]string,
|
||||
jwtFunc VolumeServerJwtFunction,
|
||||
masterClient wdclient.HasLookupFileIdFunction,
|
||||
offset int64,
|
||||
size int64,
|
||||
downloadMaxBytesPs int64,
|
||||
prefetchAhead int,
|
||||
) error {
|
||||
downloadThrottler := util.NewWriteThrottler(downloadMaxBytesPs)
|
||||
|
||||
// Create a local cancellable context so the consumer can stop the producer
|
||||
// and all in-flight fetch goroutines on error (e.g., client disconnect).
|
||||
localCtx, localCancel := context.WithCancel(ctx)
|
||||
defer localCancel()
|
||||
|
||||
// Ordered channel: one entry per chunk, in file order.
|
||||
// Capacity = prefetchAhead so the producer can run ahead.
|
||||
// Uses pointer to avoid copying the struct while fetch goroutines write to it.
|
||||
results := make(chan *chunkPipeResult, prefetchAhead)
|
||||
|
||||
// Semaphore to limit concurrent fetch goroutines (and thus HTTP connections).
|
||||
sem := make(chan struct{}, prefetchAhead)
|
||||
|
||||
// Producer: walks chunk list, launches fetch goroutines, sends results in order.
|
||||
// The producer only reads from fileId2Url (populated before streaming starts),
|
||||
// so there is no concurrent map access — the consumer never writes to it.
|
||||
var producerWg sync.WaitGroup
|
||||
producerWg.Add(1)
|
||||
go func() {
|
||||
defer producerWg.Done()
|
||||
defer close(results)
|
||||
|
||||
for x := chunkViews.Front(); x != nil; x = x.Next {
|
||||
chunkView := x.Value
|
||||
|
||||
// Check context before starting new fetch
|
||||
select {
|
||||
case <-localCtx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Acquire semaphore slot (bounds concurrent HTTP connections)
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-localCtx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
urlStrings := fileId2Url[chunkView.FileId]
|
||||
jwt := jwtFunc(chunkView.FileId)
|
||||
|
||||
result := &chunkPipeResult{
|
||||
chunkView: chunkView,
|
||||
reader: pr,
|
||||
done: make(chan struct{}),
|
||||
urlStrings: urlStrings,
|
||||
}
|
||||
|
||||
// Launch fetch goroutine
|
||||
go func(cv *ChunkView, urls []string, jwt string, pw *io.PipeWriter, res *chunkPipeResult) {
|
||||
defer func() { <-sem }() // release semaphore
|
||||
defer close(res.done)
|
||||
|
||||
written, err := retriedStreamFetchChunkData(
|
||||
localCtx, pw, urls, jwt,
|
||||
cv.CipherKey, cv.IsGzipped, cv.IsFullChunk(),
|
||||
cv.OffsetInChunk, int(cv.ViewSize),
|
||||
)
|
||||
res.written = written
|
||||
res.fetchErr = err
|
||||
|
||||
if err != nil {
|
||||
pw.CloseWithError(err)
|
||||
} else {
|
||||
pw.Close()
|
||||
}
|
||||
}(chunkView, urlStrings, jwt, pw, result)
|
||||
|
||||
// Send result to consumer (blocks if channel full, back-pressuring producer)
|
||||
select {
|
||||
case results <- result:
|
||||
case <-localCtx.Done():
|
||||
// Consumer gone; close the pipe and wait for the fetch goroutine
|
||||
// to finish so we don't leak it (this result was never sent to
|
||||
// the channel, so the drain loop won't handle it).
|
||||
pr.Close()
|
||||
<-result.done
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Consumer: reads from results channel in order, writes to response writer.
|
||||
// Use the SeaweedFS memory pool for the copy buffer to reduce GC pressure.
|
||||
copyBuf := mem.Allocate(256 * 1024)
|
||||
defer mem.Free(copyBuf)
|
||||
remaining := size
|
||||
|
||||
var consumeErr error
|
||||
for result := range results {
|
||||
chunkView := result.chunkView
|
||||
|
||||
// Handle gap before this chunk (zero-fill)
|
||||
if offset < chunkView.ViewOffset {
|
||||
gap := chunkView.ViewOffset - offset
|
||||
remaining -= gap
|
||||
glog.V(4).InfofCtx(ctx, "prefetch zero [%d,%d)", offset, chunkView.ViewOffset)
|
||||
if err := writeZero(writer, gap); err != nil {
|
||||
consumeErr = fmt.Errorf("write zero [%d,%d): %w", offset, chunkView.ViewOffset, err)
|
||||
result.reader.Close()
|
||||
break
|
||||
}
|
||||
offset = chunkView.ViewOffset
|
||||
}
|
||||
|
||||
// Stream chunk data from pipe to response
|
||||
start := time.Now()
|
||||
_, copyErr := io.CopyBuffer(writer, result.reader, copyBuf)
|
||||
result.reader.Close()
|
||||
|
||||
// Wait for fetch goroutine to finish to get final error
|
||||
<-result.done
|
||||
|
||||
// Determine the effective error
|
||||
err := copyErr
|
||||
if err == nil && result.fetchErr != nil && result.written == 0 {
|
||||
err = result.fetchErr
|
||||
}
|
||||
|
||||
// If the fetcher itself failed before writing any data, try cache invalidation
|
||||
// + re-fetch (same as sequential path stream.go:197). We check result.fetchErr
|
||||
// and result.written (not copied) to avoid wrongly retrying when the fetch
|
||||
// succeeded but the response writer failed on the first write.
|
||||
if result.fetchErr != nil && result.written == 0 {
|
||||
if err := localCtx.Err(); err != nil {
|
||||
consumeErr = err
|
||||
break
|
||||
}
|
||||
retryErr := retryWithCacheInvalidation(localCtx, writer, chunkView, result.urlStrings, jwtFunc, masterClient)
|
||||
if retryErr != nil {
|
||||
stats.FilerHandlerCounter.WithLabelValues("chunkDownloadError").Inc()
|
||||
consumeErr = fmt.Errorf("read chunk: %w", retryErr)
|
||||
break
|
||||
}
|
||||
// Retry succeeded
|
||||
err = nil
|
||||
} else if err != nil {
|
||||
if localCtx.Err() != nil {
|
||||
consumeErr = localCtx.Err()
|
||||
} else {
|
||||
stats.FilerHandlerCounter.WithLabelValues("chunkDownloadError").Inc()
|
||||
consumeErr = fmt.Errorf("read chunk: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
offset += int64(chunkView.ViewSize)
|
||||
remaining -= int64(chunkView.ViewSize)
|
||||
stats.FilerRequestHistogram.WithLabelValues("chunkDownload").Observe(time.Since(start).Seconds())
|
||||
stats.FilerHandlerCounter.WithLabelValues("chunkDownload").Inc()
|
||||
downloadThrottler.MaybeSlowdown(int64(chunkView.ViewSize))
|
||||
}
|
||||
|
||||
// Cancel the local context to stop the producer and any in-flight fetchers early.
|
||||
// This ensures goroutines don't linger after the consumer exits (e.g., on write error).
|
||||
localCancel()
|
||||
|
||||
// Drain remaining results to close pipes and unblock fetch goroutines
|
||||
for result := range results {
|
||||
result.reader.Close()
|
||||
<-result.done
|
||||
}
|
||||
|
||||
// Wait for producer to finish
|
||||
producerWg.Wait()
|
||||
|
||||
if consumeErr != nil {
|
||||
return consumeErr
|
||||
}
|
||||
|
||||
// Handle trailing zero-fill
|
||||
if remaining > 0 {
|
||||
glog.V(4).InfofCtx(ctx, "prefetch zero [%d,%d)", offset, offset+remaining)
|
||||
if err := writeZero(writer, remaining); err != nil {
|
||||
return fmt.Errorf("write zero [%d,%d): %w", offset, offset+remaining, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// retryWithCacheInvalidation attempts to re-fetch a chunk after invalidating the URL cache.
|
||||
// This mirrors the retry logic in PrepareStreamContentWithThrottler's sequential path.
|
||||
func retryWithCacheInvalidation(
|
||||
ctx context.Context,
|
||||
writer io.Writer,
|
||||
chunkView *ChunkView,
|
||||
oldUrlStrings []string,
|
||||
jwtFunc VolumeServerJwtFunction,
|
||||
masterClient wdclient.HasLookupFileIdFunction,
|
||||
) error {
|
||||
invalidator, ok := masterClient.(CacheInvalidator)
|
||||
if !ok {
|
||||
return fmt.Errorf("read chunk %s failed and no cache invalidator available", chunkView.FileId)
|
||||
}
|
||||
|
||||
glog.V(0).InfofCtx(ctx, "prefetch read chunk %s failed, invalidating cache and retrying", chunkView.FileId)
|
||||
invalidator.InvalidateCache(chunkView.FileId)
|
||||
|
||||
newUrlStrings, lookupErr := masterClient.GetLookupFileIdFunction()(ctx, chunkView.FileId)
|
||||
if lookupErr != nil {
|
||||
glog.WarningfCtx(ctx, "failed to re-lookup chunk %s after cache invalidation: %v", chunkView.FileId, lookupErr)
|
||||
return fmt.Errorf("re-lookup chunk %s: %w", chunkView.FileId, lookupErr)
|
||||
}
|
||||
if len(newUrlStrings) == 0 {
|
||||
glog.WarningfCtx(ctx, "re-lookup for chunk %s returned no locations, skipping retry", chunkView.FileId)
|
||||
return fmt.Errorf("re-lookup chunk %s: no locations", chunkView.FileId)
|
||||
}
|
||||
|
||||
if urlSlicesEqual(oldUrlStrings, newUrlStrings) {
|
||||
glog.V(0).InfofCtx(ctx, "re-lookup returned same locations for chunk %s, skipping retry", chunkView.FileId)
|
||||
return fmt.Errorf("read chunk %s failed, same locations after cache invalidation", chunkView.FileId)
|
||||
}
|
||||
|
||||
glog.V(0).InfofCtx(ctx, "retrying read chunk %s with new locations: %v", chunkView.FileId, newUrlStrings)
|
||||
jwt := jwtFunc(chunkView.FileId)
|
||||
_, err := retriedStreamFetchChunkData(
|
||||
ctx, writer, newUrlStrings, jwt,
|
||||
chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(),
|
||||
chunkView.OffsetInChunk, int(chunkView.ViewSize),
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -1,365 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
// testMasterClient implements HasLookupFileIdFunction and CacheInvalidator for tests
|
||||
type testMasterClient struct {
|
||||
urls map[string][]string
|
||||
invalidatedCount int32
|
||||
}
|
||||
|
||||
func (m *testMasterClient) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
|
||||
return func(ctx context.Context, fileId string) ([]string, error) {
|
||||
if urls, ok := m.urls[fileId]; ok {
|
||||
return urls, nil
|
||||
}
|
||||
return nil, fmt.Errorf("fileId %s not found", fileId)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *testMasterClient) InvalidateCache(fileId string) {
|
||||
atomic.AddInt32(&m.invalidatedCount, 1)
|
||||
}
|
||||
|
||||
func noopJwt(fileId string) string { return "" }
|
||||
|
||||
// createTestServer creates a mock volume server that serves chunk data.
|
||||
// Supports Range header for partial chunk reads (exercising OffsetInChunk paths).
|
||||
func createTestServer(chunkData map[string][]byte) *httptest.Server {
|
||||
var mu sync.RWMutex
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
if strings.HasPrefix(path, "/") {
|
||||
path = path[1:]
|
||||
}
|
||||
mu.RLock()
|
||||
data, ok := chunkData[path]
|
||||
mu.RUnlock()
|
||||
if !ok {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Handle Range header for partial chunk reads
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
var start, end int64
|
||||
if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err == nil {
|
||||
if start < 0 || end >= int64(len(data)) || start > end {
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", len(data)))
|
||||
w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
|
||||
return
|
||||
}
|
||||
rangeData := data[start : end+1]
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(rangeData)))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(rangeData)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(data)
|
||||
}))
|
||||
}
|
||||
|
||||
// makeChunksAndServer creates N chunks of given size, a mock server, and a master client
|
||||
func makeChunksAndServer(t *testing.T, numChunks, chunkSize int) ([]*filer_pb.FileChunk, *testMasterClient, map[string][]byte, func()) {
|
||||
t.Helper()
|
||||
|
||||
chunkData := make(map[string][]byte, numChunks)
|
||||
chunks := make([]*filer_pb.FileChunk, numChunks)
|
||||
|
||||
for i := 0; i < numChunks; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
data := make([]byte, chunkSize)
|
||||
rand.Read(data)
|
||||
chunkData[fileId] = data
|
||||
|
||||
chunks[i] = &filer_pb.FileChunk{
|
||||
FileId: fileId,
|
||||
Offset: int64(i * chunkSize),
|
||||
Size: uint64(chunkSize),
|
||||
ModifiedTsNs: int64(i),
|
||||
Fid: &filer_pb.FileId{FileKey: uint64(i)},
|
||||
}
|
||||
}
|
||||
|
||||
server := createTestServer(chunkData)
|
||||
urls := make(map[string][]string, numChunks)
|
||||
for i := 0; i < numChunks; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
urls[fileId] = []string{server.URL + "/" + fileId}
|
||||
}
|
||||
|
||||
masterClient := &testMasterClient{urls: urls}
|
||||
return chunks, masterClient, chunkData, func() { server.Close() }
|
||||
}
|
||||
|
||||
// TestPrefetchInOrderDelivery verifies chunks are written to the output in correct file order
|
||||
func TestPrefetchInOrderDelivery(t *testing.T) {
|
||||
chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 8, 4096)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(8 * 4096)
|
||||
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(), masterClient, noopJwt,
|
||||
chunks, 0, totalSize, 0, 4,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := streamFn(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify total size
|
||||
if buf.Len() != int(totalSize) {
|
||||
t.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
|
||||
}
|
||||
|
||||
// Verify data matches chunk-by-chunk in order
|
||||
result := buf.Bytes()
|
||||
for i := 0; i < 8; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
expected := chunkData[fileId]
|
||||
got := result[i*4096 : (i+1)*4096]
|
||||
if !bytes.Equal(expected, got) {
|
||||
t.Fatalf("chunk %d (%s) data mismatch at offset %d", i, fileId, i*4096)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefetchSingleChunk verifies the pipeline works with just one chunk
|
||||
func TestPrefetchSingleChunk(t *testing.T) {
|
||||
chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 1, 8192)
|
||||
defer cleanup()
|
||||
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(), masterClient, noopJwt,
|
||||
chunks, 0, 8192, 0, 4,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := streamFn(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expected := chunkData["1,0"]
|
||||
if !bytes.Equal(expected, buf.Bytes()) {
|
||||
t.Fatal("single chunk data mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefetchFallbackToSequential verifies prefetch=1 falls back to sequential path
|
||||
func TestPrefetchFallbackToSequential(t *testing.T) {
|
||||
chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 4, 1024)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(4 * 1024)
|
||||
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(), masterClient, noopJwt,
|
||||
chunks, 0, totalSize, 0, 1, // prefetch=1 -> sequential
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := streamFn(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if buf.Len() != int(totalSize) {
|
||||
t.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
|
||||
}
|
||||
|
||||
// Verify data order
|
||||
result := buf.Bytes()
|
||||
for i := 0; i < 4; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
expected := chunkData[fileId]
|
||||
got := result[i*1024 : (i+1)*1024]
|
||||
if !bytes.Equal(expected, got) {
|
||||
t.Fatalf("chunk %d data mismatch", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefetchContextCancellation verifies all goroutines clean up on cancellation
|
||||
func TestPrefetchContextCancellation(t *testing.T) {
|
||||
// Use a slow server so cancellation happens mid-stream
|
||||
var requestCount int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&requestCount, 1)
|
||||
// Slow response
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
w.Header().Set("Content-Length", "1024")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(make([]byte, 1024))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
numChunks := 16
|
||||
chunks := make([]*filer_pb.FileChunk, numChunks)
|
||||
urls := make(map[string][]string, numChunks)
|
||||
for i := 0; i < numChunks; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
chunks[i] = &filer_pb.FileChunk{
|
||||
FileId: fileId, Offset: int64(i * 1024), Size: 1024,
|
||||
ModifiedTsNs: int64(i), Fid: &filer_pb.FileId{FileKey: uint64(i)},
|
||||
}
|
||||
urls[fileId] = []string{server.URL + "/" + fileId}
|
||||
}
|
||||
masterClient := &testMasterClient{urls: urls}
|
||||
|
||||
// Cancel after a short time
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
ctx, masterClient, noopJwt,
|
||||
chunks, 0, int64(numChunks*1024), 0, 4,
|
||||
)
|
||||
if err != nil {
|
||||
// URL resolution may fail due to cancellation — that's expected
|
||||
return
|
||||
}
|
||||
|
||||
err = streamFn(io.Discard)
|
||||
if err == nil {
|
||||
t.Fatal("expected error from cancelled context")
|
||||
}
|
||||
|
||||
// Verify not all chunks were requested (cancellation stopped early)
|
||||
reqs := atomic.LoadInt32(&requestCount)
|
||||
if reqs >= int32(numChunks) {
|
||||
t.Logf("warning: all %d chunks were requested despite cancellation (got %d)", numChunks, reqs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefetchRangeRequest verifies prefetch works with offset/size subset
|
||||
func TestPrefetchRangeRequest(t *testing.T) {
|
||||
chunks, masterClient, chunkData, cleanup := makeChunksAndServer(t, 8, 4096)
|
||||
defer cleanup()
|
||||
|
||||
// Request only chunks 2-5 (offset=8192, size=16384)
|
||||
offset := int64(2 * 4096)
|
||||
size := int64(4 * 4096)
|
||||
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(), masterClient, noopJwt,
|
||||
chunks, offset, size, 0, 4,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := streamFn(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if buf.Len() != int(size) {
|
||||
t.Fatalf("expected %d bytes, got %d", size, buf.Len())
|
||||
}
|
||||
|
||||
// Verify data matches chunks 2-5
|
||||
result := buf.Bytes()
|
||||
for i := 2; i < 6; i++ {
|
||||
fileId := fmt.Sprintf("1,%x", i)
|
||||
expected := chunkData[fileId]
|
||||
start := (i - 2) * 4096
|
||||
got := result[start : start+4096]
|
||||
if !bytes.Equal(expected, got) {
|
||||
t.Fatalf("chunk %d data mismatch in range request", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefetchLargePrefetchCount verifies prefetch > numChunks is handled gracefully
|
||||
func TestPrefetchLargePrefetchCount(t *testing.T) {
|
||||
chunks, masterClient, _, cleanup := makeChunksAndServer(t, 3, 1024)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(3 * 1024)
|
||||
|
||||
// prefetch=10 but only 3 chunks — should work fine
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(), masterClient, noopJwt,
|
||||
chunks, 0, totalSize, 0, 10,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := streamFn(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if buf.Len() != int(totalSize) {
|
||||
t.Fatalf("expected %d bytes, got %d", totalSize, buf.Len())
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefetchConcurrentDownloads verifies multiple concurrent prefetch streams
|
||||
func TestPrefetchConcurrentDownloads(t *testing.T) {
|
||||
chunks, masterClient, _, cleanup := makeChunksAndServer(t, 8, 2048)
|
||||
defer cleanup()
|
||||
|
||||
totalSize := int64(8 * 2048)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 4)
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
streamFn, err := PrepareStreamContentWithPrefetch(
|
||||
context.Background(), masterClient, noopJwt,
|
||||
chunks, 0, totalSize, 0, 4,
|
||||
)
|
||||
if err != nil {
|
||||
errors <- err
|
||||
return
|
||||
}
|
||||
if err := streamFn(io.Discard); err != nil {
|
||||
errors <- err
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
|
||||
for err := range errors {
|
||||
t.Fatalf("concurrent download error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -110,92 +110,6 @@ func TestAWSIAMMatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesActionsMultipartExpansion(t *testing.T) {
|
||||
engine := &PolicyEngine{initialized: true}
|
||||
evalCtx := &EvaluationContext{}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
actions []string
|
||||
requestedAction string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "PutObject directly matches PutObject",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:PutObject",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "PutObject implicitly allows CreateMultipartUpload",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:CreateMultipartUpload",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "PutObject implicitly allows UploadPart",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:UploadPart",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "PutObject implicitly allows CompleteMultipartUpload",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:CompleteMultipartUpload",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "PutObject implicitly allows AbortMultipartUpload",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:AbortMultipartUpload",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "PutObject implicitly allows ListMultipartUploadParts",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:ListMultipartUploadParts",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "PutObject implicitly allows ListBucketMultipartUploads",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:ListBucketMultipartUploads",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "PutObject does not allow GetObject",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "s3:GetObject",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "GetObject does not allow CreateMultipartUpload",
|
||||
actions: []string{"s3:GetObject"},
|
||||
requestedAction: "s3:CreateMultipartUpload",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "wildcard s3:Put* implicitly allows multipart via PutObject match",
|
||||
actions: []string{"s3:Put*"},
|
||||
requestedAction: "s3:CreateMultipartUpload",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "case-insensitive multipart action lookup",
|
||||
actions: []string{"s3:PutObject"},
|
||||
requestedAction: "S3:CREATEMULTIPARTUPLOAD",
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := engine.matchesActions(tt.actions, tt.requestedAction, evalCtx)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandPolicyVariables(t *testing.T) {
|
||||
evalCtx := &EvaluationContext{
|
||||
RequestContext: map[string]interface{}{
|
||||
|
||||
@@ -600,31 +600,12 @@ func (e *PolicyEngine) statementMatches(statement *Statement, evalCtx *Evaluatio
|
||||
return true
|
||||
}
|
||||
|
||||
// multipartActionSet contains lowercased S3 multipart upload actions that are
|
||||
// implicitly granted when s3:PutObject is allowed, since multipart upload is an
|
||||
// implementation detail of putting objects. Keys are lowercased for
|
||||
// case-insensitive lookup (AWS IAM actions are case-insensitive).
|
||||
var multipartActionSet = map[string]bool{
|
||||
"s3:createmultipartupload": true,
|
||||
"s3:uploadpart": true,
|
||||
"s3:completemultipartupload": true,
|
||||
"s3:abortmultipartupload": true,
|
||||
"s3:listmultipartuploadparts": true,
|
||||
"s3:listbucketmultipartuploads": true,
|
||||
}
|
||||
|
||||
// matchesActions checks if any action in the list matches the requested action.
|
||||
// It also implicitly grants multipart upload actions when s3:PutObject is allowed,
|
||||
// mirroring the behavior in the S3 API policy engine (see PR #8445).
|
||||
// matchesActions checks if any action in the list matches the requested action
|
||||
func (e *PolicyEngine) matchesActions(actions []string, requestedAction string, evalCtx *EvaluationContext) bool {
|
||||
isMultipart := multipartActionSet[strings.ToLower(requestedAction)]
|
||||
for _, action := range actions {
|
||||
if awsIAMMatch(action, requestedAction, evalCtx) {
|
||||
return true
|
||||
}
|
||||
if isMultipart && awsIAMMatch(action, "s3:PutObject", evalCtx) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -116,9 +116,11 @@ func (rtm *RebalanceTimeoutManager) IsRebalanceStuck(group *ConsumerGroup, maxRe
|
||||
return time.Since(group.LastActivity) > maxRebalanceDuration
|
||||
}
|
||||
|
||||
// ForceCompleteRebalance forces completion of a stuck rebalance.
|
||||
// IMPORTANT: The caller must already hold group.Mu.Lock().
|
||||
// ForceCompleteRebalance forces completion of a stuck rebalance
|
||||
func (rtm *RebalanceTimeoutManager) ForceCompleteRebalance(group *ConsumerGroup) {
|
||||
group.Mu.Lock()
|
||||
defer group.Mu.Unlock()
|
||||
|
||||
// If stuck in preparing rebalance, move to completing
|
||||
if group.State == GroupStatePreparingRebalance {
|
||||
group.State = GroupStateCompletingRebalance
|
||||
|
||||
@@ -185,19 +185,20 @@ func TestRebalanceTimeoutManager_ForceCompleteRebalance(t *testing.T) {
|
||||
State: MemberStatePending,
|
||||
}
|
||||
group.Members["member1"] = member
|
||||
group.Mu.Unlock()
|
||||
|
||||
// ForceCompleteRebalance expects the caller to hold group.Mu.Lock()
|
||||
rtm.ForceCompleteRebalance(group)
|
||||
|
||||
group.Mu.RLock()
|
||||
if group.State != GroupStateCompletingRebalance {
|
||||
t.Errorf("Expected group state to be CompletingRebalance, got %s", group.State.String())
|
||||
}
|
||||
group.Mu.Unlock()
|
||||
group.Mu.RUnlock()
|
||||
|
||||
// Test forcing completion from CompletingRebalance
|
||||
group.Mu.Lock()
|
||||
rtm.ForceCompleteRebalance(group)
|
||||
|
||||
group.Mu.RLock()
|
||||
if group.State != GroupStateStable {
|
||||
t.Errorf("Expected group state to be Stable, got %s", group.State.String())
|
||||
}
|
||||
@@ -205,7 +206,7 @@ func TestRebalanceTimeoutManager_ForceCompleteRebalance(t *testing.T) {
|
||||
if member.State != MemberStateStable {
|
||||
t.Errorf("Expected member state to be Stable, got %s", member.State.String())
|
||||
}
|
||||
group.Mu.Unlock()
|
||||
group.Mu.RUnlock()
|
||||
}
|
||||
|
||||
func TestRebalanceTimeoutManager_GetRebalanceStatus(t *testing.T) {
|
||||
|
||||
@@ -55,10 +55,7 @@ func (h *Handler) handleFetch(ctx context.Context, correlationID uint32, apiVers
|
||||
for _, partition := range topic.Partitions {
|
||||
hwm, err := h.seaweedMQHandler.GetLatestOffset(topic.Name, partition.PartitionID)
|
||||
if err != nil {
|
||||
// HWM lookup failed (e.g. partition deactivated between consumer
|
||||
// sessions). Assume data may be available rather than blocking in
|
||||
// the long-poll loop — the actual fetch will determine the truth.
|
||||
return true
|
||||
continue
|
||||
}
|
||||
// Normalize fetch offset
|
||||
effectiveOffset := partition.FetchOffset
|
||||
|
||||
@@ -2,7 +2,6 @@ package protocol
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -137,19 +136,13 @@ func (pr *partitionReader) serveFetchRequest(ctx context.Context, req *partition
|
||||
}()
|
||||
|
||||
// Get high water mark
|
||||
hwmUnknown := false
|
||||
hwm, hwmErr := pr.handler.seaweedMQHandler.GetLatestOffset(pr.topicName, pr.partitionID)
|
||||
if hwmErr != nil {
|
||||
// HWM lookup can fail when the partition has been deactivated between consumer
|
||||
// sessions. Proceed with the fetch anyway — the broker will return the correct
|
||||
// data (or empty) based on its own state. Use math.MaxInt64 as sentinel so
|
||||
// FetchMultipleBatches doesn't artificially cap recordsAvailable, and we
|
||||
// don't hit the early-return below. The actual HWM will be derived from
|
||||
// the fetch result (newOffset) after the read.
|
||||
glog.Warningf("[%s] HWM lookup failed for %s[%d]: %v — will attempt fetch anyway",
|
||||
glog.Errorf("[%s] CRITICAL: Failed to get HWM for %s[%d]: %v",
|
||||
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, hwmErr)
|
||||
hwm = math.MaxInt64
|
||||
hwmUnknown = true
|
||||
result.recordBatch = []byte{}
|
||||
result.highWaterMark = 0
|
||||
return
|
||||
}
|
||||
result.highWaterMark = hwm
|
||||
|
||||
@@ -177,20 +170,10 @@ func (pr *partitionReader) serveFetchRequest(ctx context.Context, req *partition
|
||||
// Fetch on-demand - no pre-fetching to avoid overwhelming the broker
|
||||
recordBatch, newOffset := pr.readRecords(ctx, req.requestedOffset, req.maxBytes, req.maxWaitMs, hwm)
|
||||
|
||||
// When HWM was unknown, derive a reasonable value from the fetch result
|
||||
// so the client sees a meaningful high water mark instead of MaxInt64.
|
||||
if hwmUnknown {
|
||||
if newOffset > req.requestedOffset {
|
||||
result.highWaterMark = newOffset // best estimate: end of what we read
|
||||
} else {
|
||||
result.highWaterMark = req.requestedOffset // no data found
|
||||
}
|
||||
}
|
||||
|
||||
// Log what we got back - DETAILED for diagnostics
|
||||
if len(recordBatch) == 0 {
|
||||
glog.V(2).Infof("[%s] FETCH %s[%d]: readRecords returned EMPTY (offset=%d, hwm=%d)",
|
||||
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, result.highWaterMark)
|
||||
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, req.requestedOffset, hwm)
|
||||
result.recordBatch = []byte{}
|
||||
} else {
|
||||
result.recordBatch = recordBatch
|
||||
@@ -245,40 +228,15 @@ func (pr *partitionReader) readRecords(ctx context.Context, fromOffset int64, ma
|
||||
return fetchResult.RecordBatches, fetchResult.NextOffset
|
||||
}
|
||||
|
||||
// Multi-batch failed - try single batch with a fresh timeout
|
||||
// Multi-batch failed - try single batch WITHOUT the timeout constraint
|
||||
// to ensure we get at least some data even if multi-batch timed out
|
||||
glog.Warningf("[%s] Multi-batch fetch failed for %s[%d] offset=%d after %v, falling back to single-batch (err: %v)",
|
||||
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, fromOffset, fetchDuration, err)
|
||||
|
||||
// Compute the remaining time budget for the fallback. If the parent
|
||||
// context carries a deadline, honour it; otherwise derive remaining
|
||||
// from maxWaitMs minus elapsed time. This prevents the fallback from
|
||||
// restarting the full budget after the multi-batch fetch already
|
||||
// consumed part of it.
|
||||
var remaining time.Duration
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
remaining = time.Until(deadline)
|
||||
} else {
|
||||
remaining = time.Duration(maxWaitMs)*time.Millisecond - time.Since(fetchStartTime)
|
||||
}
|
||||
if remaining <= 0 {
|
||||
// Budget exhausted — skip the fallback entirely.
|
||||
glog.V(2).Infof("[%s] No remaining budget for fallback on %s[%d] (maxWait=%dms, elapsed=%v)",
|
||||
pr.connCtx.ConnectionID, pr.topicName, pr.partitionID, maxWaitMs, time.Since(fetchStartTime))
|
||||
return []byte{}, fromOffset
|
||||
}
|
||||
// Clamp: floor of 2s so disk reads via gRPC have a realistic chance,
|
||||
// but never exceed 10s to bound data-plane blocking.
|
||||
fallbackTimeout := remaining
|
||||
if fallbackTimeout < 2*time.Second {
|
||||
fallbackTimeout = 2 * time.Second
|
||||
}
|
||||
if fallbackTimeout > 10*time.Second {
|
||||
fallbackTimeout = 10 * time.Second
|
||||
}
|
||||
fallbackCtx, fallbackCancel := context.WithTimeout(ctx, fallbackTimeout)
|
||||
defer fallbackCancel()
|
||||
// Use original context for fallback, NOT the timed-out fetchCtx
|
||||
// This ensures the fallback has a fresh chance to fetch data
|
||||
fallbackStartTime := time.Now()
|
||||
smqRecords, err := pr.handler.seaweedMQHandler.GetStoredRecords(fallbackCtx, pr.topicName, pr.partitionID, fromOffset, 10)
|
||||
smqRecords, err := pr.handler.seaweedMQHandler.GetStoredRecords(ctx, pr.topicName, pr.partitionID, fromOffset, 10)
|
||||
fallbackDuration := time.Since(fallbackStartTime)
|
||||
|
||||
if fallbackDuration > 2*time.Second {
|
||||
|
||||
@@ -186,7 +186,6 @@ message Identity {
|
||||
bool disabled = 5; // User status: false = enabled (default), true = disabled
|
||||
repeated string service_account_ids = 6; // IDs of service accounts owned by this user
|
||||
repeated string policy_names = 7;
|
||||
bool is_static = 8; // Loaded from static config file (read-only, not editable via API)
|
||||
}
|
||||
|
||||
message Credential {
|
||||
|
||||
@@ -1399,7 +1399,6 @@ type Identity struct {
|
||||
Disabled bool `protobuf:"varint,5,opt,name=disabled,proto3" json:"disabled,omitempty"` // User status: false = enabled (default), true = disabled
|
||||
ServiceAccountIds []string `protobuf:"bytes,6,rep,name=service_account_ids,json=serviceAccountIds,proto3" json:"service_account_ids,omitempty"` // IDs of service accounts owned by this user
|
||||
PolicyNames []string `protobuf:"bytes,7,rep,name=policy_names,json=policyNames,proto3" json:"policy_names,omitempty"`
|
||||
IsStatic bool `protobuf:"varint,8,opt,name=is_static,json=isStatic,proto3" json:"is_static,omitempty"` // Loaded from static config file (read-only, not editable via API)
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -1483,13 +1482,6 @@ func (x *Identity) GetPolicyNames() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Identity) GetIsStatic() bool {
|
||||
if x != nil {
|
||||
return x.IsStatic
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Credential struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
AccessKey string `protobuf:"bytes,1,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"`
|
||||
@@ -3021,7 +3013,7 @@ const file_iam_proto_rawDesc = "" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n" +
|
||||
"\amembers\x18\x02 \x03(\tR\amembers\x12!\n" +
|
||||
"\fpolicy_names\x18\x03 \x03(\tR\vpolicyNames\x12\x1a\n" +
|
||||
"\bdisabled\x18\x04 \x01(\bR\bdisabled\"\xa5\x02\n" +
|
||||
"\bdisabled\x18\x04 \x01(\bR\bdisabled\"\x88\x02\n" +
|
||||
"\bIdentity\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x124\n" +
|
||||
"\vcredentials\x18\x02 \x03(\v2\x12.iam_pb.CredentialR\vcredentials\x12\x18\n" +
|
||||
@@ -3029,8 +3021,7 @@ const file_iam_proto_rawDesc = "" +
|
||||
"\aaccount\x18\x04 \x01(\v2\x0f.iam_pb.AccountR\aaccount\x12\x1a\n" +
|
||||
"\bdisabled\x18\x05 \x01(\bR\bdisabled\x12.\n" +
|
||||
"\x13service_account_ids\x18\x06 \x03(\tR\x11serviceAccountIds\x12!\n" +
|
||||
"\fpolicy_names\x18\a \x03(\tR\vpolicyNames\x12\x1b\n" +
|
||||
"\tis_static\x18\b \x01(\bR\bisStatic\"b\n" +
|
||||
"\fpolicy_names\x18\a \x03(\tR\vpolicyNames\"b\n" +
|
||||
"\n" +
|
||||
"Credential\x12\x1d\n" +
|
||||
"\n" +
|
||||
|
||||
@@ -2,73 +2,14 @@ package repl_util
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
// CopyFromChunkViews copies chunk data with optional SSE decryption.
|
||||
// If entry has SSE-encrypted chunks, data is decrypted before writing.
|
||||
func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error, entry *filer_pb.Entry) error {
|
||||
if entry != nil {
|
||||
sseType, err := detectSSEType(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sseType != filer_pb.SSEType_NONE {
|
||||
return copyWithDecryption(filerSource, entry, writeFunc)
|
||||
}
|
||||
}
|
||||
return copyChunkViews(chunkViews, filerSource, writeFunc)
|
||||
}
|
||||
|
||||
func copyWithDecryption(filerSource *source.FilerSource, entry *filer_pb.Entry, writeFunc func(data []byte) error) error {
|
||||
reader := filer.NewFileReader(filerSource, entry)
|
||||
decrypted, err := MaybeDecryptReader(reader, entry)
|
||||
if err != nil {
|
||||
CloseReader(reader)
|
||||
return err
|
||||
}
|
||||
defer CloseMaybeDecryptedReader(reader, decrypted)
|
||||
buf := make([]byte, 128*1024)
|
||||
for {
|
||||
n, readErr := decrypted.Read(buf)
|
||||
if n > 0 {
|
||||
if writeErr := writeFunc(buf[:n]); writeErr != nil {
|
||||
return writeErr
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CloseReader closes r if it implements io.Closer.
|
||||
func CloseReader(r io.Reader) {
|
||||
if closer, ok := r.(io.Closer); ok {
|
||||
closer.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// CloseMaybeDecryptedReader closes the decrypted reader if it implements io.Closer,
|
||||
// otherwise falls back to closing the original reader.
|
||||
func CloseMaybeDecryptedReader(original, decrypted io.Reader) {
|
||||
if closer, ok := decrypted.(io.Closer); ok {
|
||||
closer.Close()
|
||||
} else {
|
||||
CloseReader(original)
|
||||
}
|
||||
}
|
||||
|
||||
func copyChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error) error {
|
||||
func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error) error {
|
||||
|
||||
for x := chunkViews.Front(); x != nil; x = x.Next {
|
||||
chunk := x.Value
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package repl_util
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/kms"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
var (
|
||||
sseInitMu sync.Mutex
|
||||
sseInitialized bool
|
||||
)
|
||||
|
||||
// InitializeSSEForReplication sets up SSE-S3 and SSE-KMS decryption so that
|
||||
// replication sinks can transparently decrypt encrypted objects.
|
||||
// SSE-S3 is initialized from the filer (KEK stored on filer).
|
||||
// SSE-KMS is initialized from Viper config (security.toml [kms] section or
|
||||
// WEED_KMS_* environment variables).
|
||||
// SSE-C cannot be decrypted (customer key not available) and will error at
|
||||
// decryption time.
|
||||
//
|
||||
// Safe to call multiple times; only the first successful initialization takes
|
||||
// effect. Failed attempts do not prevent future retries.
|
||||
func InitializeSSEForReplication(filerSource filer_pb.FilerClient) error {
|
||||
sseInitMu.Lock()
|
||||
defer sseInitMu.Unlock()
|
||||
if sseInitialized {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Initialize SSE-S3 key manager from filer
|
||||
if err := s3api.GetSSES3KeyManager().InitializeWithFiler(filerSource); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Attempt KMS initialization from Viper config.
|
||||
// KMS configuration is typically in the S3 config file which the
|
||||
// replication commands don't load directly. Support loading from
|
||||
// security.toml [kms] section or WEED_KMS_* environment variables.
|
||||
loader := kms.NewConfigLoader(util.GetViper())
|
||||
if err := loader.LoadConfigurations(); err != nil {
|
||||
glog.Warningf("KMS initialization from config failed: %v (SSE-KMS decryption will not be available)", err)
|
||||
} else if err := loader.ValidateConfiguration(); err != nil {
|
||||
glog.Warningf("KMS configuration validation failed: %v (SSE-KMS decryption will not be available)", err)
|
||||
} else {
|
||||
glog.V(0).Infof("KMS initialized for replication")
|
||||
}
|
||||
|
||||
sseInitialized = true
|
||||
return nil
|
||||
}
|
||||
@@ -1,150 +0,0 @@
|
||||
package repl_util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// MaybeDecryptReader wraps reader with SSE decryption if the entry has encrypted chunks.
|
||||
// Returns the original reader unchanged if no SSE encryption is detected.
|
||||
func MaybeDecryptReader(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
|
||||
if entry == nil {
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
sseType, err := detectSSEType(entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sseType == filer_pb.SSEType_NONE {
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
switch sseType {
|
||||
case filer_pb.SSEType_SSE_S3:
|
||||
return decryptSSES3(reader, entry)
|
||||
case filer_pb.SSEType_SSE_KMS:
|
||||
return decryptSSEKMS(reader, entry)
|
||||
case filer_pb.SSEType_SSE_C:
|
||||
return nil, fmt.Errorf("SSE-C encrypted object cannot be decrypted during replication (customer key not available)")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported SSE type: %v", sseType)
|
||||
}
|
||||
|
||||
// MaybeDecryptContent decrypts inline entry content if SSE-encrypted.
|
||||
// Returns the original content unchanged if no SSE encryption is detected.
|
||||
func MaybeDecryptContent(content []byte, entry *filer_pb.Entry) ([]byte, error) {
|
||||
if entry == nil || len(content) == 0 {
|
||||
return content, nil
|
||||
}
|
||||
|
||||
sseType, err := detectSSEType(entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sseType == filer_pb.SSEType_NONE {
|
||||
return content, nil
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(content)
|
||||
decrypted, err := MaybeDecryptReader(reader, entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(decrypted)
|
||||
}
|
||||
|
||||
func detectSSEType(entry *filer_pb.Entry) (filer_pb.SSEType, error) {
|
||||
// Check chunk metadata first
|
||||
var detected filer_pb.SSEType
|
||||
for _, chunk := range entry.GetChunks() {
|
||||
if chunk.SseType != filer_pb.SSEType_NONE {
|
||||
if detected == filer_pb.SSEType_NONE {
|
||||
detected = chunk.SseType
|
||||
} else if chunk.SseType != detected {
|
||||
return filer_pb.SSEType_NONE, fmt.Errorf("mixed SSE types in chunks: %v and %v", detected, chunk.SseType)
|
||||
}
|
||||
}
|
||||
}
|
||||
if detected != filer_pb.SSEType_NONE {
|
||||
return detected, nil
|
||||
}
|
||||
|
||||
// Fall back to extended metadata for inline objects (no chunks)
|
||||
if entry.Extended != nil {
|
||||
hasS3 := len(entry.Extended[s3_constants.SeaweedFSSSES3Key]) > 0
|
||||
hasKMS := len(entry.Extended[s3_constants.SeaweedFSSSEKMSKey]) > 0
|
||||
hasC := len(entry.Extended[s3_constants.SeaweedFSSSEIV]) > 0
|
||||
count := 0
|
||||
if hasS3 {
|
||||
count++
|
||||
}
|
||||
if hasKMS {
|
||||
count++
|
||||
}
|
||||
if hasC {
|
||||
count++
|
||||
}
|
||||
if count > 1 {
|
||||
return filer_pb.SSEType_NONE, fmt.Errorf("conflicting SSE metadata in entry: multiple SSE key types present")
|
||||
}
|
||||
if hasS3 {
|
||||
return filer_pb.SSEType_SSE_S3, nil
|
||||
}
|
||||
if hasKMS {
|
||||
return filer_pb.SSEType_SSE_KMS, nil
|
||||
}
|
||||
if hasC {
|
||||
return filer_pb.SSEType_SSE_C, nil
|
||||
}
|
||||
}
|
||||
return filer_pb.SSEType_NONE, nil
|
||||
}
|
||||
|
||||
func decryptSSES3(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
|
||||
if entry.Extended == nil {
|
||||
return nil, fmt.Errorf("SSE-S3 encrypted entry has no extended metadata")
|
||||
}
|
||||
|
||||
keyData := entry.Extended[s3_constants.SeaweedFSSSES3Key]
|
||||
if len(keyData) == 0 {
|
||||
return nil, fmt.Errorf("SSE-S3 key metadata not found in entry")
|
||||
}
|
||||
|
||||
keyManager := s3api.GetSSES3KeyManager()
|
||||
sseS3Key, err := s3api.DeserializeSSES3Metadata(keyData, keyManager)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deserialize SSE-S3 metadata: %w", err)
|
||||
}
|
||||
|
||||
iv, err := s3api.GetSSES3IV(entry, sseS3Key, keyManager)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get SSE-S3 IV: %w", err)
|
||||
}
|
||||
|
||||
return s3api.CreateSSES3DecryptedReader(reader, sseS3Key, iv)
|
||||
}
|
||||
|
||||
func decryptSSEKMS(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
|
||||
if entry.Extended == nil {
|
||||
return nil, fmt.Errorf("SSE-KMS encrypted entry has no extended metadata")
|
||||
}
|
||||
|
||||
kmsMetadata := entry.Extended[s3_constants.SeaweedFSSSEKMSKey]
|
||||
if len(kmsMetadata) == 0 {
|
||||
return nil, fmt.Errorf("SSE-KMS key metadata not found in entry")
|
||||
}
|
||||
|
||||
sseKMSKey, err := s3api.DeserializeSSEKMSMetadata(kmsMetadata)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deserialize SSE-KMS metadata: %w", err)
|
||||
}
|
||||
|
||||
return s3api.CreateSSEKMSDecryptedReader(reader, sseKMSKey)
|
||||
}
|
||||
@@ -1,534 +0,0 @@
|
||||
package repl_util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/kms"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func TestDetectSSEType(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
entry *filer_pb.Entry
|
||||
wantType filer_pb.SSEType
|
||||
wantError bool
|
||||
}{
|
||||
{
|
||||
name: "no chunks no extended",
|
||||
entry: &filer_pb.Entry{},
|
||||
wantType: filer_pb.SSEType_NONE,
|
||||
},
|
||||
{
|
||||
name: "plaintext chunks",
|
||||
entry: &filer_pb.Entry{
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{SseType: filer_pb.SSEType_NONE},
|
||||
{SseType: filer_pb.SSEType_NONE},
|
||||
},
|
||||
},
|
||||
wantType: filer_pb.SSEType_NONE,
|
||||
},
|
||||
{
|
||||
name: "uniform SSE-S3 chunks",
|
||||
entry: &filer_pb.Entry{
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{SseType: filer_pb.SSEType_SSE_S3},
|
||||
{SseType: filer_pb.SSEType_SSE_S3},
|
||||
},
|
||||
},
|
||||
wantType: filer_pb.SSEType_SSE_S3,
|
||||
},
|
||||
{
|
||||
name: "uniform SSE-KMS chunks",
|
||||
entry: &filer_pb.Entry{
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{SseType: filer_pb.SSEType_SSE_KMS},
|
||||
},
|
||||
},
|
||||
wantType: filer_pb.SSEType_SSE_KMS,
|
||||
},
|
||||
{
|
||||
name: "mixed chunk SSE types",
|
||||
entry: &filer_pb.Entry{
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{SseType: filer_pb.SSEType_SSE_S3},
|
||||
{SseType: filer_pb.SSEType_SSE_KMS},
|
||||
},
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "inline SSE-S3 via extended",
|
||||
entry: &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSES3Key: {0x01},
|
||||
},
|
||||
},
|
||||
wantType: filer_pb.SSEType_SSE_S3,
|
||||
},
|
||||
{
|
||||
name: "inline SSE-KMS via extended",
|
||||
entry: &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSEKMSKey: {0x01},
|
||||
},
|
||||
},
|
||||
wantType: filer_pb.SSEType_SSE_KMS,
|
||||
},
|
||||
{
|
||||
name: "inline SSE-C via extended",
|
||||
entry: &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSEIV: {0x01},
|
||||
},
|
||||
},
|
||||
wantType: filer_pb.SSEType_SSE_C,
|
||||
},
|
||||
{
|
||||
name: "conflicting extended metadata",
|
||||
entry: &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSES3Key: {0x01},
|
||||
s3_constants.SeaweedFSSSEKMSKey: {0x02},
|
||||
},
|
||||
},
|
||||
wantError: true,
|
||||
},
|
||||
{
|
||||
name: "chunks take precedence over extended",
|
||||
entry: &filer_pb.Entry{
|
||||
Chunks: []*filer_pb.FileChunk{
|
||||
{SseType: filer_pb.SSEType_SSE_S3},
|
||||
},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSEKMSKey: {0x01},
|
||||
},
|
||||
},
|
||||
wantType: filer_pb.SSEType_SSE_S3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := detectSSEType(tt.entry)
|
||||
if tt.wantError {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got type %v", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != tt.wantType {
|
||||
t.Errorf("got %v, want %v", got, tt.wantType)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptReader_Plaintext(t *testing.T) {
|
||||
content := []byte("hello world")
|
||||
entry := &filer_pb.Entry{}
|
||||
reader := bytes.NewReader(content)
|
||||
|
||||
got, err := MaybeDecryptReader(reader, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result, err := io.ReadAll(got)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll error: %v", err)
|
||||
}
|
||||
if !bytes.Equal(result, content) {
|
||||
t.Errorf("got %q, want %q", result, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptReader_NilEntry(t *testing.T) {
|
||||
content := []byte("hello")
|
||||
reader := bytes.NewReader(content)
|
||||
|
||||
got, err := MaybeDecryptReader(reader, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
result, err := io.ReadAll(got)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll error: %v", err)
|
||||
}
|
||||
if !bytes.Equal(result, content) {
|
||||
t.Errorf("got %q, want %q", result, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptReader_SSEC_Error(t *testing.T) {
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSEIV: {0x01},
|
||||
},
|
||||
}
|
||||
reader := bytes.NewReader([]byte("data"))
|
||||
|
||||
_, err := MaybeDecryptReader(reader, entry)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for SSE-C")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptContent_Plaintext(t *testing.T) {
|
||||
content := []byte("hello world")
|
||||
entry := &filer_pb.Entry{}
|
||||
|
||||
got, err := MaybeDecryptContent(content, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, content) {
|
||||
t.Errorf("got %q, want %q", got, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptContent_NilEntry(t *testing.T) {
|
||||
content := []byte("data")
|
||||
got, err := MaybeDecryptContent(content, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, content) {
|
||||
t.Errorf("got %q, want %q", got, content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptContent_Empty(t *testing.T) {
|
||||
got, err := MaybeDecryptContent(nil, &filer_pb.Entry{})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Errorf("expected nil, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptContent_SSEC_Error(t *testing.T) {
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSEIV: {0x01},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := MaybeDecryptContent([]byte("data"), entry)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for SSE-C")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptContent_MixedExtended_Error(t *testing.T) {
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSES3Key: {0x01},
|
||||
s3_constants.SeaweedFSSSEKMSKey: {0x02},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := MaybeDecryptContent([]byte("data"), entry)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for conflicting SSE metadata")
|
||||
}
|
||||
}
|
||||
|
||||
// --- SSE-S3 integration tests ---
|
||||
// These tests exercise the full MaybeDecryptReader/MaybeDecryptContent path
|
||||
// for SSE-S3: detectSSEType → decryptSSES3 → DeserializeSSES3Metadata →
|
||||
// GetSSES3IV → CreateSSES3DecryptedReader. A test KEK is injected via
|
||||
// WEED_S3_SSE_KEK env var and a mock filer client.
|
||||
|
||||
// testFilerClient is a minimal filer_pb.FilerClient mock that returns
|
||||
// ErrNotFound for all lookups (no KEK on filer — we use env var instead).
|
||||
type testFilerClient struct{}
|
||||
|
||||
func (c *testFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
return fmt.Errorf("%w", filer_pb.ErrNotFound)
|
||||
}
|
||||
func (c *testFilerClient) AdjustedUrl(loc *filer_pb.Location) string { return loc.Url }
|
||||
func (c *testFilerClient) GetDataCenter() string { return "" }
|
||||
|
||||
// setupTestSSES3 initializes the global SSE-S3 key manager with a test KEK
|
||||
// via the WEED_S3_SSE_KEK env var and returns the KEK bytes + cleanup func.
|
||||
func setupTestSSES3(t *testing.T) (kek []byte, cleanup func()) {
|
||||
t.Helper()
|
||||
|
||||
kek = make([]byte, 32)
|
||||
if _, err := io.ReadFull(rand.Reader, kek); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Force Viper to pick up the new env var
|
||||
os.Setenv("WEED_S3_SSE_KEK", hex.EncodeToString(kek))
|
||||
|
||||
// Reset Viper cache so it reads the new env var
|
||||
v := util.GetViper()
|
||||
v.AutomaticEnv()
|
||||
|
||||
// Re-initialize the global key manager with the KEK from env
|
||||
km := s3api.GetSSES3KeyManager()
|
||||
if err := km.InitializeWithFiler(&testFilerClient{}); err != nil {
|
||||
os.Unsetenv("WEED_S3_SSE_KEK")
|
||||
t.Fatalf("InitializeWithFiler: %v", err)
|
||||
}
|
||||
|
||||
return kek, func() {
|
||||
os.Unsetenv("WEED_S3_SSE_KEK")
|
||||
// Re-initialize with no KEK to clear the super key
|
||||
km.InitializeWithFiler(&testFilerClient{})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptReader_SSES3(t *testing.T) {
|
||||
_, cleanup := setupTestSSES3(t)
|
||||
defer cleanup()
|
||||
|
||||
plaintext := []byte("SSE-S3 encrypted content for testing round-trip decryption")
|
||||
|
||||
// Generate a DEK and encrypt
|
||||
sseKey, err := s3api.GenerateSSES3Key()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encReader, encIV, err := s3api.CreateSSES3EncryptedReader(bytes.NewReader(plaintext), sseKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encrypt: %v", err)
|
||||
}
|
||||
ciphertext, err := io.ReadAll(encReader)
|
||||
if err != nil {
|
||||
t.Fatalf("read ciphertext: %v", err)
|
||||
}
|
||||
|
||||
// Build serialized SSE-S3 metadata (uses the global key manager to
|
||||
// envelope-encrypt the DEK with the test KEK)
|
||||
sseKey.IV = encIV
|
||||
metadataBytes, err := s3api.SerializeSSES3Metadata(sseKey)
|
||||
if err != nil {
|
||||
t.Fatalf("serialize metadata: %v", err)
|
||||
}
|
||||
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSES3Key: metadataBytes,
|
||||
},
|
||||
}
|
||||
|
||||
// Test full path: MaybeDecryptReader → decryptSSES3 → DeserializeSSES3Metadata → CreateSSES3DecryptedReader
|
||||
decrypted, err := MaybeDecryptReader(bytes.NewReader(ciphertext), entry)
|
||||
if err != nil {
|
||||
t.Fatalf("MaybeDecryptReader: %v", err)
|
||||
}
|
||||
result, err := io.ReadAll(decrypted)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll: %v", err)
|
||||
}
|
||||
if !bytes.Equal(result, plaintext) {
|
||||
t.Errorf("SSE-S3 round-trip failed: got %q, want %q", result, plaintext)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptContent_SSES3(t *testing.T) {
|
||||
_, cleanup := setupTestSSES3(t)
|
||||
defer cleanup()
|
||||
|
||||
plaintext := []byte("inline SSE-S3 content")
|
||||
|
||||
// Generate a DEK and encrypt inline content
|
||||
sseKey, err := s3api.GenerateSSES3Key()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encReader, encIV, err := s3api.CreateSSES3EncryptedReader(bytes.NewReader(plaintext), sseKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encrypt: %v", err)
|
||||
}
|
||||
ciphertext, err := io.ReadAll(encReader)
|
||||
if err != nil {
|
||||
t.Fatalf("read ciphertext: %v", err)
|
||||
}
|
||||
|
||||
sseKey.IV = encIV
|
||||
metadataBytes, err := s3api.SerializeSSES3Metadata(sseKey)
|
||||
if err != nil {
|
||||
t.Fatalf("serialize metadata: %v", err)
|
||||
}
|
||||
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSES3Key: metadataBytes,
|
||||
},
|
||||
}
|
||||
|
||||
// Test full path: MaybeDecryptContent → MaybeDecryptReader → decryptSSES3
|
||||
result, err := MaybeDecryptContent(ciphertext, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("MaybeDecryptContent: %v", err)
|
||||
}
|
||||
if !bytes.Equal(result, plaintext) {
|
||||
t.Errorf("SSE-S3 round-trip failed: got %q, want %q", result, plaintext)
|
||||
}
|
||||
}
|
||||
|
||||
// --- SSE-KMS integration tests ---
|
||||
|
||||
// testKMSProvider is a minimal KMSProvider mock for testing.
|
||||
type testKMSProvider struct {
|
||||
keyID string
|
||||
plaintext []byte // the DEK plaintext returned by Decrypt
|
||||
}
|
||||
|
||||
func (p *testKMSProvider) GenerateDataKey(_ context.Context, _ *kms.GenerateDataKeyRequest) (*kms.GenerateDataKeyResponse, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *testKMSProvider) Decrypt(_ context.Context, _ *kms.DecryptRequest) (*kms.DecryptResponse, error) {
|
||||
return &kms.DecryptResponse{
|
||||
KeyID: p.keyID,
|
||||
Plaintext: append([]byte(nil), p.plaintext...), // return a copy
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *testKMSProvider) DescribeKey(_ context.Context, _ *kms.DescribeKeyRequest) (*kms.DescribeKeyResponse, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *testKMSProvider) GetKeyID(_ context.Context, keyIdentifier string) (string, error) {
|
||||
return p.keyID, nil
|
||||
}
|
||||
|
||||
func (p *testKMSProvider) Close() error { return nil }
|
||||
|
||||
func TestMaybeDecryptReader_SSEKMS(t *testing.T) {
|
||||
plaintext := []byte("SSE-KMS encrypted content for testing")
|
||||
|
||||
// Generate a random DEK and IV
|
||||
dek := make([]byte, 32)
|
||||
iv := make([]byte, aes.BlockSize)
|
||||
if _, err := io.ReadFull(rand.Reader, dek); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Encrypt with AES-CTR (same cipher mode as SSE-KMS)
|
||||
block, err := aes.NewCipher(dek)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ciphertext := make([]byte, len(plaintext))
|
||||
cipher.NewCTR(block, iv).XORKeyStream(ciphertext, plaintext)
|
||||
|
||||
// Set up a mock KMS provider that returns our DEK
|
||||
keyID := "test-kms-key-1"
|
||||
encryptedDEK := []byte("fake-encrypted-dek") // mock doesn't validate
|
||||
kms.SetGlobalKMSProvider(&testKMSProvider{
|
||||
keyID: keyID,
|
||||
plaintext: dek,
|
||||
})
|
||||
defer kms.SetGlobalKMSProvider(nil)
|
||||
|
||||
// Build serialized KMS metadata
|
||||
kmsMetadata := s3api.SSEKMSMetadata{
|
||||
Algorithm: s3_constants.SSEAlgorithmKMS,
|
||||
KeyID: keyID,
|
||||
EncryptedDataKey: base64.StdEncoding.EncodeToString(encryptedDEK),
|
||||
IV: base64.StdEncoding.EncodeToString(iv),
|
||||
}
|
||||
metadataBytes, err := json.Marshal(kmsMetadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSEKMSKey: metadataBytes,
|
||||
},
|
||||
}
|
||||
|
||||
// Test MaybeDecryptReader
|
||||
reader := bytes.NewReader(ciphertext)
|
||||
decrypted, err := MaybeDecryptReader(reader, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("MaybeDecryptReader: %v", err)
|
||||
}
|
||||
result, err := io.ReadAll(decrypted)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll: %v", err)
|
||||
}
|
||||
if !bytes.Equal(result, plaintext) {
|
||||
t.Errorf("SSE-KMS round-trip failed: got %q, want %q", result, plaintext)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeDecryptContent_SSEKMS(t *testing.T) {
|
||||
plaintext := []byte("inline SSE-KMS content")
|
||||
|
||||
dek := make([]byte, 32)
|
||||
iv := make([]byte, aes.BlockSize)
|
||||
if _, err := io.ReadFull(rand.Reader, dek); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(dek)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ciphertext := make([]byte, len(plaintext))
|
||||
cipher.NewCTR(block, iv).XORKeyStream(ciphertext, plaintext)
|
||||
|
||||
keyID := "test-kms-key-2"
|
||||
kms.SetGlobalKMSProvider(&testKMSProvider{
|
||||
keyID: keyID,
|
||||
plaintext: dek,
|
||||
})
|
||||
defer kms.SetGlobalKMSProvider(nil)
|
||||
|
||||
kmsMetadata := s3api.SSEKMSMetadata{
|
||||
Algorithm: s3_constants.SSEAlgorithmKMS,
|
||||
KeyID: keyID,
|
||||
EncryptedDataKey: base64.StdEncoding.EncodeToString([]byte("fake-encrypted-dek")),
|
||||
IV: base64.StdEncoding.EncodeToString(iv),
|
||||
}
|
||||
metadataBytes, err := json.Marshal(kmsMetadata)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
entry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.SeaweedFSSSEKMSKey: metadataBytes,
|
||||
},
|
||||
}
|
||||
|
||||
result, err := MaybeDecryptContent(ciphertext, entry)
|
||||
if err != nil {
|
||||
t.Fatalf("MaybeDecryptContent: %v", err)
|
||||
}
|
||||
if !bytes.Equal(result, plaintext) {
|
||||
t.Errorf("SSE-KMS round-trip failed: got %q, want %q", result, plaintext)
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
@@ -26,10 +25,6 @@ func NewReplicator(sourceConfig util.Configuration, configPrefix string, dataSin
|
||||
source := &source.FilerSource{}
|
||||
source.Initialize(sourceConfig, configPrefix)
|
||||
|
||||
if err := repl_util.InitializeSSEForReplication(source); err != nil {
|
||||
glog.Warningf("SSE initialization failed: %v (encrypted objects may fail to replicate)", err)
|
||||
}
|
||||
|
||||
dataSink.SetSourceFiler(source)
|
||||
|
||||
return &Replicator{
|
||||
|
||||
@@ -138,7 +138,6 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
|
||||
_, err := appendBlobClient.Create(ctxCreate, nil)
|
||||
|
||||
needsWrite := true
|
||||
freshlyCreated := false
|
||||
if err != nil {
|
||||
if bloberror.HasCode(err, bloberror.BlobAlreadyExists) {
|
||||
// Handle existing blob - check if overwrite is needed and perform it if necessary
|
||||
@@ -147,13 +146,9 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
|
||||
if handleErr != nil {
|
||||
return handleErr
|
||||
}
|
||||
// handleExistingBlob recreates the blob when needsWrite is true
|
||||
freshlyCreated = needsWrite
|
||||
} else {
|
||||
return fmt.Errorf("azure create append blob %s/%s: %w", g.container, key, err)
|
||||
}
|
||||
} else {
|
||||
freshlyCreated = true
|
||||
}
|
||||
|
||||
// If we don't need to write (blob is up-to-date), return early
|
||||
@@ -161,23 +156,6 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupOnError deletes a freshly created blob when content write fails,
|
||||
// preventing empty blobs from being left behind.
|
||||
cleanupOnError := func(writeErr error) error {
|
||||
if !freshlyCreated {
|
||||
return writeErr
|
||||
}
|
||||
glog.Warningf("azure sink: cleaning up empty blob %s/%s after write failure: %v", g.container, key, writeErr)
|
||||
ctxCleanup, cancelCleanup := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
|
||||
defer cancelCleanup()
|
||||
if _, delErr := appendBlobClient.Delete(ctxCleanup, nil); delErr != nil {
|
||||
if !bloberror.HasCode(delErr, bloberror.BlobNotFound) {
|
||||
glog.Warningf("azure sink: failed to clean up blob %s/%s: %v", g.container, key, delErr)
|
||||
}
|
||||
}
|
||||
return writeErr
|
||||
}
|
||||
|
||||
writeFunc := func(data []byte) error {
|
||||
ctxWrite, cancelWrite := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
|
||||
defer cancelWrite()
|
||||
@@ -186,18 +164,11 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
|
||||
}
|
||||
|
||||
if len(entry.Content) > 0 {
|
||||
content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
|
||||
if err != nil {
|
||||
return cleanupOnError(fmt.Errorf("decrypt inline SSE content: %w", err))
|
||||
}
|
||||
if err := writeFunc(content); err != nil {
|
||||
return cleanupOnError(err)
|
||||
}
|
||||
return nil
|
||||
return writeFunc(entry.Content)
|
||||
}
|
||||
|
||||
if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry); err != nil {
|
||||
return cleanupOnError(err)
|
||||
if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -2,7 +2,6 @@ package B2Sink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/kurin/blazer/b2"
|
||||
@@ -117,14 +116,10 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int
|
||||
}
|
||||
|
||||
if len(entry.Content) > 0 {
|
||||
content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt inline SSE content: %w", err)
|
||||
}
|
||||
return writeFunc(content)
|
||||
return writeFunc(entry.Content)
|
||||
}
|
||||
|
||||
if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry); err != nil {
|
||||
if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -115,37 +115,23 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []in
|
||||
totalSize := filer.FileSize(entry)
|
||||
chunkViews := filer.ViewFromChunks(context.Background(), g.filerSource.LookupFileId, entry.GetChunks(), 0, int64(totalSize))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
wc := g.client.Bucket(g.bucket).Object(key).NewWriter(ctx)
|
||||
wc := g.client.Bucket(g.bucket).Object(key).NewWriter(context.Background())
|
||||
defer wc.Close()
|
||||
|
||||
writeFunc := func(data []byte) error {
|
||||
_, writeErr := wc.Write(data)
|
||||
return writeErr
|
||||
}
|
||||
|
||||
var writeErr error
|
||||
if len(entry.Content) > 0 {
|
||||
content, decErr := repl_util.MaybeDecryptContent(entry.Content, entry)
|
||||
if decErr != nil {
|
||||
writeErr = fmt.Errorf("decrypt inline SSE content: %w", decErr)
|
||||
} else {
|
||||
writeErr = writeFunc(content)
|
||||
}
|
||||
} else {
|
||||
writeErr = repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry)
|
||||
return writeFunc(entry.Content)
|
||||
}
|
||||
|
||||
if writeErr != nil {
|
||||
// Cancel the context to abort the GCS upload without touching
|
||||
// any existing object at this key.
|
||||
cancel()
|
||||
wc.Close()
|
||||
return writeErr
|
||||
if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return wc.Close()
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package localsink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -95,55 +94,36 @@ func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signa
|
||||
|
||||
mode := os.FileMode(entry.Attributes.FileMode)
|
||||
shortFileName := util.ToShortFileName(key)
|
||||
|
||||
// Write to a temp file in the same directory, then atomically rename
|
||||
// on success. This prevents leaving a truncated/empty file if
|
||||
// decryption or chunk copy fails.
|
||||
tmpFile, err := os.CreateTemp(dir, ".seaweedfs-tmp-*")
|
||||
if err := os.Remove(shortFileName); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
dstFile, err := os.OpenFile(shortFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmpFile.Name()
|
||||
defer func() {
|
||||
// Clean up temp file on any error (rename removes it on success)
|
||||
if tmpFile != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpName)
|
||||
}
|
||||
}()
|
||||
defer dstFile.Close()
|
||||
|
||||
if err := tmpFile.Chmod(mode); err != nil {
|
||||
fi, err := dstFile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fi.Mode() != mode {
|
||||
glog.V(4).Infof("Modify file mode: %o -> %o", fi.Mode(), mode)
|
||||
if err := dstFile.Chmod(mode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
writeFunc := func(data []byte) error {
|
||||
_, writeErr := tmpFile.Write(data)
|
||||
_, writeErr := dstFile.Write(data)
|
||||
return writeErr
|
||||
}
|
||||
|
||||
if len(entry.Content) > 0 {
|
||||
content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decrypt inline SSE content: %w", err)
|
||||
}
|
||||
if err := writeFunc(content); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := repl_util.CopyFromChunkViews(chunkViews, localsink.filerSource, writeFunc, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFunc(entry.Content)
|
||||
}
|
||||
|
||||
// Close before rename so the data is flushed
|
||||
if err := tmpFile.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
tmpFile = nil // prevent deferred cleanup
|
||||
|
||||
// Atomic rename into final destination
|
||||
if err := os.Rename(tmpName, shortFileName); err != nil {
|
||||
os.Remove(tmpName)
|
||||
if err := repl_util.CopyFromChunkViews(chunkViews, localsink.filerSource, writeFunc); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package S3Sink
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -18,7 +17,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
|
||||
"github.com/seaweedfs/seaweedfs/weed/replication/source"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
@@ -162,14 +160,6 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
|
||||
|
||||
reader := filer.NewFileReader(s3sink.filerSource, entry)
|
||||
|
||||
// Decrypt SSE-encrypted objects so the destination receives plaintext
|
||||
decryptedReader, err := repl_util.MaybeDecryptReader(reader, entry)
|
||||
if err != nil {
|
||||
repl_util.CloseReader(reader)
|
||||
return fmt.Errorf("decrypt SSE object: %w", err)
|
||||
}
|
||||
defer repl_util.CloseMaybeDecryptedReader(reader, decryptedReader)
|
||||
|
||||
// Create an uploader with the session and custom options
|
||||
uploader := s3manager.NewUploaderWithClient(s3sink.conn, func(u *s3manager.Uploader) {
|
||||
u.PartSize = int64(s3sink.uploaderPartSizeMb * 1024 * 1024)
|
||||
@@ -198,16 +188,20 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
|
||||
entry.Extended[s3_constants.AmzUserMetaMtime] = []byte(strconv.FormatInt(entry.Attributes.Mtime, 10))
|
||||
}
|
||||
// process tagging
|
||||
tags := buildTaggingString(entry.Extended)
|
||||
tags := ""
|
||||
for k, v := range entry.Extended {
|
||||
if len(tags) > 0 {
|
||||
tags = tags + "&"
|
||||
}
|
||||
tags = tags + k + "=" + string(v)
|
||||
}
|
||||
|
||||
// Upload the file to S3.
|
||||
uploadInput := s3manager.UploadInput{
|
||||
Bucket: aws.String(s3sink.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: decryptedReader,
|
||||
}
|
||||
if tags != "" {
|
||||
uploadInput.Tagging = aws.String(tags)
|
||||
Bucket: aws.String(s3sink.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: reader,
|
||||
Tagging: aws.String(tags),
|
||||
}
|
||||
if len(entry.Attributes.Md5) > 0 {
|
||||
uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
|
||||
@@ -229,18 +223,3 @@ func cleanKey(key string) string {
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// buildTaggingString builds the S3 Tagging header value from entry extended metadata.
|
||||
// Only keys with the AmzObjectTaggingPrefix ("X-Amz-Tagging-") are included as object
|
||||
// tags. The prefix is stripped and values are URL-encoded to produce a valid S3 tagging
|
||||
// query string.
|
||||
func buildTaggingString(extended map[string][]byte) string {
|
||||
tagValues := url.Values{}
|
||||
for k, v := range extended {
|
||||
if strings.HasPrefix(k, s3_constants.AmzObjectTaggingPrefix) {
|
||||
tagKey := k[len(s3_constants.AmzObjectTaggingPrefix):]
|
||||
tagValues.Set(tagKey, string(v))
|
||||
}
|
||||
}
|
||||
return tagValues.Encode()
|
||||
}
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
package S3Sink
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
func TestBuildTaggingString_ShouldStripTagPrefix(t *testing.T) {
|
||||
extended := map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "env": []byte("production"),
|
||||
}
|
||||
|
||||
tagging := buildTaggingString(extended)
|
||||
|
||||
if strings.Contains(tagging, s3_constants.AmzObjectTaggingPrefix) {
|
||||
t.Errorf("tagging should not contain storage prefix %q, got %q", s3_constants.AmzObjectTaggingPrefix, tagging)
|
||||
}
|
||||
|
||||
parsed, err := url.ParseQuery(tagging)
|
||||
if err != nil {
|
||||
t.Fatalf("tagging should be valid URL query: %v", err)
|
||||
}
|
||||
if v := parsed.Get("env"); v != "production" {
|
||||
t.Errorf("expected tag env=production, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTaggingString_ShouldURLEncodeValues(t *testing.T) {
|
||||
extended := map[string][]byte{
|
||||
s3_constants.AmzObjectTaggingPrefix + "path": []byte("/a/b=c&d"),
|
||||
}
|
||||
|
||||
tagging := buildTaggingString(extended)
|
||||
|
||||
parsed, err := url.ParseQuery(tagging)
|
||||
if err != nil {
|
||||
t.Fatalf("tagging should be valid URL query: %v", err)
|
||||
}
|
||||
if v := parsed.Get("path"); v != "/a/b=c&d" {
|
||||
t.Errorf("expected tag value /a/b=c&d after decoding, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTaggingString_EmptyWhenNoTags(t *testing.T) {
|
||||
extended := map[string][]byte{
|
||||
"Content-Encoding": []byte("gzip"),
|
||||
s3_constants.AmzUserMetaMtime: []byte("12345"),
|
||||
s3_constants.SeaweedFSSSES3Key: []byte(`{"algorithm":"AES256","encryptedDEK":"abc"}`),
|
||||
}
|
||||
|
||||
tagging := buildTaggingString(extended)
|
||||
|
||||
if tagging != "" {
|
||||
t.Errorf("expected empty tagging when no tag keys present, got %q", tagging)
|
||||
}
|
||||
}
|
||||
@@ -295,10 +295,6 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient
|
||||
// This serves as an in-memory "static" configuration
|
||||
iam.loadEnvironmentVariableCredentials()
|
||||
|
||||
// Update credential manager with all static identities (file + env vars)
|
||||
// so that listing operations via filer gRPC also include them.
|
||||
iam.updateCredentialManagerStaticIdentities()
|
||||
|
||||
// Determine whether to enable S3 authentication based on configuration
|
||||
// For "weed mini" without any S3 config, default to allowing all access (isAuthEnabled = false)
|
||||
// If any credentials are configured (via file, filer, or env vars), enable authentication
|
||||
@@ -1073,59 +1069,6 @@ func (iam *IdentityAccessManagement) IsStaticIdentity(identityName string) bool
|
||||
return iam.staticIdentityNames[identityName]
|
||||
}
|
||||
|
||||
// updateCredentialManagerStaticIdentities syncs the current set of static
|
||||
// identities to the credential manager. Call this after any operation that
|
||||
// changes static identities (startup, config file reload, etc.).
|
||||
func (iam *IdentityAccessManagement) updateCredentialManagerStaticIdentities() {
|
||||
if iam.credentialManager != nil {
|
||||
iam.credentialManager.SetStaticIdentities(iam.GetStaticIdentities())
|
||||
}
|
||||
}
|
||||
|
||||
// GetStaticIdentities returns protobuf representations of all static identities.
|
||||
// This is used to include static identities in listing operations (ListUsers, etc.)
|
||||
func (iam *IdentityAccessManagement) GetStaticIdentities() []*iam_pb.Identity {
|
||||
iam.m.RLock()
|
||||
defer iam.m.RUnlock()
|
||||
|
||||
var result []*iam_pb.Identity
|
||||
for _, ident := range iam.identities {
|
||||
if !ident.IsStatic {
|
||||
continue
|
||||
}
|
||||
var policyNames []string
|
||||
if len(ident.PolicyNames) > 0 {
|
||||
policyNames = make([]string, len(ident.PolicyNames))
|
||||
copy(policyNames, ident.PolicyNames)
|
||||
}
|
||||
pbIdent := &iam_pb.Identity{
|
||||
Name: ident.Name,
|
||||
Disabled: ident.Disabled,
|
||||
PolicyNames: policyNames,
|
||||
IsStatic: true,
|
||||
}
|
||||
for _, action := range ident.Actions {
|
||||
pbIdent.Actions = append(pbIdent.Actions, string(action))
|
||||
}
|
||||
for _, cred := range ident.Credentials {
|
||||
pbIdent.Credentials = append(pbIdent.Credentials, &iam_pb.Credential{
|
||||
AccessKey: cred.AccessKey,
|
||||
SecretKey: cred.SecretKey,
|
||||
Status: cred.Status,
|
||||
})
|
||||
}
|
||||
if ident.Account != nil {
|
||||
pbIdent.Account = &iam_pb.Account{
|
||||
Id: ident.Account.Id,
|
||||
DisplayName: ident.Account.DisplayName,
|
||||
EmailAddress: ident.Account.EmailAddress,
|
||||
}
|
||||
}
|
||||
result = append(result, pbIdent)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {
|
||||
iam.m.RLock()
|
||||
defer iam.m.RUnlock()
|
||||
|
||||
+31
-157
@@ -65,12 +65,6 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
||||
|
||||
uploadIdString = uploadIdString + "_" + strings.ReplaceAll(uuid.New().String(), "-", "")
|
||||
|
||||
// Validate checksum algorithm before creating the upload directory
|
||||
_, checksumHeaderName, checksumErrCode := detectRequestedChecksumAlgorithm(r)
|
||||
if checksumErrCode != s3err.ErrNone {
|
||||
return nil, checksumErrCode
|
||||
}
|
||||
|
||||
// Prepare error handling outside callback scope
|
||||
var encryptionError error
|
||||
|
||||
@@ -101,12 +95,6 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
||||
}
|
||||
s3a.applyMultipartEncryptionConfig(entry, encryptionConfig)
|
||||
|
||||
// Store the requested checksum algorithm so CompleteMultipartUpload can compute
|
||||
// a composite checksum from per-part checksums
|
||||
if checksumHeaderName != "" {
|
||||
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(checksumHeaderName)
|
||||
}
|
||||
|
||||
// Extract and store object lock metadata from request headers
|
||||
// This ensures object lock settings from create_multipart_upload are preserved
|
||||
if err := s3a.extractObjectLockMetadataFromRequest(r, entry); err != nil {
|
||||
@@ -141,11 +129,6 @@ type CompleteMultipartUploadResult struct {
|
||||
Bucket *string `xml:"Bucket,omitempty"`
|
||||
Key *string `xml:"Key,omitempty"`
|
||||
ETag *string `xml:"ETag,omitempty"`
|
||||
|
||||
// Checksum fields — returned as HTTP response headers, not in the XML body
|
||||
ChecksumHeaderName string `xml:"-"`
|
||||
ChecksumValue string `xml:"-"`
|
||||
|
||||
// VersionId is NOT included in XML body - it should only be in x-amz-version-id HTTP header
|
||||
|
||||
// Store the VersionId internally for setting HTTP header, but don't marshal to XML
|
||||
@@ -190,17 +173,15 @@ type multipartPartBoundary struct {
|
||||
}
|
||||
|
||||
type multipartCompletionState struct {
|
||||
deleteEntries []*filer_pb.Entry
|
||||
partEntries map[int][]*filer_pb.Entry
|
||||
pentry *filer_pb.Entry
|
||||
mime string
|
||||
finalParts []*filer_pb.FileChunk
|
||||
offset int64
|
||||
partBoundaries []multipartPartBoundary
|
||||
multipartETag string
|
||||
entityWithTtl bool
|
||||
checksumHeaderName string // e.g. "X-Amz-Checksum-Crc32", empty if no checksum
|
||||
checksumValue string // composite base64 checksum with "-N" suffix
|
||||
deleteEntries []*filer_pb.Entry
|
||||
partEntries map[int][]*filer_pb.Entry
|
||||
pentry *filer_pb.Entry
|
||||
mime string
|
||||
finalParts []*filer_pb.FileChunk
|
||||
offset int64
|
||||
partBoundaries []multipartPartBoundary
|
||||
multipartETag string
|
||||
entityWithTtl bool
|
||||
}
|
||||
|
||||
func completeMultipartResult(r *http.Request, input *s3.CompleteMultipartUploadInput, etag string, entry *filer_pb.Entry) *CompleteMultipartUploadResult {
|
||||
@@ -365,36 +346,16 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *
|
||||
}
|
||||
}
|
||||
|
||||
// Compute composite checksum from per-part checksums if the upload
|
||||
// was initiated with a checksum algorithm (stored in upload dir entry)
|
||||
checksumHeaderName := ""
|
||||
checksumValue := ""
|
||||
if pentry.Extended != nil {
|
||||
if algoName, ok := pentry.Extended[s3_constants.ExtChecksumAlgorithm]; ok {
|
||||
checksumHeaderName = string(algoName)
|
||||
}
|
||||
}
|
||||
if checksumHeaderName != "" {
|
||||
var checksumErr error
|
||||
checksumValue, checksumErr = computeCompositeChecksum(checksumHeaderName, partEntries, completedPartNumbers)
|
||||
if checksumErr != nil {
|
||||
glog.Errorf("completeMultipartUpload: composite checksum computation failed: %v", checksumErr)
|
||||
return nil, nil, s3err.ErrInvalidPart
|
||||
}
|
||||
}
|
||||
|
||||
return &multipartCompletionState{
|
||||
deleteEntries: deleteEntries,
|
||||
partEntries: partEntries,
|
||||
pentry: pentry,
|
||||
mime: mime,
|
||||
finalParts: finalParts,
|
||||
offset: offset,
|
||||
partBoundaries: partBoundaries,
|
||||
multipartETag: calculateMultipartETag(partEntries, completedPartNumbers),
|
||||
entityWithTtl: entityWithTtl,
|
||||
checksumHeaderName: checksumHeaderName,
|
||||
checksumValue: checksumValue,
|
||||
deleteEntries: deleteEntries,
|
||||
partEntries: partEntries,
|
||||
pentry: pentry,
|
||||
mime: mime,
|
||||
finalParts: finalParts,
|
||||
offset: offset,
|
||||
partBoundaries: partBoundaries,
|
||||
multipartETag: calculateMultipartETag(partEntries, completedPartNumbers),
|
||||
entityWithTtl: entityWithTtl,
|
||||
}, nil, s3err.ErrNone
|
||||
}
|
||||
|
||||
@@ -481,11 +442,6 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
|
||||
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
||||
versionEntry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
||||
// Store composite checksum if computed from per-part checksums
|
||||
if completionState.checksumHeaderName != "" && completionState.checksumValue != "" {
|
||||
versionEntry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(completionState.checksumHeaderName)
|
||||
versionEntry.Extended[s3_constants.ExtChecksumValue] = []byte(completionState.checksumValue)
|
||||
}
|
||||
|
||||
// Preserve ALL SSE metadata from the first part (if any)
|
||||
// SSE metadata is stored in individual parts, not the upload directory
|
||||
@@ -534,13 +490,11 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
// For versioned buckets, all content is stored in .versions directory
|
||||
// The latest version information is tracked in the .versions directory metadata
|
||||
output = &CompleteMultipartUploadResult{
|
||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||
Bucket: input.Bucket,
|
||||
ETag: aws.String(etagQuote),
|
||||
Key: objectKey(input.Key),
|
||||
VersionId: aws.String(versionId),
|
||||
ChecksumHeaderName: completionState.checksumHeaderName,
|
||||
ChecksumValue: completionState.checksumValue,
|
||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||
Bucket: input.Bucket,
|
||||
ETag: aws.String(etagQuote),
|
||||
Key: objectKey(input.Key),
|
||||
VersionId: aws.String(versionId),
|
||||
}
|
||||
return s3err.ErrNone
|
||||
}
|
||||
@@ -580,11 +534,6 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
}
|
||||
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
||||
entry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
||||
// Store composite checksum if computed from per-part checksums
|
||||
if completionState.checksumHeaderName != "" && completionState.checksumValue != "" {
|
||||
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(completionState.checksumHeaderName)
|
||||
entry.Extended[s3_constants.ExtChecksumValue] = []byte(completionState.checksumValue)
|
||||
}
|
||||
if completionState.pentry.Attributes != nil && completionState.pentry.Attributes.Mime != "" {
|
||||
entry.Attributes.Mime = completionState.pentry.Attributes.Mime
|
||||
} else if completionState.mime != "" {
|
||||
@@ -598,12 +547,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
|
||||
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
|
||||
output = &CompleteMultipartUploadResult{
|
||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||
Bucket: input.Bucket,
|
||||
ETag: aws.String(etagQuote),
|
||||
Key: objectKey(input.Key),
|
||||
ChecksumHeaderName: completionState.checksumHeaderName,
|
||||
ChecksumValue: completionState.checksumValue,
|
||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||
Bucket: input.Bucket,
|
||||
ETag: aws.String(etagQuote),
|
||||
Key: objectKey(input.Key),
|
||||
// VersionId field intentionally omitted for suspended versioning
|
||||
}
|
||||
return s3err.ErrNone
|
||||
@@ -642,11 +589,6 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
}
|
||||
// Persist ETag to ensure subsequent HEAD/GET uses the same value
|
||||
entry.Extended[s3_constants.ExtETagKey] = []byte(completionState.multipartETag)
|
||||
// Store composite checksum if computed from per-part checksums
|
||||
if completionState.checksumHeaderName != "" && completionState.checksumValue != "" {
|
||||
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(completionState.checksumHeaderName)
|
||||
entry.Extended[s3_constants.ExtChecksumValue] = []byte(completionState.checksumValue)
|
||||
}
|
||||
if completionState.pentry.Attributes != nil && completionState.pentry.Attributes.Mime != "" {
|
||||
entry.Attributes.Mime = completionState.pentry.Attributes.Mime
|
||||
} else if completionState.mime != "" {
|
||||
@@ -664,12 +606,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
|
||||
// For non-versioned buckets, return response without VersionId
|
||||
output = &CompleteMultipartUploadResult{
|
||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||
Bucket: input.Bucket,
|
||||
ETag: aws.String(etagQuote),
|
||||
Key: objectKey(input.Key),
|
||||
ChecksumHeaderName: completionState.checksumHeaderName,
|
||||
ChecksumValue: completionState.checksumValue,
|
||||
Location: aws.String(fmt.Sprintf("%s://%s/%s/%s", getRequestScheme(r), r.Host, url.PathEscape(*input.Bucket), urlPathEscape(*input.Key))),
|
||||
Bucket: input.Bucket,
|
||||
ETag: aws.String(etagQuote),
|
||||
Key: objectKey(input.Key),
|
||||
}
|
||||
return s3err.ErrNone
|
||||
})
|
||||
@@ -1090,72 +1030,6 @@ func calculateMultipartETag(partEntries map[int][]*filer_pb.Entry, completedPart
|
||||
return fmt.Sprintf("%x-%d", md5.Sum(etags), len(completedPartNumbers))
|
||||
}
|
||||
|
||||
// computeCompositeChecksum computes a composite checksum from per-part checksums.
|
||||
// It concatenates the raw (decoded) per-part checksums, hashes the result with the
|
||||
// same algorithm, and returns the value as "base64-N" where N is the part count.
|
||||
// This follows the AWS S3 multipart checksum specification.
|
||||
// Returns an error if a part is missing its checksum (the upload was initiated with
|
||||
// a checksum algorithm, so all parts must have been uploaded with checksums).
|
||||
func computeCompositeChecksum(checksumHeaderName string, partEntries map[int][]*filer_pb.Entry, completedPartNumbers []int) (string, error) {
|
||||
// Determine the algorithm from the header name
|
||||
algo := checksumAlgorithmFromHeaderName(checksumHeaderName)
|
||||
if algo == ChecksumAlgorithmNone {
|
||||
return "", fmt.Errorf("unknown checksum algorithm for header %q", checksumHeaderName)
|
||||
}
|
||||
|
||||
// Collect raw per-part checksums
|
||||
var combined []byte
|
||||
for _, partNumber := range completedPartNumbers {
|
||||
entries, ok := partEntries[partNumber]
|
||||
if !ok || len(entries) == 0 {
|
||||
return "", fmt.Errorf("part %d not found", partNumber)
|
||||
}
|
||||
if len(entries) > 1 {
|
||||
sortEntriesByLatestChunk(entries)
|
||||
}
|
||||
entry := entries[0]
|
||||
if entry.Extended == nil {
|
||||
return "", fmt.Errorf("part %d missing checksum: upload initiated with %s but part was uploaded without a checksum", partNumber, checksumHeaderName)
|
||||
}
|
||||
// Validate the part's checksum algorithm matches the upload's expected algorithm
|
||||
partAlgo, ok := entry.Extended[s3_constants.ExtChecksumAlgorithm]
|
||||
if !ok || len(partAlgo) == 0 {
|
||||
return "", fmt.Errorf("part %d missing checksum: upload initiated with %s but part was uploaded without a checksum", partNumber, checksumHeaderName)
|
||||
}
|
||||
if string(partAlgo) != checksumHeaderName {
|
||||
return "", fmt.Errorf("part %d checksum algorithm mismatch: upload expects %s but part has %s", partNumber, checksumHeaderName, string(partAlgo))
|
||||
}
|
||||
partChecksumB64, ok := entry.Extended[s3_constants.ExtChecksumValue]
|
||||
if !ok || len(partChecksumB64) == 0 {
|
||||
return "", fmt.Errorf("part %d missing checksum value: upload initiated with %s but part has no checksum value", partNumber, checksumHeaderName)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(string(partChecksumB64))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("part %d has invalid checksum encoding: %w", partNumber, err)
|
||||
}
|
||||
combined = append(combined, raw...)
|
||||
}
|
||||
|
||||
// Hash the concatenated raw checksums
|
||||
h := getCheckSumWriter(algo)
|
||||
if h == nil {
|
||||
return "", fmt.Errorf("failed to create hash writer for %s", checksumHeaderName)
|
||||
}
|
||||
h.Write(combined)
|
||||
compositeRaw := h.Sum(nil)
|
||||
return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(compositeRaw), len(completedPartNumbers)), nil
|
||||
}
|
||||
|
||||
// checksumAlgorithmFromHeaderName maps a canonical header name back to its algorithm.
|
||||
func checksumAlgorithmFromHeaderName(headerName string) ChecksumAlgorithm {
|
||||
for _, entry := range checksumHeaders {
|
||||
if entry.name == headerName {
|
||||
return entry.alg
|
||||
}
|
||||
}
|
||||
return ChecksumAlgorithmNone
|
||||
}
|
||||
|
||||
func getEtagFromEntry(entry *filer_pb.Entry) string {
|
||||
if entry.Extended != nil {
|
||||
if etagBytes, ok := entry.Extended[s3_constants.ExtETagKey]; ok {
|
||||
|
||||
@@ -20,10 +20,6 @@ const (
|
||||
ExtLatestVersionIsDeleteMarker = "Seaweed-X-Amz-Latest-Version-Is-Delete-Marker"
|
||||
ExtMultipartObjectKey = "key"
|
||||
|
||||
// S3 checksum storage keys (use x-seaweedfs- prefix to avoid leaking in generic header loop)
|
||||
ExtChecksumAlgorithm = "x-seaweedfs-checksum-algorithm"
|
||||
ExtChecksumValue = "x-seaweedfs-checksum-value"
|
||||
|
||||
// Bucket Policy
|
||||
ExtBucketPolicyKey = "Seaweed-X-Amz-Bucket-Policy"
|
||||
|
||||
|
||||
@@ -76,16 +76,6 @@ const (
|
||||
AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
|
||||
AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
|
||||
|
||||
// S3 checksum headers
|
||||
AmzChecksumAlgorithm = "X-Amz-Checksum-Algorithm"
|
||||
AmzChecksumCRC32 = "X-Amz-Checksum-Crc32"
|
||||
AmzChecksumCRC32C = "X-Amz-Checksum-Crc32c"
|
||||
AmzChecksumCRC64NVME = "X-Amz-Checksum-Crc64nvme"
|
||||
AmzChecksumSHA1 = "X-Amz-Checksum-Sha1"
|
||||
AmzChecksumSHA256 = "X-Amz-Checksum-Sha256"
|
||||
AmzTrailer = "X-Amz-Trailer"
|
||||
AmzSdkChecksumAlgorithm = "X-Amz-Sdk-Checksum-Algorithm"
|
||||
|
||||
// S3 conditional headers
|
||||
IfMatch = "If-Match"
|
||||
IfNoneMatch = "If-None-Match"
|
||||
|
||||
@@ -215,7 +215,6 @@ func (e *EmbeddedIamApi) writeIamErrorResponse(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
// GetS3ApiConfiguration loads the S3 API configuration from the credential manager.
|
||||
// The credential manager automatically includes static identities in the result.
|
||||
func (e *EmbeddedIamApi) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error {
|
||||
if e.getS3ApiConfigurationFunc != nil {
|
||||
return e.getS3ApiConfigurationFunc(s3cfg)
|
||||
@@ -229,7 +228,6 @@ func (e *EmbeddedIamApi) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration)
|
||||
}
|
||||
|
||||
// PutS3ApiConfiguration saves the S3 API configuration to the credential manager.
|
||||
// The credential manager automatically filters out static identities before saving.
|
||||
func (e *EmbeddedIamApi) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error {
|
||||
if e.putS3ApiConfigurationFunc != nil {
|
||||
return e.putS3ApiConfigurationFunc(s3cfg)
|
||||
|
||||
@@ -1053,7 +1053,7 @@ func (s3a *S3ApiServer) streamFromVolumeServers(w http.ResponseWriter, r *http.R
|
||||
// Prepare streaming function with simple master client wrapper
|
||||
tStreamPrep := time.Now()
|
||||
// Use filerClient directly (not wrapped) so it can support cache invalidation
|
||||
streamFn, err := filer.PrepareStreamContentWithPrefetch(
|
||||
streamFn, err := filer.PrepareStreamContentWithThrottler(
|
||||
ctx,
|
||||
s3a.filerClient,
|
||||
filer.JwtForVolumeServer, // Use filer's JWT function (loads config once, generates JWT locally)
|
||||
@@ -1061,7 +1061,6 @@ func (s3a *S3ApiServer) streamFromVolumeServers(w http.ResponseWriter, r *http.R
|
||||
offset,
|
||||
size,
|
||||
0, // no throttling
|
||||
4, // prefetch 4 chunks ahead for overlapped fetching
|
||||
)
|
||||
streamPrepTime = time.Since(tStreamPrep)
|
||||
if err != nil {
|
||||
@@ -1929,7 +1928,7 @@ func (s3a *S3ApiServer) getEncryptedStreamFromVolumes(ctx context.Context, entry
|
||||
}
|
||||
|
||||
// Create streaming reader - use filerClient directly for cache invalidation support
|
||||
streamFn, err := filer.PrepareStreamContentWithPrefetch(
|
||||
streamFn, err := filer.PrepareStreamContentWithThrottler(
|
||||
ctx,
|
||||
s3a.filerClient,
|
||||
filer.JwtForVolumeServer, // Use filer's JWT function (loads config once, generates JWT locally)
|
||||
@@ -1937,7 +1936,6 @@ func (s3a *S3ApiServer) getEncryptedStreamFromVolumes(ctx context.Context, entry
|
||||
0,
|
||||
totalSize,
|
||||
0,
|
||||
4, // prefetch 4 chunks ahead for overlapped fetching
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2070,21 +2068,6 @@ func (s3a *S3ApiServer) setResponseHeaders(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
}
|
||||
|
||||
// Set checksum header if stored in metadata, but only when:
|
||||
// 1. The request contains "x-amz-checksum-mode: ENABLED" (per AWS S3 spec)
|
||||
// 2. The request is NOT a ranged GET (Range header absent)
|
||||
// The stored checksum covers the full object; returning it for partial
|
||||
// responses causes SDK checksum validation failures.
|
||||
if r != nil && r.Header.Get("X-Amz-Checksum-Mode") == "ENABLED" && r.Header.Get("Range") == "" {
|
||||
if entry.Extended != nil {
|
||||
if algoName, ok := entry.Extended[s3_constants.ExtChecksumAlgorithm]; ok {
|
||||
if checksumVal, ok := entry.Extended[s3_constants.ExtChecksumValue]; ok {
|
||||
w.Header().Set(string(algoName), string(checksumVal))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply S3 passthrough headers from query parameters
|
||||
// AWS S3 supports overriding response headers via query parameters like:
|
||||
// ?response-cache-control=no-cache&response-content-type=application/json
|
||||
|
||||
@@ -180,11 +180,6 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
|
||||
w.Header().Set("x-amz-version-id", *response.VersionId)
|
||||
}
|
||||
|
||||
// Set composite checksum header if present
|
||||
if response.ChecksumHeaderName != "" && response.ChecksumValue != "" {
|
||||
w.Header().Set(response.ChecksumHeaderName, response.ChecksumValue)
|
||||
}
|
||||
|
||||
stats_collect.RecordBucketActiveTime(bucket)
|
||||
stats_collect.S3UploadedObjectsCounter.WithLabelValues(bucket).Inc()
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -72,9 +71,6 @@ type SSEResponseMetadata struct {
|
||||
SSEType string
|
||||
KMSKeyID string
|
||||
BucketKeyEnabled bool
|
||||
// Checksum fields for S3 additional checksum support
|
||||
ChecksumHeaderName string // e.g. "X-Amz-Checksum-Sha256"
|
||||
ChecksumValue string // base64-encoded checksum value
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -362,19 +358,6 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
||||
plaintextHash := md5.New()
|
||||
dataReader = io.TeeReader(dataReader, plaintextHash)
|
||||
|
||||
// Detect and set up additional checksum computation (S3 checksum algorithm support)
|
||||
checksumAlgo, checksumHeaderName, checksumErrCode := detectRequestedChecksumAlgorithm(r)
|
||||
if checksumErrCode != s3err.ErrNone {
|
||||
return "", checksumErrCode, SSEResponseMetadata{}
|
||||
}
|
||||
var checksumHash hash.Hash
|
||||
if checksumAlgo != ChecksumAlgorithmNone {
|
||||
checksumHash = getCheckSumWriter(checksumAlgo)
|
||||
if checksumHash != nil {
|
||||
dataReader = io.TeeReader(dataReader, checksumHash)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle all SSE encryption types in a unified manner
|
||||
sseResult, sseErrorCode := s3a.handleAllSSEEncryption(r, dataReader, partOffset)
|
||||
if sseErrorCode != s3err.ErrNone {
|
||||
@@ -651,24 +634,6 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
||||
// Store ETag in Extended attribute for future retrieval (e.g. multipart parts)
|
||||
entry.Extended[s3_constants.ExtETagKey] = []byte(etag)
|
||||
|
||||
// Store additional checksum if one was computed
|
||||
checksumBase64 := ""
|
||||
if checksumHash != nil && checksumHeaderName != "" {
|
||||
checksumBase64 = base64.StdEncoding.EncodeToString(checksumHash.Sum(nil))
|
||||
// Verify against client-provided checksum if present in request headers
|
||||
// (non-chunked uploads send the value directly; chunked uploads validate in the reader)
|
||||
if expectedChecksum := r.Header.Get(checksumHeaderName); expectedChecksum != "" {
|
||||
if expectedChecksum != checksumBase64 {
|
||||
glog.Warningf("putToFiler: checksum mismatch for %s: expected %s, got %s", checksumHeaderName, expectedChecksum, checksumBase64)
|
||||
s3a.deleteOrphanedChunks(chunkResult.FileChunks)
|
||||
return "", s3err.ErrBadDigest, SSEResponseMetadata{}
|
||||
}
|
||||
}
|
||||
entry.Extended[s3_constants.ExtChecksumAlgorithm] = []byte(checksumHeaderName)
|
||||
entry.Extended[s3_constants.ExtChecksumValue] = []byte(checksumBase64)
|
||||
glog.V(3).Infof("putToFiler: stored checksum %s=%s for %s", checksumHeaderName, checksumBase64, filePath)
|
||||
}
|
||||
|
||||
// Set object owner according to bucket ownership settings.
|
||||
s3a.setObjectOwnerFromRequest(r, bucket, entry)
|
||||
|
||||
@@ -837,9 +802,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
||||
|
||||
// Build SSE response metadata with encryption details
|
||||
responseMetadata := SSEResponseMetadata{
|
||||
SSEType: sseType,
|
||||
ChecksumHeaderName: checksumHeaderName,
|
||||
ChecksumValue: checksumBase64,
|
||||
SSEType: sseType,
|
||||
}
|
||||
|
||||
// For SSE-KMS, include key ID and bucket-key-enabled flag from stored metadata
|
||||
@@ -853,93 +816,6 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
||||
return etag, s3err.ErrNone, responseMetadata
|
||||
}
|
||||
|
||||
// checksumAlgorithmMapping maps algorithm name strings to their enum and header name.
|
||||
var checksumAlgorithmMapping = map[string]struct {
|
||||
alg ChecksumAlgorithm
|
||||
name string
|
||||
}{
|
||||
"CRC32": {ChecksumAlgorithmCRC32, s3_constants.AmzChecksumCRC32},
|
||||
"CRC32C": {ChecksumAlgorithmCRC32C, s3_constants.AmzChecksumCRC32C},
|
||||
"CRC64NVME": {ChecksumAlgorithmCRC64NVMe, s3_constants.AmzChecksumCRC64NVME},
|
||||
"SHA1": {ChecksumAlgorithmSHA1, s3_constants.AmzChecksumSHA1},
|
||||
"SHA256": {ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
||||
}
|
||||
|
||||
// trailerToChecksumAlgorithm maps trailer header names to their algorithm and canonical header name.
|
||||
var trailerToChecksumAlgorithm = map[string]struct {
|
||||
alg ChecksumAlgorithm
|
||||
name string
|
||||
}{
|
||||
"x-amz-checksum-crc32": {ChecksumAlgorithmCRC32, s3_constants.AmzChecksumCRC32},
|
||||
"x-amz-checksum-crc32c": {ChecksumAlgorithmCRC32C, s3_constants.AmzChecksumCRC32C},
|
||||
"x-amz-checksum-crc64nvme": {ChecksumAlgorithmCRC64NVMe, s3_constants.AmzChecksumCRC64NVME},
|
||||
"x-amz-checksum-sha1": {ChecksumAlgorithmSHA1, s3_constants.AmzChecksumSHA1},
|
||||
"x-amz-checksum-sha256": {ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
||||
}
|
||||
|
||||
// checksumHeaders is the ordered list of individual checksum headers to check.
|
||||
// Using a slice ensures deterministic selection order.
|
||||
var checksumHeaders = []struct {
|
||||
header string
|
||||
alg ChecksumAlgorithm
|
||||
name string
|
||||
}{
|
||||
{s3_constants.AmzChecksumCRC32, ChecksumAlgorithmCRC32, s3_constants.AmzChecksumCRC32},
|
||||
{s3_constants.AmzChecksumCRC32C, ChecksumAlgorithmCRC32C, s3_constants.AmzChecksumCRC32C},
|
||||
{s3_constants.AmzChecksumCRC64NVME, ChecksumAlgorithmCRC64NVMe, s3_constants.AmzChecksumCRC64NVME},
|
||||
{s3_constants.AmzChecksumSHA1, ChecksumAlgorithmSHA1, s3_constants.AmzChecksumSHA1},
|
||||
{s3_constants.AmzChecksumSHA256, ChecksumAlgorithmSHA256, s3_constants.AmzChecksumSHA256},
|
||||
}
|
||||
|
||||
// detectRequestedChecksumAlgorithm detects the checksum algorithm requested by the client.
|
||||
// It checks the x-amz-sdk-checksum-algorithm header, x-amz-checksum-algorithm header,
|
||||
// x-amz-trailer header (including comma-separated values), and individual x-amz-checksum-*
|
||||
// headers. Returns the algorithm enum, the canonical HTTP header name, and an error code
|
||||
// if an unsupported algorithm is specified.
|
||||
func detectRequestedChecksumAlgorithm(r *http.Request) (ChecksumAlgorithm, string, s3err.ErrorCode) {
|
||||
// Check x-amz-sdk-checksum-algorithm (set by AWS SDKs)
|
||||
if algo := r.Header.Get(s3_constants.AmzSdkChecksumAlgorithm); algo != "" {
|
||||
if m, ok := checksumAlgorithmMapping[strings.ToUpper(algo)]; ok {
|
||||
return m.alg, m.name, s3err.ErrNone
|
||||
}
|
||||
glog.Warningf("unsupported checksum algorithm in %s: %q", s3_constants.AmzSdkChecksumAlgorithm, algo)
|
||||
return ChecksumAlgorithmNone, "", s3err.ErrInvalidRequest
|
||||
}
|
||||
|
||||
// Check x-amz-checksum-algorithm header
|
||||
if algo := r.Header.Get(s3_constants.AmzChecksumAlgorithm); algo != "" {
|
||||
if m, ok := checksumAlgorithmMapping[strings.ToUpper(algo)]; ok {
|
||||
return m.alg, m.name, s3err.ErrNone
|
||||
}
|
||||
glog.Warningf("unsupported checksum algorithm in %s: %q", s3_constants.AmzChecksumAlgorithm, algo)
|
||||
return ChecksumAlgorithmNone, "", s3err.ErrInvalidRequest
|
||||
}
|
||||
|
||||
// Check x-amz-trailer header (used by chunked uploads, may be comma-separated)
|
||||
if trailer := r.Header.Get(s3_constants.AmzTrailer); trailer != "" {
|
||||
for _, part := range strings.Split(trailer, ",") {
|
||||
part = strings.TrimSpace(strings.ToLower(part))
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
if m, ok := trailerToChecksumAlgorithm[part]; ok {
|
||||
return m.alg, m.name, s3err.ErrNone
|
||||
}
|
||||
// Non-checksum trailers (e.g. x-amz-server-side-encryption) are fine — skip them
|
||||
}
|
||||
}
|
||||
|
||||
// Check individual checksum headers (non-chunked uploads send the value directly)
|
||||
// Uses ordered slice for deterministic selection
|
||||
for _, entry := range checksumHeaders {
|
||||
if r.Header.Get(entry.header) != "" {
|
||||
return entry.alg, entry.name, s3err.ErrNone
|
||||
}
|
||||
}
|
||||
|
||||
return ChecksumAlgorithmNone, "", s3err.ErrNone
|
||||
}
|
||||
|
||||
const defaultFileMode = uint32(0660)
|
||||
|
||||
// resolveFileMode determines the file permission mode for an S3 upload.
|
||||
@@ -1007,11 +883,6 @@ func (s3a *S3ApiServer) setSSEResponseHeaders(w http.ResponseWriter, r *http.Req
|
||||
w.Header().Set(s3_constants.AmzServerSideEncryptionBucketKeyEnabled, "true")
|
||||
}
|
||||
}
|
||||
|
||||
// Set checksum response header if a checksum was computed
|
||||
if sseMetadata.ChecksumHeaderName != "" && sseMetadata.ChecksumValue != "" {
|
||||
w.Header().Set(sseMetadata.ChecksumHeaderName, sseMetadata.ChecksumValue)
|
||||
}
|
||||
}
|
||||
|
||||
func filerErrorToS3Error(err error) s3err.ErrorCode {
|
||||
|
||||
@@ -276,7 +276,6 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
|
||||
glog.Errorf("fail to load config file %s: %v", option.Config, err)
|
||||
} else {
|
||||
glog.V(1).Infof("Loaded %d identities from config file %s", len(s3ApiServer.iam.identities), option.Config)
|
||||
s3ApiServer.iam.updateCredentialManagerStaticIdentities()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -82,7 +82,6 @@ type FilerOption struct {
|
||||
AllowedOrigins []string
|
||||
ExposeDirectoryData bool
|
||||
TusBasePath string
|
||||
S3ConfigFile string // optional path to static S3 identity config file
|
||||
CredentialManager *credential.CredentialManager
|
||||
}
|
||||
|
||||
|
||||
@@ -101,10 +101,6 @@ func (s *IamGrpcServer) GetUser(ctx context.Context, req *iam_pb.GetUserRequest)
|
||||
identity, err := s.credentialManager.GetUser(ctx, req.Username)
|
||||
if err != nil {
|
||||
if err == credential.ErrUserNotFound {
|
||||
// Fall back to static identities (loaded from -s3.config file)
|
||||
if si := s.credentialManager.GetStaticIdentity(req.Username); si != nil {
|
||||
return &iam_pb.GetUserResponse{Identity: si}, nil
|
||||
}
|
||||
return nil, status.Errorf(codes.NotFound, "user %s not found", req.Username)
|
||||
}
|
||||
glog.Errorf("Failed to get user %s: %v", req.Username, err)
|
||||
@@ -170,20 +166,6 @@ func (s *IamGrpcServer) ListUsers(ctx context.Context, req *iam_pb.ListUsersRequ
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Merge static identities (from -s3.config file) into the result
|
||||
staticNames := s.credentialManager.GetStaticUsernames()
|
||||
if len(staticNames) > 0 {
|
||||
dynamicSet := make(map[string]bool, len(usernames))
|
||||
for _, name := range usernames {
|
||||
dynamicSet[name] = true
|
||||
}
|
||||
for _, name := range staticNames {
|
||||
if !dynamicSet[name] {
|
||||
usernames = append(usernames, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &iam_pb.ListUsersResponse{
|
||||
Usernames: usernames,
|
||||
}, nil
|
||||
|
||||
@@ -223,7 +223,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
||||
// Matches S3 API behavior. Request context (ctx) is used for metadata operations above.
|
||||
streamCtx, streamCancel := context.WithCancel(context.WithoutCancel(ctx))
|
||||
|
||||
streamFn, err := filer.PrepareStreamContentWithPrefetch(streamCtx, fs.filer.MasterClient, fs.maybeGetVolumeReadJwtAuthorizationToken, chunks, offset, size, fs.option.DownloadMaxBytesPs, 4)
|
||||
streamFn, err := filer.PrepareStreamContentWithThrottler(streamCtx, fs.filer.MasterClient, fs.maybeGetVolumeReadJwtAuthorizationToken, chunks, offset, size, fs.option.DownloadMaxBytesPs)
|
||||
if err != nil {
|
||||
streamCancel()
|
||||
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadStream).Inc()
|
||||
|
||||
@@ -41,7 +41,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
|
||||
// Add the current server itself (Peers() only returns other peers)
|
||||
resp.ClusterServers = append(resp.ClusterServers, &master_pb.RaftListClusterServersResponse_ClusterServers{
|
||||
Id: currentServerName,
|
||||
Address: ms.option.Master.ToGrpcAddress(),
|
||||
Address: string(ms.option.Master),
|
||||
Suffrage: "Voter",
|
||||
IsLeader: currentServerName == leader,
|
||||
})
|
||||
|
||||
@@ -32,7 +32,6 @@ type RaftServerOption struct {
|
||||
DataDir string
|
||||
Topo *topology.Topology
|
||||
RaftResumeState bool
|
||||
SingleMaster bool
|
||||
HeartbeatInterval time.Duration
|
||||
ElectionTimeout time.Duration
|
||||
RaftBootstrap bool
|
||||
@@ -177,38 +176,10 @@ func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
||||
if err := s.raftServer.LoadSnapshot(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// In single-master mode resuming state, the log is not empty so the
|
||||
// normal self-join path won't promote to leader. The server will
|
||||
// self-elect after the election timeout, so use a tiny timeout to
|
||||
// make this near-instant, then restore the original after election.
|
||||
fastResume := option.SingleMaster && option.RaftResumeState && !s.raftServer.IsLogEmpty()
|
||||
if fastResume {
|
||||
s.raftServer.SetElectionTimeout(time.Millisecond)
|
||||
}
|
||||
|
||||
if err := s.raftServer.Start(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fastResume {
|
||||
go func() {
|
||||
defer s.raftServer.SetElectionTimeout(option.ElectionTimeout)
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
timeout := time.After(option.ElectionTimeout)
|
||||
for s.raftServer.Leader() == "" {
|
||||
select {
|
||||
case <-timeout:
|
||||
glog.Warningf("Fast resume timed out waiting for leader election, restoring election timeout to %v", option.ElectionTimeout)
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
glog.V(0).Infof("Resumed as leader with election timeout restored to %v", option.ElectionTimeout)
|
||||
}()
|
||||
}
|
||||
|
||||
for name, peer := range s.peers {
|
||||
if err := s.raftServer.AddPeer(name, peer.ToGrpcAddress()); err != nil {
|
||||
return nil, err
|
||||
@@ -302,4 +273,3 @@ func (s *RaftServer) DoJoinCommand() {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -31,13 +31,8 @@ func (c *commandS3Configure) Name() string {
|
||||
func (c *commandS3Configure) Help() string {
|
||||
return `configure and apply s3 options for each bucket
|
||||
|
||||
# for workflow-oriented user management, prefer:
|
||||
s3.user.create -name=username -generate_credentials
|
||||
s3.user.show -name=username
|
||||
s3.policy.attach -user=username -name=mypolicy
|
||||
|
||||
# see the current configuration file content
|
||||
s3.configure
|
||||
# see the current configuration file content
|
||||
s3.configure
|
||||
|
||||
# create a new identity with account information
|
||||
s3.configure -user=username -actions=Read,Write,List,Tagging -buckets=bucket-name -policies=policy1,policy2 -access_key=key -secret_key=secret -account_id=id -account_display_name=name -account_email=email@example.com -apply
|
||||
|
||||
@@ -29,18 +29,10 @@ func (c *commandS3Policy) Name() string {
|
||||
func (c *commandS3Policy) Help() string {
|
||||
return `manage s3 policies
|
||||
|
||||
# preferred exact commands
|
||||
s3.policy.create -name=mypolicy -file=policy.json
|
||||
s3.policy.show -name=mypolicy
|
||||
s3.policy.list
|
||||
s3.policy.delete -name=mypolicy
|
||||
s3.policy.attach -user=alice -name=mypolicy
|
||||
s3.policy.detach -user=alice -name=mypolicy
|
||||
# create or update a policy
|
||||
s3.policy -put -name=mypolicy -file=policy.json
|
||||
|
||||
# create or update a policy
|
||||
s3.policy -put -name=mypolicy -file=policy.json
|
||||
|
||||
# list all policies
|
||||
# list all policies
|
||||
s3.policy -list
|
||||
|
||||
# get a policy
|
||||
|
||||
@@ -1,282 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands,
|
||||
&s3ShellCommand{
|
||||
name: "s3.policy.create",
|
||||
help: `create or update a managed S3 policy
|
||||
|
||||
s3.policy.create -name photos-rw -file policy.json`,
|
||||
do: runS3PolicyCreateCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.policy.show",
|
||||
help: `show one managed S3 policy
|
||||
|
||||
s3.policy.show -name photos-rw`,
|
||||
do: runS3PolicyShowCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.policy.list",
|
||||
help: `list managed S3 policies`,
|
||||
do: runS3PolicyListCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.policy.delete",
|
||||
help: `delete a managed S3 policy
|
||||
|
||||
s3.policy.delete -name photos-rw`,
|
||||
do: runS3PolicyDeleteCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.policy.attach",
|
||||
help: `attach a managed S3 policy to a filer-backed user
|
||||
|
||||
s3.policy.attach -user alice -name photos-rw`,
|
||||
do: runS3PolicyAttachCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.policy.detach",
|
||||
help: `detach a managed S3 policy from a filer-backed user
|
||||
|
||||
s3.policy.detach -user alice -name photos-rw`,
|
||||
do: runS3PolicyDetachCommand,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func runS3PolicyCreateCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.policy.create", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
name := fs.String("name", "", "policy name")
|
||||
file := fs.String("file", "", "policy file (json)")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
if *file == "" {
|
||||
return fmt.Errorf("-file is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3PolicyCreate(ctx, store, *name, *file, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3PolicyShowCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.policy.show", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
name := fs.String("name", "", "policy name")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3PolicyShow(ctx, store, *name, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3PolicyListCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.policy.list", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3PolicyList(ctx, store, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3PolicyDeleteCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.policy.delete", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
name := fs.String("name", "", "policy name")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3PolicyDelete(ctx, store, *name, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3PolicyAttachCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
return runS3PolicyAttachDetachCommand("s3.policy.attach", true, args, commandEnv, writer)
|
||||
}
|
||||
|
||||
func runS3PolicyDetachCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
return runS3PolicyAttachDetachCommand("s3.policy.detach", false, args, commandEnv, writer)
|
||||
}
|
||||
|
||||
func runS3PolicyAttachDetachCommand(commandName string, attach bool, args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet(commandName, flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
user := fs.String("user", "", "user name")
|
||||
name := fs.String("name", "", "policy name")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *user == "" {
|
||||
return fmt.Errorf("-user is required")
|
||||
}
|
||||
if *name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
if attach {
|
||||
return runS3PolicyAttach(ctx, store, *user, *name, writer)
|
||||
}
|
||||
return runS3PolicyDetach(ctx, store, *user, *name, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3PolicyCreate(ctx context.Context, store s3ShellStore, name, file string, writer io.Writer) error {
|
||||
document, err := loadPolicyDocumentFromFile(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := store.CreatePolicy(ctx, name, *document); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(writer, "Saved policy %q.\n", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3PolicyShow(ctx context.Context, store s3ShellStore, name string, writer io.Writer) error {
|
||||
document, err := store.GetPolicy(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if document == nil {
|
||||
return fmt.Errorf("policy %q not found", name)
|
||||
}
|
||||
formatted, err := marshalPolicyDocument(document)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(writer, formatted)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3PolicyList(ctx context.Context, store s3ShellStore, writer io.Writer) error {
|
||||
policies, err := store.GetPolicies(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(policies))
|
||||
for name := range policies {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
fmt.Fprintln(writer, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3PolicyDelete(ctx context.Context, store s3ShellStore, name string, writer io.Writer) error {
|
||||
if err := store.DeletePolicy(ctx, name); err != nil {
|
||||
if s3ShellIsNotFound(err) {
|
||||
return fmt.Errorf("policy %q not found", name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(writer, "Deleted policy %q.\n", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3PolicyAttach(ctx context.Context, store s3ShellStore, username, policyName string, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMutableIdentity(identity, "attach policies to"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := store.AttachUserPolicy(ctx, username, policyName); err != nil {
|
||||
if errors.Is(err, credential.ErrPolicyNotFound) {
|
||||
return fmt.Errorf("policy %q not found", policyName)
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(writer, "Attached policy %q to user %q.\n", policyName, username)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3PolicyDetach(ctx context.Context, store s3ShellStore, username, policyName string, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMutableIdentity(identity, "detach policies from"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := store.DetachUserPolicy(ctx, username, policyName); err != nil {
|
||||
if errors.Is(err, credential.ErrPolicyNotAttached) {
|
||||
return fmt.Errorf("policy %q is not attached to user %q", policyName, username)
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(writer, "Detached policy %q from user %q.\n", policyName, username)
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadPolicyDocumentFromFile(file string) (*policy_engine.PolicyDocument, error) {
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read policy file: %v", err)
|
||||
}
|
||||
var document policy_engine.PolicyDocument
|
||||
if err := json.Unmarshal(data, &document); err != nil {
|
||||
return nil, fmt.Errorf("invalid policy json: %v", err)
|
||||
}
|
||||
return &document, nil
|
||||
}
|
||||
|
||||
func marshalPolicyDocument(document *policy_engine.PolicyDocument) (string, error) {
|
||||
data, err := json.MarshalIndent(document, "", " ")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(data)), nil
|
||||
}
|
||||
@@ -1,712 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
iamgrpc "github.com/seaweedfs/seaweedfs/weed/credential/grpc"
|
||||
weediam "github.com/seaweedfs/seaweedfs/weed/iam"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
var errS3AccessKeyInUse = errors.New("access key already in use")
|
||||
|
||||
type s3ShellCommand struct {
|
||||
name string
|
||||
help string
|
||||
do func([]string, *CommandEnv, io.Writer) error
|
||||
}
|
||||
|
||||
func (c *s3ShellCommand) Name() string { return c.name }
|
||||
|
||||
func (c *s3ShellCommand) Help() string { return c.help }
|
||||
|
||||
func (c *s3ShellCommand) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
return c.do(args, commandEnv, writer)
|
||||
}
|
||||
|
||||
func (c *s3ShellCommand) HasTag(CommandTag) bool { return false }
|
||||
|
||||
type s3ShellStore interface {
|
||||
CreateUser(ctx context.Context, identity *iam_pb.Identity) error
|
||||
GetUser(ctx context.Context, username string) (*iam_pb.Identity, error)
|
||||
UpdateUser(ctx context.Context, username string, identity *iam_pb.Identity) error
|
||||
DeleteUser(ctx context.Context, username string) error
|
||||
ListUsers(ctx context.Context) ([]string, error)
|
||||
GetUserByAccessKey(ctx context.Context, accessKey string) (*iam_pb.Identity, error)
|
||||
CreateAccessKey(ctx context.Context, username string, credential *iam_pb.Credential) error
|
||||
DeleteAccessKey(ctx context.Context, username string, accessKey string) error
|
||||
GetPolicy(ctx context.Context, name string) (*policy_engine.PolicyDocument, error)
|
||||
GetPolicies(ctx context.Context) (map[string]policy_engine.PolicyDocument, error)
|
||||
CreatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error
|
||||
DeletePolicy(ctx context.Context, name string) error
|
||||
AttachUserPolicy(ctx context.Context, username string, policyName string) error
|
||||
DetachUserPolicy(ctx context.Context, username string, policyName string) error
|
||||
ListAttachedUserPolicies(ctx context.Context, username string) ([]string, error)
|
||||
}
|
||||
|
||||
type s3UserCreateOptions struct {
|
||||
name string
|
||||
accessKey string
|
||||
secretKey string
|
||||
generateCredentials bool
|
||||
accountID string
|
||||
displayName string
|
||||
email string
|
||||
}
|
||||
|
||||
type s3AccessKeyCreateOptions struct {
|
||||
username string
|
||||
accessKey string
|
||||
secretKey string
|
||||
generateCredentials bool
|
||||
}
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands,
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.list",
|
||||
help: `list S3 users with source and status`,
|
||||
do: runS3UserListCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.show",
|
||||
help: `show one S3 user's details
|
||||
|
||||
s3.user.show -name alice`,
|
||||
do: runS3UserShowCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.create",
|
||||
help: `create a filer-backed S3 user
|
||||
|
||||
s3.user.create -name alice -generate_credentials
|
||||
s3.user.create -name alice -access_key AKIA... -secret_key secret
|
||||
s3.user.create -name alice -email alice@example.com -display_name "Alice"`,
|
||||
do: runS3UserCreateCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.delete",
|
||||
help: `delete a filer-backed S3 user
|
||||
|
||||
s3.user.delete -name alice`,
|
||||
do: runS3UserDeleteCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.enable",
|
||||
help: `enable a filer-backed S3 user
|
||||
|
||||
s3.user.enable -name alice`,
|
||||
do: runS3UserEnableCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.disable",
|
||||
help: `disable a filer-backed S3 user
|
||||
|
||||
s3.user.disable -name alice`,
|
||||
do: runS3UserDisableCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.accesskey.list",
|
||||
help: `list access keys for a filer-backed S3 user
|
||||
|
||||
s3.user.accesskey.list -user alice`,
|
||||
do: runS3UserAccessKeyListCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.accesskey.create",
|
||||
help: `create an access key for a filer-backed S3 user
|
||||
|
||||
s3.user.accesskey.create -user alice -generate_credentials
|
||||
s3.user.accesskey.create -user alice -access_key AKIA... -secret_key secret`,
|
||||
do: runS3UserAccessKeyCreateCommand,
|
||||
},
|
||||
&s3ShellCommand{
|
||||
name: "s3.user.accesskey.delete",
|
||||
help: `delete an access key for a filer-backed S3 user
|
||||
|
||||
s3.user.accesskey.delete -user alice -access_key AKIA...`,
|
||||
do: runS3UserAccessKeyDeleteCommand,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func runS3UserListCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.user.list", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserList(ctx, store, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserShowCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.user.show", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
name := fs.String("name", "", "user name")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserShow(ctx, store, *name, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserCreateCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.user.create", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
opts := s3UserCreateOptions{}
|
||||
fs.StringVar(&opts.name, "name", "", "user name")
|
||||
fs.StringVar(&opts.accessKey, "access_key", "", "access key to create")
|
||||
fs.StringVar(&opts.secretKey, "secret_key", "", "secret key to create")
|
||||
fs.BoolVar(&opts.generateCredentials, "generate_credentials", false, "generate an initial access key pair")
|
||||
fs.StringVar(&opts.accountID, "account_id", "", "account id")
|
||||
fs.StringVar(&opts.displayName, "display_name", "", "account display name")
|
||||
fs.StringVar(&opts.email, "email", "", "account email")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if opts.name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserCreate(ctx, store, opts, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserDeleteCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.user.delete", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
name := fs.String("name", "", "user name")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserDelete(ctx, store, *name, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserEnableCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
return runS3UserStatusCommand("s3.user.enable", false, args, commandEnv, writer)
|
||||
}
|
||||
|
||||
func runS3UserDisableCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
return runS3UserStatusCommand("s3.user.disable", true, args, commandEnv, writer)
|
||||
}
|
||||
|
||||
func runS3UserStatusCommand(commandName string, disabled bool, args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet(commandName, flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
name := fs.String("name", "", "user name")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *name == "" {
|
||||
return fmt.Errorf("-name is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserSetDisabled(ctx, store, *name, disabled, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserAccessKeyListCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.user.accesskey.list", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
user := fs.String("user", "", "user name")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *user == "" {
|
||||
return fmt.Errorf("-user is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserAccessKeyList(ctx, store, *user, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserAccessKeyCreateCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.user.accesskey.create", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
opts := s3AccessKeyCreateOptions{}
|
||||
fs.StringVar(&opts.username, "user", "", "user name")
|
||||
fs.StringVar(&opts.accessKey, "access_key", "", "access key to create")
|
||||
fs.StringVar(&opts.secretKey, "secret_key", "", "secret key to create")
|
||||
fs.BoolVar(&opts.generateCredentials, "generate_credentials", false, "generate an access key pair")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if opts.username == "" {
|
||||
return fmt.Errorf("-user is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserAccessKeyCreate(ctx, store, opts, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserAccessKeyDeleteCommand(args []string, commandEnv *CommandEnv, writer io.Writer) error {
|
||||
fs := flag.NewFlagSet("s3.user.accesskey.delete", flag.ContinueOnError)
|
||||
fs.SetOutput(writer)
|
||||
user := fs.String("user", "", "user name")
|
||||
accessKey := fs.String("access_key", "", "access key to delete")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if *user == "" {
|
||||
return fmt.Errorf("-user is required")
|
||||
}
|
||||
if *accessKey == "" {
|
||||
return fmt.Errorf("-access_key is required")
|
||||
}
|
||||
|
||||
return withS3ShellStore(commandEnv, func(ctx context.Context, store s3ShellStore) error {
|
||||
return runS3UserAccessKeyDelete(ctx, store, *user, *accessKey, writer)
|
||||
})
|
||||
}
|
||||
|
||||
func runS3UserList(ctx context.Context, store s3ShellStore, writer io.Writer) error {
|
||||
usernames, err := store.ListUsers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Strings(usernames)
|
||||
|
||||
tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "NAME\tSOURCE\tSTATUS\tACCESS KEYS\tPOLICIES")
|
||||
for _, username := range usernames {
|
||||
identity, err := store.GetUser(ctx, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\t%d\t%d\n",
|
||||
identity.Name,
|
||||
s3IdentitySource(identity),
|
||||
s3IdentityStatus(identity),
|
||||
len(identity.Credentials),
|
||||
len(identity.PolicyNames),
|
||||
)
|
||||
}
|
||||
return tw.Flush()
|
||||
}
|
||||
|
||||
func runS3UserShow(ctx context.Context, store s3ShellStore, username string, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
writeS3IdentityDetails(writer, identity)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3UserCreate(ctx context.Context, store s3ShellStore, opts s3UserCreateOptions, writer io.Writer) error {
|
||||
existing, err := store.GetUser(ctx, opts.name)
|
||||
switch {
|
||||
case err == nil && existing != nil:
|
||||
if existing.IsStatic {
|
||||
return fmt.Errorf("user %q already exists in -s3.config; edit the static config for bootstrap users", opts.name)
|
||||
}
|
||||
return fmt.Errorf("user %q already exists", opts.name)
|
||||
case err != nil && !s3ShellIsNotFound(err):
|
||||
return err
|
||||
}
|
||||
|
||||
identity := &iam_pb.Identity{Name: opts.name}
|
||||
if opts.accountID != "" || opts.displayName != "" || opts.email != "" {
|
||||
identity.Account = &iam_pb.Account{
|
||||
Id: opts.accountID,
|
||||
DisplayName: opts.displayName,
|
||||
EmailAddress: opts.email,
|
||||
}
|
||||
}
|
||||
|
||||
createdCredential, err := s3BuildCredentialForCreate(ctx, store, opts.accessKey, opts.secretKey, opts.generateCredentials || opts.accessKey != "" || opts.secretKey != "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if createdCredential != nil {
|
||||
identity.Credentials = []*iam_pb.Credential{createdCredential}
|
||||
}
|
||||
|
||||
if err := store.CreateUser(ctx, identity); err != nil {
|
||||
if s3ShellIsAlreadyExists(err) {
|
||||
return fmt.Errorf("user %q already exists", opts.name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintf(writer, "Created user %q.\n", opts.name)
|
||||
if createdCredential != nil {
|
||||
writeCreatedCredential(writer, createdCredential)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3UserDelete(ctx context.Context, store s3ShellStore, username string, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMutableIdentity(identity, "delete"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := store.DeleteUser(ctx, username); err != nil {
|
||||
if s3ShellIsNotFound(err) {
|
||||
return fmt.Errorf("user %q not found", username)
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(writer, "Deleted user %q.\n", username)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3UserSetDisabled(ctx context.Context, store s3ShellStore, username string, disabled bool, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMutableIdentity(identity, ternary(disabled, "disable", "enable")); err != nil {
|
||||
return err
|
||||
}
|
||||
if identity.Disabled == disabled {
|
||||
fmt.Fprintf(writer, "User %q is already %s.\n", username, s3IdentityStatus(identity))
|
||||
return nil
|
||||
}
|
||||
|
||||
updated := proto.Clone(identity).(*iam_pb.Identity)
|
||||
updated.Disabled = disabled
|
||||
if err := store.UpdateUser(ctx, username, updated); err != nil {
|
||||
return err
|
||||
}
|
||||
if disabled {
|
||||
fmt.Fprintf(writer, "Disabled user %q.\n", username)
|
||||
} else {
|
||||
fmt.Fprintf(writer, "Enabled user %q.\n", username)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3UserAccessKeyList(ctx context.Context, store s3ShellStore, username string, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tw := tabwriter.NewWriter(writer, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "USER\tACCESS KEY\tSTATUS")
|
||||
for _, credential := range sortedCredentials(identity.Credentials) {
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\n", username, credential.AccessKey, s3CredentialStatus(credential))
|
||||
}
|
||||
if len(identity.Credentials) == 0 {
|
||||
fmt.Fprintf(tw, "%s\t%s\t%s\n", username, "-", "-")
|
||||
}
|
||||
return tw.Flush()
|
||||
}
|
||||
|
||||
func runS3UserAccessKeyCreate(ctx context.Context, store s3ShellStore, opts s3AccessKeyCreateOptions, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, opts.username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMutableIdentity(identity, "create access keys for"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
createdCredential, err := s3BuildCredentialForCreate(ctx, store, opts.accessKey, opts.secretKey, opts.generateCredentials || opts.accessKey != "" || opts.secretKey != "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if createdCredential == nil {
|
||||
return fmt.Errorf("set -generate_credentials or provide -access_key and/or -secret_key")
|
||||
}
|
||||
|
||||
if err := store.CreateAccessKey(ctx, opts.username, createdCredential); err != nil {
|
||||
if s3ShellIsAlreadyExists(err) {
|
||||
return fmt.Errorf("access key %q is already in use", createdCredential.AccessKey)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Fprintf(writer, "Created access key for user %q.\n", opts.username)
|
||||
writeCreatedCredential(writer, createdCredential)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runS3UserAccessKeyDelete(ctx context.Context, store s3ShellStore, username string, accessKey string, writer io.Writer) error {
|
||||
identity, err := getS3User(ctx, store, username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureMutableIdentity(identity, "delete access keys for"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := store.DeleteAccessKey(ctx, username, accessKey); err != nil {
|
||||
if s3ShellIsNotFound(err) {
|
||||
return fmt.Errorf("access key %q not found for user %q", accessKey, username)
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(writer, "Deleted access key %q for user %q.\n", accessKey, username)
|
||||
return nil
|
||||
}
|
||||
|
||||
func withS3ShellStore(commandEnv *CommandEnv, fn func(ctx context.Context, store s3ShellStore) error) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
store := &iamgrpc.IamGrpcStore{}
|
||||
store.SetFilerAddressFunc(func() pb.ServerAddress {
|
||||
return commandEnv.option.FilerAddress
|
||||
}, commandEnv.option.GrpcDialOption)
|
||||
|
||||
return fn(ctx, store)
|
||||
}
|
||||
|
||||
func ensureMutableIdentity(identity *iam_pb.Identity, action string) error {
|
||||
if identity != nil && identity.IsStatic {
|
||||
return fmt.Errorf("cannot %s user %q because it comes from -s3.config; edit the static config for bootstrap users", action, identity.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getS3User(ctx context.Context, store s3ShellStore, username string) (*iam_pb.Identity, error) {
|
||||
identity, err := store.GetUser(ctx, username)
|
||||
if err != nil {
|
||||
if s3ShellIsNotFound(err) {
|
||||
return nil, fmt.Errorf("user %q not found", username)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func s3BuildCredentialForCreate(ctx context.Context, store s3ShellStore, accessKey, secretKey string, shouldCreate bool) (*iam_pb.Credential, error) {
|
||||
if !shouldCreate {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if accessKey == "" {
|
||||
var err error
|
||||
accessKey, err = generateUniqueAccessKey(ctx, store)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := ensureAccessKeyAvailable(ctx, store, accessKey); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if secretKey == "" {
|
||||
var err error
|
||||
secretKey, err = weediam.GenerateSecretAccessKey()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("generate secret key: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &iam_pb.Credential{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
Status: weediam.AccessKeyStatusActive,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ensureAccessKeyAvailable(ctx context.Context, store s3ShellStore, accessKey string) error {
|
||||
_, err := store.GetUserByAccessKey(ctx, accessKey)
|
||||
if err == nil {
|
||||
return fmt.Errorf("%w: %s", errS3AccessKeyInUse, accessKey)
|
||||
}
|
||||
if s3ShellIsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func generateUniqueAccessKey(ctx context.Context, store s3ShellStore) (string, error) {
|
||||
for range 16 {
|
||||
accessKey, err := generateAccessKey()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := ensureAccessKeyAvailable(ctx, store, accessKey); err == nil {
|
||||
return accessKey, nil
|
||||
} else if !errors.Is(err, errS3AccessKeyInUse) {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("failed to generate a unique access key")
|
||||
}
|
||||
|
||||
func generateAccessKey() (string, error) {
|
||||
const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
var raw [20]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", fmt.Errorf("generate access key: %w", err)
|
||||
}
|
||||
for i := range raw {
|
||||
raw[i] = charset[int(raw[i])%len(charset)]
|
||||
}
|
||||
return string(raw[:]), nil
|
||||
}
|
||||
|
||||
func s3IdentitySource(identity *iam_pb.Identity) string {
|
||||
if identity != nil && identity.IsStatic {
|
||||
return "static"
|
||||
}
|
||||
return "dynamic"
|
||||
}
|
||||
|
||||
func s3IdentityStatus(identity *iam_pb.Identity) string {
|
||||
if identity != nil && identity.Disabled {
|
||||
return "disabled"
|
||||
}
|
||||
return "enabled"
|
||||
}
|
||||
|
||||
func s3CredentialStatus(credential *iam_pb.Credential) string {
|
||||
if credential == nil || credential.Status == "" {
|
||||
return weediam.AccessKeyStatusActive
|
||||
}
|
||||
return credential.Status
|
||||
}
|
||||
|
||||
func sortedCredentials(credentials []*iam_pb.Credential) []*iam_pb.Credential {
|
||||
out := append([]*iam_pb.Credential(nil), credentials...)
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return out[i].AccessKey < out[j].AccessKey
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func writeCreatedCredential(writer io.Writer, credential *iam_pb.Credential) {
|
||||
fmt.Fprintf(writer, "Access Key: %s\n", credential.AccessKey)
|
||||
fmt.Fprintf(writer, "Secret Key: %s\n", credential.SecretKey)
|
||||
}
|
||||
|
||||
func writeS3IdentityDetails(writer io.Writer, identity *iam_pb.Identity) {
|
||||
fmt.Fprintf(writer, "Name: %s\n", identity.Name)
|
||||
fmt.Fprintf(writer, "Source: %s\n", s3IdentitySource(identity))
|
||||
fmt.Fprintf(writer, "Status: %s\n", s3IdentityStatus(identity))
|
||||
if identity.Account != nil {
|
||||
if identity.Account.Id != "" {
|
||||
fmt.Fprintf(writer, "Account ID: %s\n", identity.Account.Id)
|
||||
}
|
||||
if identity.Account.DisplayName != "" {
|
||||
fmt.Fprintf(writer, "Display Name: %s\n", identity.Account.DisplayName)
|
||||
}
|
||||
if identity.Account.EmailAddress != "" {
|
||||
fmt.Fprintf(writer, "Email: %s\n", identity.Account.EmailAddress)
|
||||
}
|
||||
}
|
||||
|
||||
if len(identity.Credentials) == 0 {
|
||||
fmt.Fprintln(writer, "Access Keys: none")
|
||||
} else {
|
||||
fmt.Fprintln(writer, "Access Keys:")
|
||||
for _, credential := range sortedCredentials(identity.Credentials) {
|
||||
fmt.Fprintf(writer, " %s (%s)\n", credential.AccessKey, s3CredentialStatus(credential))
|
||||
}
|
||||
}
|
||||
|
||||
if len(identity.PolicyNames) == 0 {
|
||||
fmt.Fprintln(writer, "Policies: none")
|
||||
} else {
|
||||
policies := append([]string(nil), identity.PolicyNames...)
|
||||
sort.Strings(policies)
|
||||
fmt.Fprintf(writer, "Policies: %s\n", strings.Join(policies, ", "))
|
||||
}
|
||||
|
||||
if len(identity.Actions) == 0 {
|
||||
fmt.Fprintln(writer, "Actions: none")
|
||||
} else {
|
||||
actions := append([]string(nil), identity.Actions...)
|
||||
sort.Strings(actions)
|
||||
fmt.Fprintf(writer, "Actions: %s\n", strings.Join(actions, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
func s3ShellIsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, credential.ErrUserNotFound) ||
|
||||
errors.Is(err, credential.ErrAccessKeyNotFound) ||
|
||||
errors.Is(err, credential.ErrPolicyNotFound) {
|
||||
return true
|
||||
}
|
||||
if st, ok := status.FromError(err); ok && st.Code() == codes.NotFound {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func s3ShellIsAlreadyExists(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, credential.ErrUserAlreadyExists) {
|
||||
return true
|
||||
}
|
||||
if st, ok := status.FromError(err); ok && st.Code() == codes.AlreadyExists {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ternary[T any](cond bool, onTrue, onFalse T) T {
|
||||
if cond {
|
||||
return onTrue
|
||||
}
|
||||
return onFalse
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential/memory"
|
||||
weediam "github.com/seaweedfs/seaweedfs/weed/iam"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
)
|
||||
|
||||
func newTestS3ShellStore(t *testing.T) *memory.MemoryStore {
|
||||
t.Helper()
|
||||
|
||||
store := &memory.MemoryStore{}
|
||||
if err := store.Initialize(nil, ""); err != nil {
|
||||
t.Fatalf("initialize memory store: %v", err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func TestRunS3UserCreateGeneratesCredentials(t *testing.T) {
|
||||
store := newTestS3ShellStore(t)
|
||||
|
||||
var out bytes.Buffer
|
||||
err := runS3UserCreate(context.Background(), store, s3UserCreateOptions{
|
||||
name: "alice",
|
||||
generateCredentials: true,
|
||||
email: "alice@example.com",
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runS3UserCreate: %v", err)
|
||||
}
|
||||
|
||||
user, err := store.GetUser(context.Background(), "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("GetUser: %v", err)
|
||||
}
|
||||
if len(user.Credentials) != 1 {
|
||||
t.Fatalf("expected 1 credential, got %d", len(user.Credentials))
|
||||
}
|
||||
if got := user.Credentials[0].Status; got != weediam.AccessKeyStatusActive {
|
||||
t.Fatalf("expected %q status, got %q", weediam.AccessKeyStatusActive, got)
|
||||
}
|
||||
if user.Account == nil || user.Account.EmailAddress != "alice@example.com" {
|
||||
t.Fatalf("expected account email to be set, got %+v", user.Account)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
if !strings.Contains(output, `Created user "alice".`) {
|
||||
t.Fatalf("expected created message, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "Access Key:") || !strings.Contains(output, "Secret Key:") {
|
||||
t.Fatalf("expected credential output, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunS3UserSetDisabledRejectsStaticUser(t *testing.T) {
|
||||
store := newTestS3ShellStore(t)
|
||||
if err := store.CreateUser(context.Background(), &iam_pb.Identity{
|
||||
Name: "bootstrap-admin",
|
||||
IsStatic: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateUser: %v", err)
|
||||
}
|
||||
|
||||
err := runS3UserSetDisabled(context.Background(), store, "bootstrap-admin", true, &bytes.Buffer{})
|
||||
if err == nil {
|
||||
t.Fatal("expected static user mutation to fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "-s3.config") {
|
||||
t.Fatalf("expected static user guidance, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunS3UserListIncludesSourceAndStatus(t *testing.T) {
|
||||
store := newTestS3ShellStore(t)
|
||||
if err := store.CreateUser(context.Background(), &iam_pb.Identity{
|
||||
Name: "alice",
|
||||
Credentials: []*iam_pb.Credential{
|
||||
{AccessKey: "ALICEKEY", SecretKey: "secret", Status: weediam.AccessKeyStatusActive},
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateUser alice: %v", err)
|
||||
}
|
||||
if err := store.CreateUser(context.Background(), &iam_pb.Identity{
|
||||
Name: "bootstrap-admin",
|
||||
IsStatic: true,
|
||||
Disabled: true,
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateUser bootstrap-admin: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := runS3UserList(context.Background(), store, &out); err != nil {
|
||||
t.Fatalf("runS3UserList: %v", err)
|
||||
}
|
||||
|
||||
output := out.String()
|
||||
if !strings.Contains(output, "NAME") || !strings.Contains(output, "SOURCE") || !strings.Contains(output, "STATUS") {
|
||||
t.Fatalf("expected header, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "alice") || !strings.Contains(output, "dynamic") || !strings.Contains(output, "enabled") {
|
||||
t.Fatalf("expected dynamic user row, got %q", output)
|
||||
}
|
||||
if !strings.Contains(output, "bootstrap-admin") || !strings.Contains(output, "static") || !strings.Contains(output, "disabled") {
|
||||
t.Fatalf("expected static user row, got %q", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunS3UserAccessKeyCreateGeneratesCredential(t *testing.T) {
|
||||
store := newTestS3ShellStore(t)
|
||||
if err := store.CreateUser(context.Background(), &iam_pb.Identity{Name: "alice"}); err != nil {
|
||||
t.Fatalf("CreateUser: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
err := runS3UserAccessKeyCreate(context.Background(), store, s3AccessKeyCreateOptions{
|
||||
username: "alice",
|
||||
generateCredentials: true,
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("runS3UserAccessKeyCreate: %v", err)
|
||||
}
|
||||
|
||||
user, err := store.GetUser(context.Background(), "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("GetUser: %v", err)
|
||||
}
|
||||
if len(user.Credentials) != 1 {
|
||||
t.Fatalf("expected 1 credential, got %d", len(user.Credentials))
|
||||
}
|
||||
if !strings.Contains(out.String(), `Created access key for user "alice".`) {
|
||||
t.Fatalf("expected create message, got %q", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunS3PolicyAttachUpdatesUserPolicies(t *testing.T) {
|
||||
store := newTestS3ShellStore(t)
|
||||
if err := store.CreateUser(context.Background(), &iam_pb.Identity{Name: "alice"}); err != nil {
|
||||
t.Fatalf("CreateUser: %v", err)
|
||||
}
|
||||
if err := store.CreatePolicy(context.Background(), "photos-rw", policy_engine.PolicyDocument{
|
||||
Version: "2012-10-17",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreatePolicy: %v", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := runS3PolicyAttach(context.Background(), store, "alice", "photos-rw", &out); err != nil {
|
||||
t.Fatalf("runS3PolicyAttach: %v", err)
|
||||
}
|
||||
|
||||
user, err := store.GetUser(context.Background(), "alice")
|
||||
if err != nil {
|
||||
t.Fatalf("GetUser: %v", err)
|
||||
}
|
||||
if len(user.PolicyNames) != 1 || user.PolicyNames[0] != "photos-rw" {
|
||||
t.Fatalf("expected attached policy, got %v", user.PolicyNames)
|
||||
}
|
||||
if !strings.Contains(out.String(), `Attached policy "photos-rw" to user "alice".`) {
|
||||
t.Fatalf("expected attach message, got %q", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunS3PolicyDetachRejectsStaticUser(t *testing.T) {
|
||||
store := newTestS3ShellStore(t)
|
||||
if err := store.CreateUser(context.Background(), &iam_pb.Identity{
|
||||
Name: "bootstrap-admin",
|
||||
IsStatic: true,
|
||||
PolicyNames: []string{"photos-rw"},
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateUser: %v", err)
|
||||
}
|
||||
if err := store.CreatePolicy(context.Background(), "photos-rw", policy_engine.PolicyDocument{
|
||||
Version: "2012-10-17",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreatePolicy: %v", err)
|
||||
}
|
||||
|
||||
err := runS3PolicyDetach(context.Background(), store, "bootstrap-admin", "photos-rw", &bytes.Buffer{})
|
||||
if err == nil {
|
||||
t.Fatal("expected static user mutation to fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "-s3.config") {
|
||||
t.Fatalf("expected static user guidance, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -15,8 +16,11 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
)
|
||||
|
||||
@@ -386,10 +390,30 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co
|
||||
return true, nil
|
||||
}
|
||||
|
||||
err := replicateVolumeToServer(commandEnv.option.GrpcDialOption, writer, needle.VolumeId(replica.info.Id),
|
||||
pb.NewServerAddressFromDataNode(replica.location.dataNode),
|
||||
pb.NewServerAddressFromDataNode(dst.dataNode),
|
||||
replica.info.DiskType)
|
||||
err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dst.dataNode), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
||||
stream, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: replica.info.Id,
|
||||
SourceDataNode: string(pb.NewServerAddressFromDataNode(replica.location.dataNode)),
|
||||
})
|
||||
if replicateErr != nil {
|
||||
return fmt.Errorf("copying from %s => %s : %v", replica.location.dataNode.Id, dst.dataNode.Id, replicateErr)
|
||||
}
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
} else {
|
||||
return recvErr
|
||||
}
|
||||
}
|
||||
if resp.ProcessedBytes > 0 {
|
||||
fmt.Fprintf(writer, "volume %d processed %s bytes\n", replica.info.Id, util.BytesToHumanReadable(uint64(resp.ProcessedBytes)))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
|
||||
@@ -2,7 +2,6 @@ package shell
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -241,47 +240,3 @@ func markVolumeReplicasWritable(grpcDialOption grpc.DialOption, volumeId needle.
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replicateVolumeToServer copies a volume from sourceAddress to targetAddress via the VolumeCopy gRPC stream.
|
||||
func replicateVolumeToServer(grpcDialOption grpc.DialOption, writer io.Writer, volumeId needle.VolumeId, sourceAddress, targetAddress pb.ServerAddress, diskType string) error {
|
||||
return operation.WithVolumeServerClient(false, targetAddress, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
||||
stream, replicateErr := volumeServerClient.VolumeCopy(context.Background(), &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: uint32(volumeId),
|
||||
SourceDataNode: string(sourceAddress),
|
||||
DiskType: diskType,
|
||||
})
|
||||
if replicateErr != nil {
|
||||
return replicateErr
|
||||
}
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return recvErr
|
||||
}
|
||||
if resp.ProcessedBytes > 0 {
|
||||
fmt.Fprintf(writer, "volume %d processed %s bytes\n", volumeId, util.BytesToHumanReadable(uint64(resp.ProcessedBytes)))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// configureVolumeReplication sets the replication setting on a volume at the given server.
|
||||
func configureVolumeReplication(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, targetAddress pb.ServerAddress, replicationString string) error {
|
||||
return operation.WithVolumeServerClient(false, targetAddress, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
||||
resp, configureErr := volumeServerClient.VolumeConfigure(context.Background(), &volume_server_pb.VolumeConfigureRequest{
|
||||
VolumeId: uint32(volumeId),
|
||||
Replication: replicationString,
|
||||
})
|
||||
if configureErr != nil {
|
||||
return configureErr
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return errors.New(resp.Error)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -11,10 +13,11 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
@@ -44,12 +47,8 @@ func (c *commandVolumeTierMove) Help() string {
|
||||
|
||||
volume.tier.move -fromDiskType=hdd -toDiskType=ssd [-collectionPattern=""] [-fullPercent=95] [-quietFor=1h] [-parallelLimit=4] [-toReplication=XYZ]
|
||||
|
||||
The command ensures the target replication is fully achieved on the destination tier
|
||||
before deleting old replicas. This prevents data loss if a destination disk fails
|
||||
before replication repair completes.
|
||||
|
||||
When -toReplication is specified, the moved volume is reconfigured with the new
|
||||
replication setting. Otherwise, the volume's existing replication is preserved.
|
||||
Even if the volume is replicated, only one replica will be changed and the rest replicas will be dropped.
|
||||
So "volume.fix.replication" and "volume.balance" should be followed.
|
||||
|
||||
Note:
|
||||
Use -collectionPattern="_default" to match only the default collection (volumes with no collection name).
|
||||
@@ -272,201 +271,36 @@ func (c *commandVolumeTierMove) doMoveOneVolume(commandEnv *CommandEnv, writer i
|
||||
|
||||
// If move is successful and replication is not empty, alter moved volume's replication setting
|
||||
if *replicationString != "" {
|
||||
if err = configureVolumeReplication(commandEnv.option.GrpcDialOption, vid, newAddress, *replicationString); err != nil {
|
||||
// LiveMoveVolume already deleted sourceVolumeServer; mark surviving
|
||||
// old replicas writable before aborting so the volume stays accessible.
|
||||
restoreSurvivingReplicasWritable(commandEnv, vid, locations, sourceVolumeServer)
|
||||
return fmt.Errorf("configure replication %s on volume %d at %s: %v", *replicationString, vid, newAddress, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the required number of replicas exist on the target tier BEFORE
|
||||
// deleting old replicas to avoid data-loss risk.
|
||||
// Use the explicit -toReplication if given, otherwise preserve the volume's
|
||||
// existing replication from the source tier.
|
||||
preserveServers, replicateErr := c.ensureReplicationFulfilled(commandEnv, writer, vid, toDiskType, dst, *replicationString)
|
||||
if replicateErr != nil {
|
||||
// Replication not fully achieved — do NOT delete old replicas.
|
||||
restoreSurvivingReplicasWritable(commandEnv, vid, locations, sourceVolumeServer)
|
||||
return fmt.Errorf("volume %d moved to %s but failed to fulfill replication, old replicas preserved: %v", vid, dst.dataNode.Id, replicateErr)
|
||||
}
|
||||
|
||||
// Mark preserved pre-existing target-tier replicas as writable.
|
||||
// They were marked read-only at the start of the move and would otherwise
|
||||
// stay read-only since we're keeping rather than deleting them.
|
||||
for _, loc := range locations {
|
||||
if preserveServers[loc.Url] {
|
||||
if markErr := markVolumeWritable(commandEnv.option.GrpcDialOption, vid, loc.ServerAddress(), true, false); markErr != nil {
|
||||
glog.Errorf("mark volume %d as writable on preserved replica %s: %v", vid, loc.Url, markErr)
|
||||
err = operation.WithVolumeServerClient(false, newAddress, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
||||
resp, configureErr := volumeServerClient.VolumeConfigure(context.Background(), &volume_server_pb.VolumeConfigureRequest{
|
||||
VolumeId: uint32(vid),
|
||||
Replication: *replicationString,
|
||||
})
|
||||
if configureErr != nil {
|
||||
return configureErr
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return errors.New(resp.Error)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.Errorf("update volume %d replication on %s: %v", vid, locations[0].Url, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove old replicas that are NOT needed by the fulfilled replication.
|
||||
// Skip the move destination, the already-deleted source, and any pre-existing
|
||||
// target-tier replicas that were counted toward replication fulfillment.
|
||||
// remove the remaining replicas
|
||||
for _, loc := range locations {
|
||||
if loc.Url == dst.dataNode.Id || loc.ServerAddress() == sourceVolumeServer {
|
||||
continue
|
||||
}
|
||||
if preserveServers[loc.Url] {
|
||||
continue
|
||||
}
|
||||
if err = deleteVolume(commandEnv.option.GrpcDialOption, vid, loc.ServerAddress(), false); err != nil {
|
||||
fmt.Fprintf(writer, "failed to delete volume %d on %s: %v\n", vid, loc.Url, err)
|
||||
if loc.Url != dst.dataNode.Id && loc.ServerAddress() != sourceVolumeServer {
|
||||
if err = deleteVolume(commandEnv.option.GrpcDialOption, vid, loc.ServerAddress(), false); err != nil {
|
||||
fmt.Fprintf(writer, "failed to delete volume %d on %s: %v\n", vid, loc.Url, err)
|
||||
}
|
||||
// reduce volume count? Not really necessary since they are "more" full and will not be a candidate to move to
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreSurvivingReplicasWritable marks old replicas writable after a failure,
|
||||
// skipping the source that was already deleted by LiveMoveVolume.
|
||||
func restoreSurvivingReplicasWritable(commandEnv *CommandEnv, vid needle.VolumeId, locations []wdclient.Location, deletedSource pb.ServerAddress) {
|
||||
for _, loc := range locations {
|
||||
if loc.ServerAddress() == deletedSource {
|
||||
continue
|
||||
}
|
||||
if markErr := markVolumeWritable(commandEnv.option.GrpcDialOption, vid, loc.ServerAddress(), true, false); markErr != nil {
|
||||
glog.Errorf("mark volume %d as writable on %s: %v", vid, loc.Url, markErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensureReplicationFulfilled creates additional replicas of the volume on the target tier
|
||||
// to satisfy the requested replication placement. It re-collects topology after the initial
|
||||
// move so it can see the newly placed volume and find suitable destinations for additional copies.
|
||||
// It returns a set of server URLs (from the original locations) that host target-tier replicas
|
||||
// counted toward fulfillment, so the caller can avoid deleting them during cleanup.
|
||||
func (c *commandVolumeTierMove) ensureReplicationFulfilled(commandEnv *CommandEnv, writer io.Writer, vid needle.VolumeId, toDiskType types.DiskType, movedDst location, replicationString string) (preserveServers map[string]bool, err error) {
|
||||
preserveServers = make(map[string]bool)
|
||||
sourceAddress := pb.NewServerAddressFromDataNode(movedDst.dataNode)
|
||||
|
||||
// Wait briefly for the master to receive heartbeats reflecting the move,
|
||||
// then re-collect topology to get the current state.
|
||||
topologyInfo, _, err := collectTopologyInfo(commandEnv, 5*time.Second)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("collect topology: %v", err)
|
||||
}
|
||||
|
||||
volumeReplicas, allLocations := collectVolumeReplicaLocations(topologyInfo)
|
||||
allLocations = filterLocationsByDiskType(allLocations, toDiskType)
|
||||
keepDataNodesSorted(allLocations, toDiskType)
|
||||
|
||||
existingReplicas := volumeReplicas[uint32(vid)]
|
||||
if len(existingReplicas) == 0 {
|
||||
return nil, fmt.Errorf("volume %d not found in topology after move", vid)
|
||||
}
|
||||
|
||||
// Build a set of all data nodes that already host this volume (any disk type)
|
||||
// so we don't try to VolumeCopy to a server that already has it.
|
||||
nodesWithVolume := make(map[string]bool)
|
||||
for _, r := range existingReplicas {
|
||||
nodesWithVolume[r.location.dataNode.Id] = true
|
||||
}
|
||||
|
||||
// Determine the target replication: use explicit -toReplication if given,
|
||||
// otherwise read the volume's existing replication setting.
|
||||
var replicaPlacement *super_block.ReplicaPlacement
|
||||
if replicationString != "" {
|
||||
replicaPlacement, err = super_block.NewReplicaPlacementFromString(replicationString)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse replication %s: %v", replicationString, err)
|
||||
}
|
||||
} else {
|
||||
replicaPlacement, err = super_block.NewReplicaPlacementFromByte(byte(existingReplicas[0].info.ReplicaPlacement))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse existing replication for volume %d: %v", vid, err)
|
||||
}
|
||||
}
|
||||
|
||||
requiredCopies := replicaPlacement.GetCopyCount()
|
||||
if requiredCopies <= 1 {
|
||||
// No additional replicas needed (e.g., replication "000")
|
||||
return preserveServers, nil
|
||||
}
|
||||
|
||||
// Filter to only replicas on the target disk type (the newly moved one).
|
||||
var targetTierReplicas []*VolumeReplica
|
||||
for _, r := range existingReplicas {
|
||||
if types.ToDiskType(r.info.DiskType) == toDiskType {
|
||||
targetTierReplicas = append(targetTierReplicas, r)
|
||||
// Track pre-existing target-tier replicas so the caller won't delete them.
|
||||
preserveServers[r.location.dataNode.Id] = true
|
||||
}
|
||||
}
|
||||
if len(targetTierReplicas) == 0 {
|
||||
return nil, fmt.Errorf("volume %d not found on target tier %s in topology after move", vid, toDiskType)
|
||||
}
|
||||
|
||||
// Ensure all existing target-tier replicas have the correct replication metadata.
|
||||
// The primary moved replica is already configured in doMoveOneVolume, but there may
|
||||
// be pre-existing replicas on the target tier that need updating.
|
||||
if replicationString != "" {
|
||||
for _, r := range targetTierReplicas {
|
||||
addr := pb.NewServerAddressFromDataNode(r.location.dataNode)
|
||||
if configErr := configureVolumeReplication(commandEnv.option.GrpcDialOption, vid, addr, replicationString); configErr != nil {
|
||||
return nil, fmt.Errorf("volume %d: failed to configure replication on existing replica %s: %v", vid, r.location.dataNode.Id, configErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
additionalCopiesNeeded := requiredCopies - len(targetTierReplicas)
|
||||
if additionalCopiesNeeded <= 0 {
|
||||
return preserveServers, nil
|
||||
}
|
||||
|
||||
fmt.Fprintf(writer, "volume %d: creating %d additional replica(s) for replication %s\n", vid, additionalCopiesNeeded, replicaPlacement)
|
||||
|
||||
fn := capacityByFreeVolumeCount(toDiskType)
|
||||
copiesMade := 0
|
||||
for _, candidateDst := range allLocations {
|
||||
if copiesMade >= additionalCopiesNeeded {
|
||||
break
|
||||
}
|
||||
if fn(candidateDst.dataNode) <= 0 {
|
||||
continue
|
||||
}
|
||||
// Skip nodes that already host this volume on any disk type to avoid
|
||||
// VolumeCopy conflicts (e.g., same volume on source tier and target tier).
|
||||
if nodesWithVolume[candidateDst.dataNode.Id] {
|
||||
continue
|
||||
}
|
||||
if !satisfyReplicaPlacement(replicaPlacement, targetTierReplicas, candidateDst) {
|
||||
continue
|
||||
}
|
||||
|
||||
candidateAddress := pb.NewServerAddressFromDataNode(candidateDst.dataNode)
|
||||
fmt.Fprintf(writer, "volume %d: replicating from %s to %s\n", vid, sourceAddress, candidateDst.dataNode.Id)
|
||||
|
||||
if copyErr := replicateVolumeToServer(commandEnv.option.GrpcDialOption, writer, vid, sourceAddress, candidateAddress, toDiskType.ReadableString()); copyErr != nil {
|
||||
return nil, fmt.Errorf("replicate volume %d to %s: %v", vid, candidateDst.dataNode.Id, copyErr)
|
||||
}
|
||||
|
||||
// Configure replication on the new replica if an explicit -toReplication was given.
|
||||
// Without it, VolumeCopy already preserves the source's replication from the super block.
|
||||
if replicationString != "" {
|
||||
if configErr := configureVolumeReplication(commandEnv.option.GrpcDialOption, vid, candidateAddress, replicationString); configErr != nil {
|
||||
return nil, fmt.Errorf("volume %d: failed to configure replication on %s: %v", vid, candidateDst.dataNode.Id, configErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Track the new replica for placement decisions
|
||||
targetTierReplicas = append(targetTierReplicas, &VolumeReplica{
|
||||
location: &candidateDst,
|
||||
info: targetTierReplicas[0].info,
|
||||
})
|
||||
addVolumeCount(candidateDst.dataNode.DiskInfos[string(toDiskType)], 1)
|
||||
copiesMade++
|
||||
}
|
||||
|
||||
if copiesMade < additionalCopiesNeeded {
|
||||
return nil, fmt.Errorf("could only create %d of %d additional replicas for volume %d (replication %s): not enough eligible destinations", copiesMade, additionalCopiesNeeded, vid, replicaPlacement)
|
||||
}
|
||||
|
||||
fmt.Fprintf(writer, "volume %d: replication %s fulfilled with %d total copies\n", vid, replicaPlacement, requiredCopies)
|
||||
return preserveServers, nil
|
||||
}
|
||||
|
||||
func collectVolumeIdsForTierChange(topologyInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, sourceTier types.DiskType, collectionPattern string, fullPercentage float64, quietPeriod time.Duration) (vids []needle.VolumeId, err error) {
|
||||
|
||||
quietSeconds := int64(quietPeriod / time.Second)
|
||||
|
||||
@@ -17,9 +17,7 @@ func fillInDiskStatus(disk *volume_server_pb.DiskStatus) {
|
||||
|
||||
disk.All = fs.F_blocks * uint64(fs.F_bsize)
|
||||
disk.Free = fs.F_bfree * uint64(fs.F_bsize)
|
||||
disk.Used = disk.All - disk.Free
|
||||
disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
|
||||
disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
|
||||
calculateDiskRemaining(disk)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -17,9 +17,7 @@ func fillInDiskStatus(disk *volume_server_pb.DiskStatus) {
|
||||
|
||||
disk.All = stat.Blocks * uint64(stat.Bsize)
|
||||
disk.Free = stat.Bfree * uint64(stat.Bsize)
|
||||
disk.Used = disk.All - disk.Free
|
||||
disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
|
||||
disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
|
||||
calculateDiskRemaining(disk)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -40,9 +40,7 @@ func fillInDiskStatus(disk *volume_server_pb.DiskStatus) {
|
||||
|
||||
return
|
||||
}
|
||||
disk.Used = disk.All - disk.Free
|
||||
disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
|
||||
disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
|
||||
calculateDiskRemaining(disk)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -15,8 +15,8 @@ import (
|
||||
// The server runs in a goroutine and serves pprof endpoints at /debug/pprof/*.
|
||||
func StartDebugServer(debugPort int) {
|
||||
go func() {
|
||||
addr := fmt.Sprintf("127.0.0.1:%d", debugPort)
|
||||
glog.V(0).Infof("Starting debug server for pprof at http://%s/debug/pprof/", addr)
|
||||
addr := fmt.Sprintf(":%d", debugPort)
|
||||
glog.V(0).Infof("Starting debug server for pprof at http://localhost%s/debug/pprof/", addr)
|
||||
if err := http.ListenAndServe(addr, nil); err != nil && err != http.ErrServerClosed {
|
||||
glog.Errorf("Failed to start debug server on %s: %v", addr, err)
|
||||
}
|
||||
|
||||
@@ -372,7 +372,7 @@ func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte,
|
||||
var (
|
||||
m int
|
||||
)
|
||||
buf := mem.Allocate(256 * 1024)
|
||||
buf := mem.Allocate(64 * 1024)
|
||||
defer mem.Free(buf)
|
||||
|
||||
for {
|
||||
|
||||
@@ -3,8 +3,6 @@ package ec_balance
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -24,8 +22,7 @@ type ECBalanceTask struct {
|
||||
volumeID uint32
|
||||
collection string
|
||||
grpcDialOption grpc.DialOption
|
||||
progress uint64 // atomic; stores float64 bits via math.Float64bits
|
||||
reporting int32 // atomic; re-entry guard to prevent recursive reportProgress calls
|
||||
progress float64
|
||||
}
|
||||
|
||||
// NewECBalanceTask creates a new EC balance task instance
|
||||
@@ -209,16 +206,12 @@ func (t *ECBalanceTask) EstimateTime(params *worker_pb.TaskParams) time.Duration
|
||||
|
||||
// GetProgress returns current progress
|
||||
func (t *ECBalanceTask) GetProgress() float64 {
|
||||
return math.Float64frombits(atomic.LoadUint64(&t.progress))
|
||||
return t.progress
|
||||
}
|
||||
|
||||
// reportProgress updates the stored progress and reports it via the callback
|
||||
func (t *ECBalanceTask) reportProgress(progress float64, stage string) {
|
||||
if !atomic.CompareAndSwapInt32(&t.reporting, 0, 1) {
|
||||
return
|
||||
}
|
||||
defer atomic.StoreInt32(&t.reporting, 0)
|
||||
atomic.StoreUint64(&t.progress, math.Float64bits(progress))
|
||||
t.progress = progress
|
||||
t.ReportProgressWithStage(progress, stage)
|
||||
glog.Infof("EC balance volume %d: [%.2f] %s", t.volumeID, progress, stage)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user