mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-20 21:42:33 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Copilot
|
91b834bd9d | ||
|
Copilot
|
2f84fdfc8d | ||
|
Copilot
|
c820280996 | ||
|
Copilot
|
2033ec28d7 | ||
|
Copilot
|
e3ef3a5b56 | ||
|
Copilot
|
0a4ec5ba15 |
@@ -1 +0,0 @@
|
||||
{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}
|
||||
@@ -18,7 +18,6 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
|
||||
@@ -9,96 +9,17 @@ permissions:
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -109,40 +30,40 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v7
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -23,22 +23,15 @@ on:
|
||||
- all
|
||||
- standard
|
||||
- large_disk
|
||||
publish:
|
||||
description: 'Publish images and manifests'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
variants: ${{ steps.set-variants.outputs.variants }}
|
||||
publish: ${{ steps.set-publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Select variants for this run
|
||||
id: set-variants
|
||||
@@ -49,89 +42,9 @@ jobs:
|
||||
variants='["standard","large_disk"]'
|
||||
fi
|
||||
echo "variants=$variants" >> "$GITHUB_OUTPUT"
|
||||
- name: Select publish mode
|
||||
id: set-publish
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
needs: [setup]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -166,32 +79,11 @@ jobs:
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
@@ -219,14 +111,14 @@ jobs:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -237,7 +129,7 @@ jobs:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ needs.setup.outputs.publish == 'true' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
@@ -257,151 +149,10 @@ jobs:
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Build local scan image tarball
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64
|
||||
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
# since they all use the same alpine base, so a single-arch scan
|
||||
# provides sufficient coverage without multiplying CI time.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
- name: Trivy gate (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, trivy-scan]
|
||||
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
|
||||
needs: [setup, build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
@@ -429,12 +180,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -39,82 +39,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
|
||||
# emulated cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
# ── Build Docker containers ─────────────────────────────────────────
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# Build sequentially to avoid rate limits
|
||||
@@ -127,23 +52,20 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk - multi-arch
|
||||
|
||||
# Large disk - multi-arch
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# Full tags - multi-arch
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- variant: large_disk_full
|
||||
@@ -151,42 +73,19 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
rust_variant: large-disk
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -236,14 +135,14 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -299,14 +198,14 @@ jobs:
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v5
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
run: |
|
||||
cd docker
|
||||
make build_e2e || {
|
||||
echo "Retrying without buildx cache..."
|
||||
make binary_race
|
||||
docker build --no-cache -t chrislusf/seaweedfs:e2e -f Dockerfile.e2e .
|
||||
}
|
||||
|
||||
- name: Build pjdfstest image
|
||||
run: |
|
||||
docker build -t chrislusf/seaweedfs:pjdfstest test/pjdfstest/
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
@@ -156,7 +156,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -169,7 +169,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ jobs:
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@@ -48,7 +47,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -119,7 +118,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
|
||||
@@ -39,19 +39,11 @@ jobs:
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -88,7 +80,6 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
@@ -97,15 +88,6 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -130,7 +112,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -165,7 +147,6 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
@@ -174,15 +155,6 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -198,7 +170,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -234,7 +206,6 @@ jobs:
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
@@ -242,12 +213,3 @@ jobs:
|
||||
weed-volume_windows_amd64.zip
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
name: "S3 Mutation Regression Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/delete/**'
|
||||
- 'test/s3/distributed_lock/**'
|
||||
- 'test/s3/versioning/**'
|
||||
- 'test/volume_server/framework/**'
|
||||
- 'docker/compose/s3.json'
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-mutation-regression-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload versioning logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-regression-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload delete logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-delete-regression-logs
|
||||
path: test/s3/delete/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run distributed lock regressions
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
||||
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
||||
|
||||
- name: Upload distributed lock logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-distributed-lock-regression-logs
|
||||
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
||||
retention-days: 3
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"setup": [],
|
||||
"teardown": [],
|
||||
"run": []
|
||||
}
|
||||
@@ -42,7 +42,8 @@ test: admin-generate
|
||||
|
||||
# Admin component targets
|
||||
admin-generate:
|
||||
@cd $(ADMIN_DIR) && $(MAKE) generate
|
||||
@echo "Generating admin component templates..."
|
||||
@cd $(ADMIN_DIR) && templ generate ./view
|
||||
|
||||
admin-build: admin-generate
|
||||
@echo "Building admin component..."
|
||||
|
||||
@@ -16,22 +16,15 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
# Rust volume server: use pre-built binary from CI when available (placed in
|
||||
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
|
||||
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
|
||||
# for non-native architectures.
|
||||
FROM alpine:3.23 as rust_builder
|
||||
# Rust volume server builder (amd64/arm64 only)
|
||||
FROM rust:1-alpine as rust_builder
|
||||
ARG TARGETARCH
|
||||
ARG TAGS
|
||||
COPY weed-volume-prebuilt/ /prebuilt/
|
||||
RUN apk add musl-dev protobuf-dev git
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/proto /build/proto
|
||||
WORKDIR /build/seaweed-volume
|
||||
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
|
||||
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
|
||||
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
|
||||
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
|
||||
ARG TAGS
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
if [ "$TAGS" = "5BytesOffset" ]; then \
|
||||
cargo build --release; \
|
||||
else \
|
||||
@@ -56,8 +49,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse curl su-exec libgcc && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -7,8 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse curl su-exec && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -34,8 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -17,8 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "steward",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "steward-key",
|
||||
"secretKey": "steward-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "le001",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "le001-key",
|
||||
"secretKey": "le001-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -34,7 +34,7 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
@@ -83,10 +83,10 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/viant/ptrie v1.0.1
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.etcd.io/etcd/client/v3 v3.6.7
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.45.0
|
||||
@@ -94,26 +94,27 @@ require (
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
|
||||
golang.org/x/image v0.38.0
|
||||
golang.org/x/net v0.52.0
|
||||
golang.org/x/image v0.36.0
|
||||
golang.org/x/net v0.51.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.274.0
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/api v0.267.0
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.46.1
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.26.0
|
||||
cloud.google.com/go/kms v1.25.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
@@ -123,21 +124,22 @@ require (
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/getsentry/sentry-go v0.43.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.28.0
|
||||
@@ -155,7 +157,7 @@ require (
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
@@ -179,7 +181,7 @@ require (
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -289,7 +291,7 @@ require (
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth v0.18.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
@@ -321,22 +323,22 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
@@ -369,7 +371,7 @@ require (
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
@@ -385,7 +387,7 @@ require (
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
@@ -484,22 +486,22 @@ require (
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.41.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
|
||||
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
|
||||
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
||||
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
|
||||
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
|
||||
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
|
||||
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
|
||||
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
|
||||
cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
|
||||
cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
|
||||
cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
|
||||
cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
|
||||
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
|
||||
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
|
||||
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
|
||||
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA=
|
||||
cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
@@ -714,48 +714,48 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
|
||||
github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
@@ -1076,8 +1076,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
|
||||
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
|
||||
@@ -1383,8 +1383,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
|
||||
github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
|
||||
github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
|
||||
github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
|
||||
github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
@@ -1838,6 +1838,8 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
@@ -2036,8 +2038,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@@ -2101,12 +2103,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
|
||||
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
@@ -2128,8 +2130,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
|
||||
@@ -2144,14 +2146,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
@@ -2240,8 +2242,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2344,8 +2346,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -2552,8 +2554,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -2703,8 +2705,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
||||
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -2838,12 +2840,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.19"
|
||||
appVersion: "4.17"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.19.0
|
||||
version: 4.17.0
|
||||
|
||||
@@ -1339,14 +1339,9 @@ worker:
|
||||
# Admin server to connect to
|
||||
adminServer: ""
|
||||
|
||||
# Worker job types - categories or comma-separated list of names/aliases
|
||||
# Categories: all, default, heavy
|
||||
# default: vacuum, volume_balance, ec_balance, admin_script
|
||||
# heavy: erasure_coding, iceberg_maintenance
|
||||
# all: every registered job type
|
||||
# Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
|
||||
# Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
|
||||
jobType: "all"
|
||||
# Worker job types - comma-separated list
|
||||
# Available: vacuum, volume_balance, erasure_coding
|
||||
jobType: "vacuum,volume_balance,erasure_coding"
|
||||
|
||||
# Maximum number of concurrent detection requests
|
||||
maxDetect: 1
|
||||
|
||||
@@ -84,8 +84,6 @@ service SeaweedFiler {
|
||||
// distributed lock management internal use only
|
||||
rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
|
||||
}
|
||||
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
@@ -524,7 +522,6 @@ message LockResponse {
|
||||
string lock_owner = 2;
|
||||
string lock_host_moved_to = 3;
|
||||
string error = 4;
|
||||
int64 generation = 5;
|
||||
}
|
||||
message UnlockRequest {
|
||||
string name = 1;
|
||||
@@ -547,26 +544,12 @@ message Lock {
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_backup = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message TransferLocksRequest {
|
||||
repeated Lock locks = 1;
|
||||
}
|
||||
message TransferLocksResponse {
|
||||
}
|
||||
message ReplicateLockRequest {
|
||||
string name = 1;
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_unlock = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message ReplicateLockResponse {
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
|
||||
|
||||
Generated
+310
-150
File diff suppressed because it is too large
Load Diff
@@ -20,7 +20,7 @@ default = ["5bytes"]
|
||||
[dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
tokio-stream = "0.1"
|
||||
tokio-io-timeout = "1"
|
||||
|
||||
# gRPC + protobuf
|
||||
@@ -65,6 +65,8 @@ reed-solomon-erasure = "6"
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
pprof = { version = "0.15", features = ["prost-codec"] }
|
||||
|
||||
# Config
|
||||
toml = "0.8"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -82,7 +84,7 @@ memmap2 = "0.9"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
# HTTP client (for proxying, remote fetch)
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
reqwest = { version = "0.12", features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
|
||||
# Content hashing
|
||||
md-5 = "0.10"
|
||||
@@ -125,10 +127,6 @@ aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4
|
||||
aws-credential-types = "1"
|
||||
aws-types = "1"
|
||||
|
||||
# pprof is Unix-only (requires libc/nix APIs not available on Windows)
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
pprof = { version = "0.15", features = ["prost-codec"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
|
||||
+37
-85
@@ -10,11 +10,9 @@ use seaweed_volume::security::tls::{
|
||||
GrpcClientAuthPolicy, TlsPolicy,
|
||||
};
|
||||
use seaweed_volume::security::{Guard, SigningKey};
|
||||
#[cfg(unix)]
|
||||
use seaweed_volume::server::debug::build_debug_router;
|
||||
use seaweed_volume::server::grpc_client::load_outgoing_grpc_tls;
|
||||
use seaweed_volume::server::grpc_server::VolumeGrpcService;
|
||||
#[cfg(unix)]
|
||||
use seaweed_volume::server::profiling::CpuProfileSession;
|
||||
use seaweed_volume::server::request_id::GrpcRequestIdLayer;
|
||||
use seaweed_volume::server::volume_server::{
|
||||
@@ -26,11 +24,6 @@ use seaweed_volume::storage::types::DiskType;
|
||||
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
|
||||
#[cfg(unix)]
|
||||
type CpuProfileParam = Option<CpuProfileSession>;
|
||||
#[cfg(not(unix))]
|
||||
type CpuProfileParam = Option<()>;
|
||||
|
||||
const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
|
||||
const GRPC_KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
const GRPC_KEEPALIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
@@ -49,7 +42,6 @@ fn main() {
|
||||
|
||||
let config = config::parse_cli();
|
||||
seaweed_volume::server::server_stats::init_process_start();
|
||||
#[cfg(unix)]
|
||||
let cpu_profile = match CpuProfileSession::start(&config) {
|
||||
Ok(session) => session,
|
||||
Err(e) => {
|
||||
@@ -57,8 +49,6 @@ fn main() {
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let cpu_profile: Option<()> = None;
|
||||
info!(
|
||||
"SeaweedFS Volume Server (Rust) v{}",
|
||||
seaweed_volume::version::full_version()
|
||||
@@ -267,7 +257,7 @@ where
|
||||
|
||||
async fn run(
|
||||
config: VolumeServerConfig,
|
||||
#[allow(unused_variables)] cpu_profile: CpuProfileParam,
|
||||
cpu_profile: Option<CpuProfileSession>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Initialize the store
|
||||
let mut store = Store::new(config.index_type);
|
||||
@@ -441,12 +431,10 @@ async fn run(
|
||||
}
|
||||
|
||||
// Build HTTP routers
|
||||
#[allow(unused_mut)]
|
||||
let mut admin_router = seaweed_volume::server::volume_server::build_admin_router_with_ui(
|
||||
state.clone(),
|
||||
config.ui_enabled,
|
||||
);
|
||||
#[cfg(unix)]
|
||||
if config.pprof {
|
||||
admin_router = admin_router.merge(build_debug_router());
|
||||
}
|
||||
@@ -600,35 +588,31 @@ async fn run(
|
||||
})
|
||||
};
|
||||
|
||||
// Bind the gRPC listener before spawning to propagate bind errors at startup.
|
||||
let grpc_listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let grpc_local_addr = grpc_listener
|
||||
.local_addr()
|
||||
.unwrap_or_else(|e| panic!("Failed to get gRPC local addr: {}", e));
|
||||
|
||||
let grpc_handle = {
|
||||
let grpc_state = state.clone();
|
||||
let grpc_addr = grpc_addr.clone();
|
||||
let grpc_tls_acceptor = grpc_tls_acceptor.clone();
|
||||
let mut shutdown_rx = shutdown_tx.subscribe();
|
||||
let shutdown_tx_grpc = shutdown_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let addr = grpc_addr.parse().expect("Invalid gRPC address");
|
||||
let grpc_service = VolumeGrpcService {
|
||||
state: grpc_state.clone(),
|
||||
};
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
let result = if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let incoming = grpc_tls_incoming(grpc_listener, tls_acceptor);
|
||||
info!("gRPC server listening on {} (TLS enabled)", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let incoming = grpc_tls_incoming(listener, tls_acceptor);
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {} (TLS enabled)", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
@@ -637,25 +621,32 @@ async fn run(
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
} else {
|
||||
let incoming =
|
||||
tokio_stream::wrappers::TcpListenerStream::new(grpc_listener);
|
||||
info!("gRPC server listening on {}", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {}", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
.add_service(build_volume_grpc_service(grpc_service))
|
||||
.serve_with_incoming_shutdown(incoming, async move {
|
||||
.serve_with_shutdown(addr, async move {
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
};
|
||||
if let Err(ref e) = result {
|
||||
error!("gRPC server error: {}", e);
|
||||
let _ = shutdown_tx_grpc.send(());
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
}
|
||||
result
|
||||
})
|
||||
};
|
||||
|
||||
@@ -730,7 +721,6 @@ async fn run(
|
||||
None
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let debug_handle = if config.debug {
|
||||
let debug_addr = format!("0.0.0.0:{}", config.debug_port);
|
||||
info!("Debug pprof server listening on {}", debug_addr);
|
||||
@@ -752,8 +742,6 @@ async fn run(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let debug_handle: Option<tokio::task::JoinHandle<()>> = None;
|
||||
|
||||
let metrics_push_handle = {
|
||||
let push_state = state.clone();
|
||||
@@ -764,40 +752,9 @@ async fn run(
|
||||
}))
|
||||
};
|
||||
|
||||
// Wait for servers. Use select! with &mut so the losing handle is not
|
||||
// dropped, then await it explicitly afterward.
|
||||
let mut server_err: Option<String> = None;
|
||||
let mut http_handle = http_handle;
|
||||
let mut grpc_handle = grpc_handle;
|
||||
let grpc_finished_first = tokio::select! {
|
||||
_ = &mut http_handle => false,
|
||||
_ = &mut grpc_handle => true,
|
||||
};
|
||||
// Inspect the gRPC result (already resolved if it finished first,
|
||||
// otherwise await it now).
|
||||
let grpc_result = if grpc_finished_first {
|
||||
grpc_handle.await
|
||||
} else {
|
||||
// HTTP finished first; gRPC is still running. Await it.
|
||||
grpc_handle.await
|
||||
};
|
||||
match grpc_result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(e)) => {
|
||||
let msg = format!("gRPC server exited with error: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
// serve error already sent shutdown inside the task
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("gRPC task panicked: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
let _ = shutdown_tx.send(());
|
||||
}
|
||||
}
|
||||
// Ensure the HTTP handle completes too.
|
||||
// Wait for all servers
|
||||
let _ = http_handle.await;
|
||||
let _ = grpc_handle.await;
|
||||
if let Some(h) = public_handle {
|
||||
let _ = h.await;
|
||||
}
|
||||
@@ -817,15 +774,10 @@ async fn run(
|
||||
// Close all volumes (flush and release file handles) matching Go's Shutdown()
|
||||
state.store.write().unwrap().close();
|
||||
|
||||
#[cfg(unix)]
|
||||
if let Some(cpu_profile) = cpu_profile {
|
||||
cpu_profile.finish().map_err(std::io::Error::other)?;
|
||||
}
|
||||
|
||||
if let Some(err_msg) = server_err {
|
||||
return Err(std::io::Error::other(err_msg).into());
|
||||
}
|
||||
|
||||
info!("Volume server stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -486,11 +486,6 @@ impl S3TierRegistry {
|
||||
self.backends.keys().cloned().collect()
|
||||
}
|
||||
|
||||
/// Remove a backend by name.
|
||||
pub fn remove(&mut self, name: &str) {
|
||||
self.backends.remove(name);
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.backends.clear();
|
||||
}
|
||||
|
||||
@@ -1867,7 +1867,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
{
|
||||
let needle_header = resp.needle_header;
|
||||
let mut needle_body = resp.needle_body;
|
||||
let resp_version = resp.version;
|
||||
|
||||
if needle_header.is_empty() {
|
||||
continue;
|
||||
@@ -1892,36 +1891,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Parse needle from header + body
|
||||
let mut n = Needle::default();
|
||||
n.read_header(&needle_header);
|
||||
|
||||
if n.size.0 < 0 {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"unexpected negative needle size {} for needle {}",
|
||||
n.size.0, n.id.0
|
||||
)));
|
||||
} else if n.size.0 > 0 {
|
||||
// Normal needle: parse the body fields (DataSize, Data, flags, etc.)
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
} else {
|
||||
// Delete tombstone (size == 0): body is checksum + timestamp
|
||||
// (V3) or checksum only (V2) + padding. Validate minimum
|
||||
// footer length for the protocol version.
|
||||
use crate::storage::types::{
|
||||
NEEDLE_CHECKSUM_SIZE, TIMESTAMP_SIZE, VERSION_3, Version,
|
||||
};
|
||||
let version = Version(resp_version as u8);
|
||||
let min_footer = if version >= VERSION_3 {
|
||||
NEEDLE_CHECKSUM_SIZE + TIMESTAMP_SIZE
|
||||
} else {
|
||||
NEEDLE_CHECKSUM_SIZE
|
||||
};
|
||||
if needle_body.len() < min_footer {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"tombstone needle {} body too short: got {} bytes, need >= {} for version {}",
|
||||
n.id.0, needle_body.len(), min_footer, resp_version
|
||||
)));
|
||||
}
|
||||
}
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
|
||||
// Write needle to local volume
|
||||
let mut store = state.store.write().unwrap();
|
||||
@@ -4076,12 +4047,11 @@ fn find_last_append_at_ns(idx_path: &str, dat_path: &str, version: u32) -> Optio
|
||||
let mut header = [0u8; 16];
|
||||
dat_file.read_exact(&mut header).ok()?;
|
||||
let needle_size = i32::from_be_bytes([header[12], header[13], header[14], header[15]]);
|
||||
if needle_size < 0 {
|
||||
if needle_size <= 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Seek to tail: offset + 16 (header) + size -> checksum (4) + timestamp (8)
|
||||
// For delete needles (size == 0), the tail is right after the header.
|
||||
let tail_offset = actual_offset as u64 + 16 + needle_size as u64;
|
||||
dat_file.seek(SeekFrom::Start(tail_offset)).ok()?;
|
||||
|
||||
@@ -4192,7 +4162,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -4254,7 +4223,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -4264,8 +4232,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -4311,9 +4277,7 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
let tier_config = S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4325,16 +4289,14 @@ mod tests {
|
||||
};
|
||||
{
|
||||
let mut registry = global_s3_tier_registry().write().unwrap();
|
||||
registry.register(
|
||||
"s3.incr_copy_test".to_string(),
|
||||
S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register("s3.default".to_string(), S3TierBackend::new(&tier_config));
|
||||
registry.register("s3".to_string(), S3TierBackend::new(&tier_config));
|
||||
}
|
||||
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
files: vec![crate::storage::volume::VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "incr_copy_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4543,10 +4505,7 @@ mod tests {
|
||||
assert_eq!(copied, dat_bytes[super_block_size as usize..]);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.incr_copy_test");
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1408,8 +1408,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_registers_s3_default_aliases() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently. Just register our entries and verify them.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1441,8 +1443,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_ignores_unsupported_types() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1453,17 +1457,12 @@ mod tests {
|
||||
}],
|
||||
);
|
||||
|
||||
// The per-state registry is freshly created and should have no entries
|
||||
// since "rclone" is unsupported.
|
||||
let registry = state.s3_tier_registry.read().unwrap();
|
||||
assert!(registry.names().is_empty());
|
||||
// Only check that the unsupported type was not added to the global
|
||||
// registry. Other tests may have their own entries present.
|
||||
let global_registry = crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.read()
|
||||
.unwrap();
|
||||
assert!(global_registry.get("rclone.default").is_none());
|
||||
assert!(global_registry.get("rclone").is_none());
|
||||
assert!(global_registry.names().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
#[cfg(unix)]
|
||||
pub mod debug;
|
||||
pub mod grpc_client;
|
||||
pub mod grpc_server;
|
||||
pub mod handlers;
|
||||
pub mod heartbeat;
|
||||
pub mod memory_status;
|
||||
#[cfg(unix)]
|
||||
pub mod profiling;
|
||||
pub mod request_id;
|
||||
pub mod server_stats;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
#[cfg(not(unix))]
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::io::{Seek, SeekFrom};
|
||||
|
||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||
|
||||
|
||||
@@ -1158,13 +1158,7 @@ mod tests {
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// Use a large volume_size_limit so the unused-space difference between
|
||||
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
|
||||
// that integer-division rounding and disk-free fluctuations between the
|
||||
// two get_disk_stats calls cannot make the quotients equal.
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(100 * 1024 * 1024, Ordering::Relaxed);
|
||||
store.volume_size_limit.store(1024, Ordering::Relaxed);
|
||||
store
|
||||
.add_volume(
|
||||
VolumeId(61),
|
||||
|
||||
@@ -3225,7 +3225,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -3280,7 +3279,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -3290,8 +3288,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -3837,7 +3833,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "vif_rw_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: v.dat_file_size().unwrap(),
|
||||
@@ -3853,27 +3849,6 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Register a dummy S3 backend so load_remote_dat_file can look it
|
||||
// up. This test never reads remote data, so a dead endpoint is fine.
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry.
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket-a".to_string(),
|
||||
endpoint: "http://127.0.0.1:1".to_string(),
|
||||
storage_class: "STANDARD".to_string(),
|
||||
force_path_style: true,
|
||||
};
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.register(
|
||||
"s3.vif_rw_test".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
|
||||
v.dat_file_size().unwrap()
|
||||
};
|
||||
|
||||
@@ -3917,11 +3892,6 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(deleted_size.0 > 0);
|
||||
assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
|
||||
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.vif_rw_test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4084,9 +4054,10 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4101,7 +4072,11 @@ mod tests {
|
||||
.write()
|
||||
.unwrap();
|
||||
registry.register(
|
||||
"s3.remote_only_rw".to_string(),
|
||||
"s3.default".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register(
|
||||
"s3".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
}
|
||||
@@ -4109,7 +4084,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "remote_only_rw".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4166,10 +4141,6 @@ mod tests {
|
||||
assert_eq!(meta.data_size, 11);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.remote_only_rw");
|
||||
}
|
||||
|
||||
/// Volume destroy removes .vif alongside the primary data files.
|
||||
|
||||
@@ -33,11 +33,11 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/image v0.36.0 // indirect
|
||||
golang.org/x/net v0.50.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
||||
@@ -27,8 +27,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -84,35 +84,35 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
|
||||
@@ -1,277 +0,0 @@
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same
|
||||
// file concurrently produce valid (non-corrupted) data. With DLM enabled, the
|
||||
// writes are serialized — one blocks until the other completes.
|
||||
//
|
||||
// Note: cross-mount read consistency depends on FUSE kernel cache invalidation
|
||||
// and filer metadata subscription, which are asynchronous. This test verifies
|
||||
// write integrity, not instant read convergence.
|
||||
func TestDLMConcurrentWritersSameFile(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
const iterations = 5
|
||||
for iter := 0; iter < iterations; iter++ {
|
||||
fileName := fmt.Sprintf("concurrent_write_%d.txt", iter)
|
||||
payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter))
|
||||
payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644)
|
||||
assert.NoError(t, err, "mount0 write iteration %d", iter)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644)
|
||||
assert.NoError(t, err, "mount1 write iteration %d", iter)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Verify file is readable and contains one of the expected payloads
|
||||
// (read from mount0 — its own view is authoritative for write success).
|
||||
content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
|
||||
require.NoError(t, err, "read from mount0 iteration %d", iter)
|
||||
validPayload := string(content) == string(payloadA) || string(content) == string(payloadB)
|
||||
assert.True(t, validPayload,
|
||||
"iteration %d: content must be one of the expected payloads, got: %q", iter, content)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles
|
||||
// from both mounts all succeed without errors.
|
||||
func TestDLMRepeatedOpenWriteClose(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
const cycles = 20
|
||||
fileName := "repeated_write.txt"
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < cycles; i++ {
|
||||
data := []byte(fmt.Sprintf("mount0-cycle-%d", i))
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644)
|
||||
assert.NoError(t, err, "mount0 cycle %d", i)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < cycles; i++ {
|
||||
data := []byte(fmt.Sprintf("mount1-cycle-%d", i))
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644)
|
||||
assert.NoError(t, err, "mount1 cycle %d", i)
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// File must be readable from at least one mount
|
||||
content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, content, "file must not be empty")
|
||||
}
|
||||
|
||||
// TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one
|
||||
// mount has a file open for writing, another mount's write-open blocks until
|
||||
// the first mount closes the file.
|
||||
func TestDLMWriteBlocksSecondWriter(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
fileName := "blocking_test.txt"
|
||||
path0 := filepath.Join(cluster.mountPoints[0], fileName)
|
||||
path1 := filepath.Join(cluster.mountPoints[1], fileName)
|
||||
|
||||
// Mount 0 opens the file for writing and holds it open
|
||||
f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
require.NoError(t, err, "mount0 open")
|
||||
_, err = f.Write([]byte("mount0-holds-lock"))
|
||||
require.NoError(t, err, "mount0 write")
|
||||
|
||||
// Mount 1 tries to write — should block (we use a goroutine with atomic flag)
|
||||
var mount1Completed atomic.Bool
|
||||
mount1Done := make(chan error, 1)
|
||||
go func() {
|
||||
err := os.WriteFile(path1, []byte("mount1-waited"), 0644)
|
||||
mount1Completed.Store(true)
|
||||
mount1Done <- err
|
||||
}()
|
||||
|
||||
// Give mount 1 a moment — it should NOT complete while mount 0 holds the file open
|
||||
time.Sleep(3 * time.Second)
|
||||
require.False(t, mount1Completed.Load(),
|
||||
"mount1 write must not complete while mount0 holds the file open")
|
||||
t.Log("mount1 write is blocked as expected while mount0 holds the file")
|
||||
|
||||
// Mount 0 closes the file — this releases the DLM lock
|
||||
require.NoError(t, f.Close(), "mount0 close")
|
||||
|
||||
// Mount 1 should now complete
|
||||
select {
|
||||
case err := <-mount1Done:
|
||||
assert.NoError(t, err, "mount1 write after mount0 close")
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("mount1 write did not complete within 30s after mount0 closed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM:
|
||||
// while mount0 has a file open for writing (re-opened after creation),
|
||||
// mount1 cannot rename it until mount0 closes the file.
|
||||
func TestDLMRenameWhileWriteOpen(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
origName := "rename_source.txt"
|
||||
newName := "rename_dest.txt"
|
||||
|
||||
// Create and close the file first so it's flushed to the filer and
|
||||
// visible on both mounts.
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(cluster.mountPoints[0], origName),
|
||||
[]byte("initial-content"), 0644))
|
||||
time.Sleep(2 * time.Second) // metadata propagation
|
||||
|
||||
// Verify mount1 can see the file
|
||||
_, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName))
|
||||
require.NoError(t, err, "mount1 should see the file")
|
||||
|
||||
// Mount 0 re-opens the file for writing and holds it open
|
||||
f, err := os.OpenFile(
|
||||
filepath.Join(cluster.mountPoints[0], origName),
|
||||
os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
require.NoError(t, err, "mount0 reopen")
|
||||
_, err = f.Write([]byte("data-while-holding-lock"))
|
||||
require.NoError(t, err, "mount0 write")
|
||||
|
||||
// Mount 1 tries to rename — should block because mount0 holds the
|
||||
// DLM lock on the old path
|
||||
var renameCompleted atomic.Bool
|
||||
renameDone := make(chan error, 1)
|
||||
go func() {
|
||||
err := os.Rename(
|
||||
filepath.Join(cluster.mountPoints[1], origName),
|
||||
filepath.Join(cluster.mountPoints[1], newName))
|
||||
renameCompleted.Store(true)
|
||||
renameDone <- err
|
||||
}()
|
||||
|
||||
// Rename must NOT complete while mount0 holds the file open
|
||||
time.Sleep(3 * time.Second)
|
||||
require.False(t, renameCompleted.Load(),
|
||||
"rename must not complete while mount0 holds the file open")
|
||||
t.Log("rename is blocked as expected while mount0 holds the file")
|
||||
|
||||
// Mount 0 closes → releases DLM lock → rename should proceed
|
||||
require.NoError(t, f.Close(), "mount0 close")
|
||||
|
||||
select {
|
||||
case err := <-renameDone:
|
||||
assert.NoError(t, err, "rename after mount0 close")
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("rename did not complete within 30s after mount0 closed")
|
||||
}
|
||||
}
|
||||
|
||||
// Note: Same-mount rename while a file is open for writing is not tested here
|
||||
// because macOS FUSE serializes operations on the same inode, causing a
|
||||
// kernel-level deadlock between the Rename handler's internal flush and the
|
||||
// pending Close. Same-mount coordination is already handled by the per-mount
|
||||
// fhLockTable and FUSE kernel serialization, so DLM is not needed for it.
|
||||
|
||||
// TestDLMConcurrentRenames verifies that two concurrent renames of the same
|
||||
// file from different mounts don't corrupt metadata. DLM locks on both old
|
||||
// and new paths ensure renames are serialized.
|
||||
func TestDLMConcurrentRenames(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
// Create a file first
|
||||
origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt")
|
||||
require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644))
|
||||
time.Sleep(1 * time.Second) // propagation
|
||||
|
||||
// Both mounts try to rename the same file concurrently
|
||||
var wg sync.WaitGroup
|
||||
var errA, errB error
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errA = os.Rename(
|
||||
filepath.Join(cluster.mountPoints[0], "rename_race.txt"),
|
||||
filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"),
|
||||
)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errB = os.Rename(
|
||||
filepath.Join(cluster.mountPoints[1], "rename_race.txt"),
|
||||
filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"),
|
||||
)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// At least one rename should succeed; the other may fail with ENOENT
|
||||
// since the source was already moved.
|
||||
succeeded := 0
|
||||
if errA == nil {
|
||||
succeeded++
|
||||
t.Logf("mount0 rename succeeded")
|
||||
} else {
|
||||
t.Logf("mount0 rename failed: %v", errA)
|
||||
}
|
||||
if errB == nil {
|
||||
succeeded++
|
||||
t.Logf("mount1 rename succeeded")
|
||||
} else {
|
||||
t.Logf("mount1 rename failed: %v", errB)
|
||||
}
|
||||
assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed")
|
||||
}
|
||||
@@ -1,516 +0,0 @@
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
const filerGroup = "fuse-dlm-test"
|
||||
|
||||
// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
|
||||
// mounts for testing DLM-based cross-mount write coordination.
|
||||
type dlmTestCluster struct {
|
||||
t testing.TB
|
||||
baseDir string
|
||||
weedBinary string
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
volumePort int
|
||||
volumeGrpcPort int
|
||||
filerPorts [2]int
|
||||
filerGrpcPorts [2]int
|
||||
mountPoints [2]string
|
||||
|
||||
masterCmd *exec.Cmd
|
||||
volumeCmd *exec.Cmd
|
||||
filerCmds [2]*exec.Cmd
|
||||
mountCmds [2]*exec.Cmd
|
||||
logFiles []*os.File
|
||||
|
||||
cleanupOnce sync.Once
|
||||
}
|
||||
|
||||
func startDLMTestCluster(t testing.TB) *dlmTestCluster {
|
||||
binary := findWeedBinary()
|
||||
if binary == "" {
|
||||
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
|
||||
}
|
||||
|
||||
baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_dlm_test_")
|
||||
require.NoError(t, err)
|
||||
|
||||
c := &dlmTestCluster{
|
||||
t: t,
|
||||
baseDir: baseDir,
|
||||
weedBinary: binary,
|
||||
}
|
||||
// Register cleanup early so processes are stopped even if a require fails below.
|
||||
t.Cleanup(c.Stop)
|
||||
|
||||
// Allocate ports: master(2) + volume(2) + filer0(2) + filer1(2) = 8
|
||||
ports := allocatePorts(t, 8)
|
||||
c.masterPort = ports[0]
|
||||
c.masterGrpcPort = ports[1]
|
||||
c.volumePort = ports[2]
|
||||
c.volumeGrpcPort = ports[3]
|
||||
c.filerPorts[0] = ports[4]
|
||||
c.filerGrpcPorts[0] = ports[5]
|
||||
c.filerPorts[1] = ports[6]
|
||||
c.filerGrpcPorts[1] = ports[7]
|
||||
|
||||
// Write empty security.toml
|
||||
configDir := filepath.Join(baseDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
|
||||
|
||||
// Start master
|
||||
require.NoError(t, c.startMaster(configDir))
|
||||
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second),
|
||||
"master not ready\n%s", c.tailLog("master"))
|
||||
|
||||
// Start volume
|
||||
require.NoError(t, c.startVolume(configDir))
|
||||
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second),
|
||||
"volume not ready\n%s", c.tailLog("volume"))
|
||||
|
||||
// Start 2 filers
|
||||
for i := 0; i < 2; i++ {
|
||||
require.NoError(t, c.startFiler(i, configDir))
|
||||
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[i]), 30*time.Second),
|
||||
"filer %d not ready\n%s", i, c.tailLog(fmt.Sprintf("filer%d", i)))
|
||||
}
|
||||
require.NoError(t, c.waitForFilerCount(2, 30*time.Second), "filer group registration")
|
||||
require.NoError(t, c.waitForLockRingConverged(30*time.Second), "lock ring convergence")
|
||||
|
||||
// Start 2 mounts, both pointing at filer0 for metadata consistency.
|
||||
// (filer1 exists for the DLM lock ring but both mounts share filer0's
|
||||
// metadata store since leveldb is per-filer.)
|
||||
for i := 0; i < 2; i++ {
|
||||
mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
|
||||
require.NoError(t, os.MkdirAll(mp, 0755))
|
||||
c.mountPoints[i] = mp
|
||||
require.NoError(t, c.startMount(i, configDir))
|
||||
require.NoError(t, c.waitForMount(mp, 30*time.Second),
|
||||
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) Stop() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.cleanupOnce.Do(func() {
|
||||
// Stop mounts first (triggers flush + DLM unlock).
|
||||
// Use stopCmd for bounded wait to avoid hanging on wedged FUSE processes.
|
||||
for i := 1; i >= 0; i-- {
|
||||
stopCmd(c.mountCmds[i])
|
||||
// Backup unmount in case FUSE teardown didn't clean up
|
||||
exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
|
||||
exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
|
||||
}
|
||||
// Stop filers, volume, master
|
||||
for i := 1; i >= 0; i-- {
|
||||
stopCmd(c.filerCmds[i])
|
||||
}
|
||||
stopCmd(c.volumeCmd)
|
||||
stopCmd(c.masterCmd)
|
||||
|
||||
for _, f := range c.logFiles {
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Copy logs for CI
|
||||
c.copyLogsForCI()
|
||||
|
||||
if !c.t.Failed() {
|
||||
os.RemoveAll(c.baseDir)
|
||||
}
|
||||
|
||||
// Wait for ports to be fully released before the next test
|
||||
// allocates new ports (avoids TIME_WAIT collisions).
|
||||
time.Sleep(2 * time.Second)
|
||||
})
|
||||
}
|
||||
|
||||
// masterAddress returns the master address in the format that encodes both
|
||||
// HTTP and gRPC ports: "host:httpPort.grpcPort". This is the format that
|
||||
// SeaweedFS uses to communicate non-default gRPC ports between components.
|
||||
func (c *dlmTestCluster) masterAddress() string {
|
||||
return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startMaster(configDir string) error {
|
||||
c.masterCmd = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"master",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.masterPort),
|
||||
"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
|
||||
"-mdir="+filepath.Join(c.baseDir, "master"),
|
||||
)
|
||||
return c.startCmd(c.masterCmd, "master")
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startVolume(configDir string) error {
|
||||
volDir := filepath.Join(c.baseDir, "volume")
|
||||
if err := os.MkdirAll(volDir, 0755); err != nil {
|
||||
return fmt.Errorf("create volume dir: %w", err)
|
||||
}
|
||||
c.volumeCmd = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"volume",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.volumePort),
|
||||
"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
|
||||
"-master="+c.masterAddress(),
|
||||
"-dir="+volDir,
|
||||
"-max=10",
|
||||
)
|
||||
return c.startCmd(c.volumeCmd, "volume")
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startFiler(idx int, configDir string) error {
|
||||
filerDir := filepath.Join(c.baseDir, fmt.Sprintf("filer%d", idx))
|
||||
if err := os.MkdirAll(filerDir, 0755); err != nil {
|
||||
return fmt.Errorf("create filer dir: %w", err)
|
||||
}
|
||||
c.filerCmds[idx] = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"filer",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.filerPorts[idx]),
|
||||
"-port.grpc="+strconv.Itoa(c.filerGrpcPorts[idx]),
|
||||
"-master="+c.masterAddress(),
|
||||
"-filerGroup="+filerGroup,
|
||||
"-defaultStoreDir="+filerDir,
|
||||
)
|
||||
return c.startCmd(c.filerCmds[idx], fmt.Sprintf("filer%d", idx))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) filerAddress(idx int) string {
|
||||
return string(pb.NewServerAddress("127.0.0.1", c.filerPorts[idx], c.filerGrpcPorts[idx]))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startMount(idx int, configDir string) error {
|
||||
cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("create cache dir: %w", err)
|
||||
}
|
||||
c.mountCmds[idx] = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"mount",
|
||||
"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
|
||||
"-dir="+c.mountPoints[idx],
|
||||
"-filer.path=/",
|
||||
"-dirAutoCreate",
|
||||
"-allowOthers=false",
|
||||
"-cacheDir="+cacheDir,
|
||||
"-dlm",
|
||||
)
|
||||
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startCmd(cmd *exec.Cmd, name string) error {
|
||||
logPath := filepath.Join(c.baseDir, "logs")
|
||||
if err := os.MkdirAll(logPath, 0755); err != nil {
|
||||
return fmt.Errorf("create log dir: %w", err)
|
||||
}
|
||||
logFile, err := os.Create(filepath.Join(logPath, name+".log"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.logFiles = append(c.logFiles, logFile)
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
return cmd.Start()
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) tailLog(name string) string {
|
||||
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
|
||||
if err != nil {
|
||||
return fmt.Sprintf("(log %s not available: %v)", name, err)
|
||||
}
|
||||
const maxTail = 8192
|
||||
if len(data) > maxTail {
|
||||
data = data[len(data)-maxTail:]
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) copyLogsForCI() {
|
||||
ciLogDir := "/tmp/seaweedfs-fuse-dlm-logs"
|
||||
os.MkdirAll(ciLogDir, 0755)
|
||||
logsDir := filepath.Join(c.baseDir, "logs")
|
||||
entries, err := os.ReadDir(logsDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) waitForTCP(addr string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", addr, time.Second)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("service at %s not ready within timeout", addr)
|
||||
}
|
||||
|
||||
// waitForMount waits for a FUSE filesystem to actually be mounted at
|
||||
// mountPoint by comparing the device ID of the mount point against its parent.
|
||||
// A plain directory (pre-created before mount) has the same device as its
|
||||
// parent; a mounted FUSE filesystem has a different device.
|
||||
func (c *dlmTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
|
||||
parentDir := filepath.Dir(mountPoint)
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
parentStat, err := os.Stat(parentDir)
|
||||
if err != nil {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
mountStat, err := os.Stat(mountPoint)
|
||||
if err != nil {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
parentSys := parentStat.Sys().(*syscall.Stat_t)
|
||||
mountSys := mountStat.Sys().(*syscall.Stat_t)
|
||||
if parentSys.Dev != mountSys.Dev {
|
||||
// Different device = FUSE mounted
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) filerGRPCAddress(idx int) string {
|
||||
return fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[idx])
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) waitForFilerCount(expected int, timeout time.Duration) error {
|
||||
addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
|
||||
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := master_pb.NewSeaweedClient(conn)
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
|
||||
ClientType: "filer",
|
||||
FilerGroup: filerGroup,
|
||||
})
|
||||
cancel()
|
||||
if err == nil && len(resp.ClusterNodes) >= expected {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for %d filers in group %q", expected, filerGroup)
|
||||
}
|
||||
|
||||
// waitForLockRingConverged verifies that both filers have a consistent view of
|
||||
// the lock ring by acquiring the same lock through each filer and checking
|
||||
// mutual exclusion. Adapted from test/s3/distributed_lock/.
|
||||
func (c *dlmTestCluster) waitForLockRingConverged(timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
owners := []pb.ServerAddress{
|
||||
pb.ServerAddress(c.filerGRPCAddress(0)),
|
||||
pb.ServerAddress(c.filerGRPCAddress(1)),
|
||||
}
|
||||
ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
|
||||
ring.SetServers(owners)
|
||||
|
||||
attempt := 0
|
||||
for time.Now().Before(deadline) {
|
||||
testKeys := convergenceKeysPerPrimary(ring, owners, attempt)
|
||||
attempt++
|
||||
|
||||
allConverged := true
|
||||
for _, key := range testKeys {
|
||||
converged, _ := c.checkLockMutualExclusion(key)
|
||||
if !converged {
|
||||
allConverged = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allConverged {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("lock ring did not converge")
|
||||
}
|
||||
|
||||
// convergenceKeysPerPrimary generates one test key per primary filer.
|
||||
func convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
|
||||
found := make(map[pb.ServerAddress]bool)
|
||||
var keys []string
|
||||
for i := 0; len(found) < len(owners) && i < 10000; i++ {
|
||||
key := fmt.Sprintf("convergence-test-%d-%d", attempt, i)
|
||||
primary, _ := ring.GetPrimaryAndBackup(key)
|
||||
if !found[primary] {
|
||||
found[primary] = true
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) checkLockMutualExclusion(key string) (bool, error) {
|
||||
// Try to lock via filer0
|
||||
conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer conn0.Close()
|
||||
|
||||
client0 := filer_pb.NewSeaweedFilerClient(conn0)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
resp0, err := client0.DistributedLock(ctx, &filer_pb.LockRequest{
|
||||
Name: key,
|
||||
SecondsToLock: 5,
|
||||
Owner: "convergence-test-0",
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if resp0.Error != "" {
|
||||
return false, fmt.Errorf("lock0: %s", resp0.Error)
|
||||
}
|
||||
|
||||
// Try to lock via filer1 — should fail (already locked)
|
||||
conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer conn1.Close()
|
||||
|
||||
client1 := filer_pb.NewSeaweedFilerClient(conn1)
|
||||
resp1, err := client1.DistributedLock(ctx, &filer_pb.LockRequest{
|
||||
Name: key,
|
||||
SecondsToLock: 5,
|
||||
Owner: "convergence-test-1",
|
||||
})
|
||||
|
||||
// Unlock via filer0
|
||||
client0.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
|
||||
Name: key,
|
||||
RenewToken: resp0.RenewToken,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// If filer1 also got the lock, the ring hasn't converged
|
||||
if resp1.Error == "" {
|
||||
// Unlock the second one too
|
||||
client1.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
|
||||
Name: key,
|
||||
RenewToken: resp1.RenewToken,
|
||||
})
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func stopCmd(cmd *exec.Cmd) {
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
cmd.Process.Signal(syscall.SIGTERM)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
cmd.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
cmd.Process.Kill()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func allocatePorts(t testing.TB, n int) []int {
|
||||
t.Helper()
|
||||
// Hold all listeners open until all ports are collected, then close
|
||||
// them together. This prevents the OS from reassigning a just-freed
|
||||
// port to the next Listen call within the same allocation batch.
|
||||
listeners := make([]net.Listener, 0, n)
|
||||
ports := make([]int, 0, n)
|
||||
defer func() {
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
for i := 0; i < n; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
listeners = append(listeners, l)
|
||||
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
return ports
|
||||
}
|
||||
|
||||
func findWeedBinary() string {
|
||||
if p := os.Getenv("WEED_BINARY"); p != "" {
|
||||
return p
|
||||
}
|
||||
if p, err := exec.LookPath("weed"); err == nil {
|
||||
return p
|
||||
}
|
||||
candidates := []string{
|
||||
"../../weed/weed",
|
||||
"./weed",
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if info, err := os.Stat(c); err == nil && !info.IsDir() {
|
||||
abs, _ := filepath.Abs(c)
|
||||
return abs
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -249,18 +249,11 @@ func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramewor
|
||||
return
|
||||
}
|
||||
|
||||
// Create file in subdirectory with retry for transient FUSE errors
|
||||
// Create file in subdirectory
|
||||
testFile := filepath.Join(subDir, "test.txt")
|
||||
content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
|
||||
var writeErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if writeErr != nil {
|
||||
addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
|
||||
if err := os.WriteFile(testFile, content, 0644); err != nil {
|
||||
addError(fmt.Errorf("worker %d file %d: %v", workerID, i, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,13 +82,12 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
|
||||
}
|
||||
}
|
||||
|
||||
// freePort asks the OS for a free TCP port in a range where the gRPC
|
||||
// offset (port + 10000) won't collide with well-known ports.
|
||||
// freePort asks the OS for a free TCP port.
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
const (
|
||||
minServicePort = 20000
|
||||
maxServicePort = 55535
|
||||
maxServicePort = 55535 // SeaweedFS gRPC service uses httpPort + 10000.
|
||||
)
|
||||
|
||||
portCount := maxServicePort - minServicePort + 1
|
||||
|
||||
@@ -85,19 +85,9 @@ func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
|
||||
branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
gitRun(t, localClone, "push", "origin", branch)
|
||||
|
||||
// The bare repo lives on the FUSE mount and can briefly disappear after
|
||||
// the push completes, and pushed pack objects may not be immediately
|
||||
// consistent on the FUSE layer. Give the mount a chance to settle, then
|
||||
// recover from the local clone if the remote is still missing.
|
||||
if !waitForBareRepoEventually(t, bareRepo, 10*time.Second) {
|
||||
t.Logf("bare repo %s did not stabilise after push; forcing recovery before clone", bareRepo)
|
||||
}
|
||||
refreshDirEntry(t, bareRepo)
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
|
||||
t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
|
||||
ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
|
||||
gitRun(t, "", "clone", bareRepo, mountClone)
|
||||
|
||||
assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
|
||||
assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
|
||||
@@ -300,7 +290,7 @@ func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Durat
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if isBareRepoAccessible(bareRepo) {
|
||||
if isBareRepo(bareRepo) {
|
||||
return true
|
||||
}
|
||||
refreshDirEntry(t, bareRepo)
|
||||
@@ -322,14 +312,6 @@ func isBareRepo(bareRepo string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func isBareRepoAccessible(bareRepo string) bool {
|
||||
if !isBareRepo(bareRepo) {
|
||||
return false
|
||||
}
|
||||
out, err := tryGitCommand("", "--git-dir="+bareRepo, "rev-parse", "--is-bare-repository")
|
||||
return err == nil && out == "true"
|
||||
}
|
||||
|
||||
func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
|
||||
t.Helper()
|
||||
require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
|
||||
@@ -338,7 +320,7 @@ func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
|
||||
// tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
|
||||
// If it has vanished, it re-creates it from the local clone.
|
||||
func tryEnsureBareRepo(bareRepo, localClone string) error {
|
||||
if isBareRepoAccessible(bareRepo) {
|
||||
if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
|
||||
return nil
|
||||
}
|
||||
branch, err := tryGitCommand(localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
@@ -360,36 +342,6 @@ func tryEnsureBareRepo(bareRepo, localClone string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMountCloneFromBareWithRecovery(t *testing.T, bareRepo, localClone, mountClone string) {
|
||||
t.Helper()
|
||||
const maxAttempts = 3
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if lastErr = tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone); lastErr == nil {
|
||||
return
|
||||
}
|
||||
if attempt == maxAttempts {
|
||||
require.NoError(t, lastErr, "git clone %s %s failed after %d recovery attempts", bareRepo, mountClone, maxAttempts)
|
||||
}
|
||||
t.Logf("clone recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone string) error {
|
||||
if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
|
||||
return fmt.Errorf("ensure bare repo: %w", err)
|
||||
}
|
||||
if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
|
||||
return fmt.Errorf("ensure mount clone: %w", err)
|
||||
}
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
return fmt.Errorf("verify mount clone: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryEnsureMountClone is like ensureMountClone but returns an error instead
|
||||
// of failing the test, for use in recovery loops.
|
||||
func tryEnsureMountClone(bareRepo, mountClone string) error {
|
||||
@@ -479,31 +431,6 @@ func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, c
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Verify git commands actually work in the directory — the kernel
|
||||
// dcache can transiently show the dir then lose it after reset.
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
lastErr = fmt.Errorf("post-reset verification failed: %w", err)
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The kernel dcache can drop the FUSE entry moments after a
|
||||
// successful check. Wait briefly and re-verify to confirm the
|
||||
// directory has stabilised before returning to the caller.
|
||||
time.Sleep(1 * time.Second)
|
||||
refreshDirEntry(t, mountClone)
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
|
||||
@@ -531,10 +458,6 @@ func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit
|
||||
}
|
||||
refreshDirEntry(t, cloneDir)
|
||||
|
||||
// Let the dcache stabilise before proceeding.
|
||||
time.Sleep(1 * time.Second)
|
||||
refreshDirEntry(t, cloneDir)
|
||||
|
||||
head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("rev-parse after reset: %w", err)
|
||||
@@ -598,33 +521,3 @@ func TestTryEnsureBareRepoPreservesCurrentBranch(t *testing.T) {
|
||||
assert.Equal(t, branch, restoredHead, "clone from recovered bare repo should check out the current branch")
|
||||
assertFileContains(t, filepath.Join(restoredClone, "README.md"), "hello recovery")
|
||||
}
|
||||
|
||||
func TestEnsureMountCloneFromBareWithRecoveryRecreatesMissingBareRepo(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "git_mount_clone_recovery_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
bareRepo := filepath.Join(tempDir, "repo.git")
|
||||
localClone := filepath.Join(tempDir, "clone")
|
||||
mountClone := filepath.Join(tempDir, "mount-clone")
|
||||
|
||||
gitRun(t, "", "init", "--bare", bareRepo)
|
||||
gitRun(t, "", "clone", bareRepo, localClone)
|
||||
gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
|
||||
gitRun(t, localClone, "config", "user.name", "Test")
|
||||
|
||||
writeFile(t, localClone, "README.md", "hello clone recovery\n")
|
||||
gitRun(t, localClone, "add", "README.md")
|
||||
gitRun(t, localClone, "commit", "-m", "initial commit")
|
||||
|
||||
branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
gitRun(t, localClone, "push", "origin", branch)
|
||||
|
||||
require.NoError(t, os.RemoveAll(bareRepo))
|
||||
|
||||
ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
|
||||
|
||||
head := gitOutput(t, mountClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
assert.Equal(t, branch, head, "recovered clone should stay on the pushed branch")
|
||||
assertFileContains(t, filepath.Join(mountClone, "README.md"), "hello clone recovery")
|
||||
}
|
||||
|
||||
+39
-37
@@ -14,7 +14,7 @@ require (
|
||||
replace github.com/seaweedfs/seaweedfs => ../../
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
|
||||
@@ -43,25 +43,25 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/aws/aws-sdk-go v1.55.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -104,7 +104,7 @@ require (
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
@@ -124,8 +124,8 @@ require (
|
||||
github.com/google/btree v1.1.3 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -147,9 +147,9 @@ require (
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/klauspost/compress v1.18.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.13.3 // indirect
|
||||
github.com/klauspost/reedsolomon v1.13.0 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
@@ -197,6 +197,7 @@ require (
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
|
||||
github.com/rdleal/intervalst v1.5.0 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
@@ -236,34 +237,35 @@ require (
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.9 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.49.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
|
||||
golang.org/x/image v0.36.0 // indirect
|
||||
golang.org/x/net v0.49.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/api v0.274.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/api v0.258.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
|
||||
+92
-88
@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
|
||||
cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
@@ -112,44 +112,44 @@ github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
@@ -284,8 +284,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -385,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
|
||||
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
|
||||
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
@@ -467,12 +467,12 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
|
||||
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
|
||||
github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/klauspost/reedsolomon v1.13.0 h1:E0Cmgf2kMuhZTj6eefnvpKC4/Q4jhCi9YIjcZjK4arc=
|
||||
github.com/klauspost/reedsolomon v1.13.0/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
|
||||
@@ -600,6 +600,8 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
|
||||
github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
@@ -705,8 +707,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@@ -737,8 +739,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
|
||||
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
@@ -752,16 +754,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
@@ -785,8 +787,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -797,12 +799,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -867,8 +869,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -892,8 +894,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -957,8 +959,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -974,13 +976,13 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
@@ -1028,14 +1030,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
@@ -1052,8 +1054,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
|
||||
google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -1089,12 +1091,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -1152,6 +1154,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs=
|
||||
moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -16,8 +17,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -74,23 +73,23 @@ func StartMasterCluster(t testing.TB) *MasterCluster {
|
||||
logsDir := filepath.Join(baseDir, "logs")
|
||||
os.MkdirAll(logsDir, 0o755)
|
||||
|
||||
// Allocate 3 mini-safe ports (each guarantees port+10000 is also free).
|
||||
httpPorts, err := testutil.AllocateMiniPorts(3)
|
||||
// Allocate 3 port pairs (http, grpc) atomically to prevent reuse.
|
||||
portPairs, err := allocateMultipleMasterPortPairs(3)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate ports: %v", err)
|
||||
}
|
||||
var nodes [3]*masterNode
|
||||
var peerParts []string
|
||||
for i, hp := range httpPorts {
|
||||
for i, pp := range portPairs {
|
||||
dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i))
|
||||
os.MkdirAll(dataDir, 0o755)
|
||||
nodes[i] = &masterNode{
|
||||
port: hp,
|
||||
grpcPort: hp + testutil.GrpcPortOffset,
|
||||
port: pp[0],
|
||||
grpcPort: pp[1],
|
||||
dataDir: dataDir,
|
||||
logFile: filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)),
|
||||
}
|
||||
peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp))
|
||||
peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", pp[0]))
|
||||
}
|
||||
|
||||
mc := &MasterCluster{
|
||||
@@ -358,6 +357,41 @@ func (mc *MasterCluster) tailLog(i int) string {
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// --- port and binary helpers (adapted from test/volume_server/framework) ---
|
||||
|
||||
// allocateMultipleMasterPortPairs finds n non-overlapping (http, grpc) port
|
||||
// pairs, holding all listeners until all are found, then releasing them
|
||||
// together to avoid races between consecutive allocations.
|
||||
func allocateMultipleMasterPortPairs(n int) ([][2]int, error) {
|
||||
var listeners []net.Listener
|
||||
var pairs [][2]int
|
||||
|
||||
defer func() {
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
for masterPort := 10000; masterPort <= 55535 && len(pairs) < n; masterPort++ {
|
||||
grpcPort := masterPort + 10000
|
||||
l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(grpcPort)))
|
||||
if err != nil {
|
||||
l1.Close()
|
||||
continue
|
||||
}
|
||||
listeners = append(listeners, l1, l2)
|
||||
pairs = append(pairs, [2]int{masterPort, grpcPort})
|
||||
}
|
||||
|
||||
if len(pairs) < n {
|
||||
return nil, fmt.Errorf("could only allocate %d of %d master port pairs", len(pairs), n)
|
||||
}
|
||||
return pairs, nil
|
||||
}
|
||||
|
||||
func findOrBuildWeedBinary() (string, error) {
|
||||
if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
FROM chrislusf/seaweedfs:e2e
|
||||
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
autoconf \
|
||||
automake \
|
||||
build-essential \
|
||||
git \
|
||||
libtap-harness-archive-perl \
|
||||
perl \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ARG PJDFSTEST_REPO=https://github.com/pjd/pjdfstest.git
|
||||
ARG PJDFSTEST_REF=03eb25706d8dbf3611c3f820b45b7a5e09a36c06
|
||||
|
||||
RUN git clone "${PJDFSTEST_REPO}" /opt/pjdfstest && \
|
||||
cd /opt/pjdfstest && \
|
||||
git checkout "${PJDFSTEST_REF}" && \
|
||||
autoreconf -ifs && \
|
||||
./configure && \
|
||||
make pjdfstest
|
||||
|
||||
COPY run_inside_container.sh /run.sh
|
||||
COPY known_failures.txt /known_failures.txt
|
||||
RUN chmod +x /run.sh
|
||||
@@ -1,57 +0,0 @@
|
||||
services:
|
||||
master:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
master:
|
||||
condition: service_healthy
|
||||
|
||||
filer:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "-I", "http://localhost:8888/healthz"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
volume:
|
||||
condition: service_healthy
|
||||
|
||||
mount:
|
||||
image: chrislusf/seaweedfs:pjdfstest
|
||||
build:
|
||||
context: .
|
||||
command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs -allowOthers"
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- /dev/fuse
|
||||
security_opt:
|
||||
- apparmor:unconfined
|
||||
healthcheck:
|
||||
test: ["CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 10s
|
||||
depends_on:
|
||||
filer:
|
||||
condition: service_healthy
|
||||
@@ -1,60 +0,0 @@
|
||||
# Known pjdfstest failures for SeaweedFS FUSE mount.
|
||||
#
|
||||
# Tests listed here are skipped during CI runs. Each entry must be a path
|
||||
# relative to the pjdfstest root (e.g. tests/chmod/02.t).
|
||||
#
|
||||
# A failure in any test NOT listed here will cause the CI job to fail,
|
||||
# catching regressions immediately.
|
||||
|
||||
# ── Linux FUSE NAME_MAX=255 limitation ──────────────────────────────────
|
||||
# The Linux FUSE kernel module enforces NAME_MAX=255 at the VFS layer.
|
||||
# These tests create filenames >255 bytes which cannot be looked up via
|
||||
# normal syscalls (stat, chmod, etc.) after creation.
|
||||
tests/chmod/02.t
|
||||
tests/chmod/03.t
|
||||
tests/chown/02.t
|
||||
tests/chown/03.t
|
||||
tests/ftruncate/02.t
|
||||
tests/ftruncate/03.t
|
||||
tests/link/02.t
|
||||
tests/link/03.t
|
||||
tests/mkdir/02.t
|
||||
tests/mkdir/03.t
|
||||
tests/mkfifo/02.t
|
||||
tests/mkfifo/03.t
|
||||
tests/mknod/02.t
|
||||
tests/mknod/03.t
|
||||
tests/open/02.t
|
||||
tests/open/03.t
|
||||
tests/rename/01.t
|
||||
tests/rename/02.t
|
||||
tests/rmdir/02.t
|
||||
tests/rmdir/03.t
|
||||
tests/symlink/02.t
|
||||
tests/symlink/03.t
|
||||
tests/truncate/02.t
|
||||
tests/truncate/03.t
|
||||
tests/unlink/02.t
|
||||
tests/unlink/03.t
|
||||
|
||||
# ── Hard link nlink/ctime tracking (requires filer changes) ────────────
|
||||
# nlink counts are not correctly maintained across rename operations.
|
||||
tests/rename/23.t
|
||||
tests/rename/24.t
|
||||
|
||||
# ── Parent directory mtime/ctime on deferred file create ───────────────
|
||||
# When file creation is deferred (not flushed to filer immediately),
|
||||
# the parent directory mtime/ctime cannot be updated without invalidating
|
||||
# the just-cached child entry in the metadata cache.
|
||||
tests/open/00.t
|
||||
|
||||
# ── Directory rename permission edge case ──────────────────────────────
|
||||
# Cross-directory rename of a subdirectory with restricted permissions
|
||||
# causes cascading test failures within the test file.
|
||||
tests/rename/21.t
|
||||
|
||||
# ── rmdir after hard link unlink ───────────────────────────────────────
|
||||
# The filer may still report a directory as non-empty after all hard-linked
|
||||
# entries have been unlinked.
|
||||
tests/unlink/14.t
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Run the pjdfstest POSIX compliance suite against a SeaweedFS FUSE mount.
|
||||
#
|
||||
# This script:
|
||||
# 1. Starts a self-contained "weed mini" server (master+volume+filer in one)
|
||||
# 2. Mounts the filesystem with "weed mount"
|
||||
# 3. Builds pjdfstest from upstream and runs it under prove(1)
|
||||
#
|
||||
# Requirements: weed in $PATH, fusermount3, perl with TAP::Harness::Archive,
|
||||
# autoconf, make, sudo (pjdfstest exercises chown/chmod which need root).
|
||||
#
|
||||
# Usage:
|
||||
# test/pjdfstest/run.sh # runs full suite
|
||||
# PJDFSTEST_TESTS=tests/chmod ./run.sh # runs a subset
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
WEED_BIN="${WEED_BIN:-weed}"
|
||||
WORK_DIR="${WORK_DIR:-$(mktemp -d /tmp/seaweedfs-pjdfstest.XXXXXX)}"
|
||||
MOUNT_DIR="${MOUNT_DIR:-${WORK_DIR}/mnt}"
|
||||
DATA_DIR="${DATA_DIR:-${WORK_DIR}/data}"
|
||||
LOG_DIR="${LOG_DIR:-${WORK_DIR}/logs}"
|
||||
FILER_PORT="${FILER_PORT:-28888}"
|
||||
FILER_ADDR="127.0.0.1:${FILER_PORT}"
|
||||
|
||||
# Pin to an immutable upstream commit so CI is reproducible. Override via env
|
||||
# if you want to test against a different ref or fork.
|
||||
PJDFSTEST_REPO="${PJDFSTEST_REPO:-https://github.com/pjd/pjdfstest.git}"
|
||||
PJDFSTEST_REF="${PJDFSTEST_REF:-03eb25706d8dbf3611c3f820b45b7a5e09a36c06}"
|
||||
PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
|
||||
|
||||
mini_pid=""
|
||||
mount_pid=""
|
||||
|
||||
CI_LOG_DIR="/tmp/seaweedfs-pjdfstest-logs"
|
||||
|
||||
cleanup() {
|
||||
set +e
|
||||
if [[ -n "${mount_pid}" ]] && kill -0 "${mount_pid}" 2>/dev/null; then
|
||||
kill -TERM "${mount_pid}" 2>/dev/null || true
|
||||
wait "${mount_pid}" 2>/dev/null || true
|
||||
fi
|
||||
if mountpoint -q "${MOUNT_DIR}" 2>/dev/null; then
|
||||
fusermount3 -u "${MOUNT_DIR}" 2>/dev/null || \
|
||||
fusermount -u "${MOUNT_DIR}" 2>/dev/null || \
|
||||
sudo umount "${MOUNT_DIR}" 2>/dev/null || true
|
||||
fi
|
||||
if [[ -n "${mini_pid}" ]] && kill -0 "${mini_pid}" 2>/dev/null; then
|
||||
kill -TERM "${mini_pid}" 2>/dev/null || true
|
||||
wait "${mini_pid}" 2>/dev/null || true
|
||||
fi
|
||||
# Copy logs to a fixed path for CI artifact upload.
|
||||
mkdir -p "${CI_LOG_DIR}"
|
||||
cp "${LOG_DIR}"/*.log "${CI_LOG_DIR}/" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
mkdir -p "${MOUNT_DIR}" "${DATA_DIR}" "${LOG_DIR}"
|
||||
|
||||
echo "==> Starting weed mini on ${FILER_ADDR}"
|
||||
"${WEED_BIN}" mini \
|
||||
-dir="${DATA_DIR}" \
|
||||
-ip=127.0.0.1 \
|
||||
-filer.port="${FILER_PORT}" \
|
||||
-s3=false \
|
||||
-webdav=false \
|
||||
-admin.ui=false \
|
||||
>"${LOG_DIR}/mini.log" 2>&1 &
|
||||
mini_pid=$!
|
||||
|
||||
# Wait for filer to accept connections.
|
||||
for i in $(seq 1 60); do
|
||||
if (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "${mini_pid}" 2>/dev/null; then
|
||||
echo "weed mini exited early; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if ! (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
|
||||
echo "weed mini filer did not become reachable within 30s; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Mounting SeaweedFS at ${MOUNT_DIR}"
|
||||
# allowOthers is required so that pjdfstest's setuid/setgid sub-tests (run via
|
||||
# sudo) can access files created as the invoking user.
|
||||
sudo "${WEED_BIN}" mount \
|
||||
-filer="${FILER_ADDR}" \
|
||||
-dir="${MOUNT_DIR}" \
|
||||
-filer.path=/ \
|
||||
-dirAutoCreate \
|
||||
-allowOthers=true \
|
||||
>"${LOG_DIR}/mount.log" 2>&1 &
|
||||
mount_pid=$!
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if mountpoint -q "${MOUNT_DIR}"; then
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "${mount_pid}" 2>/dev/null; then
|
||||
echo "weed mount exited early; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if ! mountpoint -q "${MOUNT_DIR}"; then
|
||||
echo "FUSE mount did not come up within 30s" >&2
|
||||
tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Cloning and building pjdfstest"
|
||||
PJDFSTEST_DIR="${WORK_DIR}/pjdfstest"
|
||||
if [[ ! -d "${PJDFSTEST_DIR}/.git" ]]; then
|
||||
git clone "${PJDFSTEST_REPO}" "${PJDFSTEST_DIR}"
|
||||
fi
|
||||
git -C "${PJDFSTEST_DIR}" remote set-url origin "${PJDFSTEST_REPO}"
|
||||
git -C "${PJDFSTEST_DIR}" fetch --depth 1 origin "${PJDFSTEST_REF}"
|
||||
git -C "${PJDFSTEST_DIR}" checkout --detach FETCH_HEAD
|
||||
(
|
||||
cd "${PJDFSTEST_DIR}"
|
||||
autoreconf -ifs
|
||||
./configure
|
||||
make pjdfstest
|
||||
)
|
||||
|
||||
# pjdfstest must run inside the filesystem under test.
|
||||
TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
|
||||
sudo mkdir -p "${TEST_ROOT}"
|
||||
sudo cp -r "${PJDFSTEST_DIR}/." "${TEST_ROOT}/"
|
||||
|
||||
cd "${TEST_ROOT}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
KNOWN_FAILURES="${SCRIPT_DIR}/known_failures.txt"
|
||||
|
||||
# Build the list of tests to run, excluding known failures.
|
||||
if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
|
||||
mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
|
||||
all_tests=()
|
||||
while IFS= read -r -d '' t; do
|
||||
all_tests+=("$t")
|
||||
done < <(find tests/ -name '*.t' -print0 | sort -z)
|
||||
|
||||
run_tests=()
|
||||
for t in "${all_tests[@]}"; do
|
||||
is_skipped=false
|
||||
for s in "${skip[@]}"; do
|
||||
if [[ "$t" == "$s" ]]; then
|
||||
is_skipped=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ! $is_skipped; then
|
||||
run_tests+=("$t")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
|
||||
sudo prove -rv "${run_tests[@]}"
|
||||
else
|
||||
echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
|
||||
sudo prove -rv "${PJDFSTEST_TESTS}"
|
||||
fi
|
||||
@@ -1,48 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Runs pjdfstest inside the mount container against /mnt/seaweedfs.
|
||||
# Invoked via: docker compose exec mount /run.sh
|
||||
#
|
||||
# Uses known_failures.txt to skip tests with known issues. Any failure
|
||||
# in a test NOT in the skip list causes a non-zero exit (regression).
|
||||
set -euo pipefail
|
||||
|
||||
MOUNT_DIR="/mnt/seaweedfs"
|
||||
PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
|
||||
KNOWN_FAILURES="/known_failures.txt"
|
||||
|
||||
# Copy pjdfstest into the mounted filesystem so tests exercise the FS under test.
|
||||
TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
|
||||
mkdir -p "${TEST_ROOT}"
|
||||
cp -r /opt/pjdfstest/. "${TEST_ROOT}/"
|
||||
|
||||
cd "${TEST_ROOT}"
|
||||
|
||||
# Build the list of tests to run, excluding known failures.
|
||||
if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
|
||||
mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
|
||||
all_tests=()
|
||||
while IFS= read -r -d '' t; do
|
||||
all_tests+=("$t")
|
||||
done < <(find tests/ -name '*.t' -print0 | sort -z)
|
||||
|
||||
run_tests=()
|
||||
for t in "${all_tests[@]}"; do
|
||||
is_skipped=false
|
||||
for s in "${skip[@]}"; do
|
||||
if [[ "$t" == "$s" ]]; then
|
||||
is_skipped=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ! $is_skipped; then
|
||||
run_tests+=("$t")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
|
||||
prove -rv "${run_tests[@]}"
|
||||
else
|
||||
echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
|
||||
prove -rv "${PJDFSTEST_TESTS}"
|
||||
fi
|
||||
@@ -11,11 +11,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// VolumeServer provides a minimal volume server for erasure coding tests.
|
||||
@@ -198,25 +196,12 @@ func (v *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream vo
|
||||
defer file.Close()
|
||||
|
||||
buf := make([]byte, 64*1024)
|
||||
remaining := int64(req.GetStopOffset())
|
||||
for {
|
||||
if remaining == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
readBuf := buf
|
||||
if remaining > 0 && remaining < int64(len(buf)) {
|
||||
readBuf = buf[:remaining]
|
||||
}
|
||||
|
||||
n, readErr := file.Read(readBuf)
|
||||
n, readErr := file.Read(buf)
|
||||
if n > 0 {
|
||||
if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: readBuf[:n]}); err != nil {
|
||||
if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: buf[:n]}); err != nil {
|
||||
return err
|
||||
}
|
||||
if remaining > 0 {
|
||||
remaining -= int64(n)
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
@@ -322,21 +307,10 @@ func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_ser
|
||||
v.mu.Lock()
|
||||
v.readFileStatusCalls++
|
||||
v.mu.Unlock()
|
||||
|
||||
datInfo, err := os.Stat(v.filePath(req.VolumeId, ".dat"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
idxInfo, err := os.Stat(v.filePath(req.VolumeId, ".idx"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &volume_server_pb.ReadVolumeFileStatusResponse{
|
||||
VolumeId: req.VolumeId,
|
||||
DatFileSize: uint64(datInfo.Size()),
|
||||
IdxFileSize: uint64(idxInfo.Size()),
|
||||
DatFileSize: 1024,
|
||||
IdxFileSize: 16,
|
||||
FileCount: 1,
|
||||
}, nil
|
||||
}
|
||||
@@ -375,27 +349,7 @@ func (v *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, strea
|
||||
v.volumeCopyCalls++
|
||||
v.mu.Unlock()
|
||||
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
var statusResp *volume_server_pb.ReadVolumeFileStatusResponse
|
||||
if err := operation.WithVolumeServerClient(false, pb.ServerAddress(req.SourceDataNode), dialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
var readErr error
|
||||
statusResp, readErr = client.ReadVolumeFileStatus(stream.Context(), &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: req.VolumeId,
|
||||
})
|
||||
return readErr
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".dat", statusResp.DatFileSize, dialOption); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".idx", statusResp.IdxFileSize, dialOption); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: int64(statusResp.DatFileSize + statusResp.IdxFileSize)}); err != nil {
|
||||
if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: 1024}); err != nil {
|
||||
return err
|
||||
}
|
||||
return stream.Send(&volume_server_pb.VolumeCopyResponse{LastAppendAtNs: uint64(time.Now().UnixNano())})
|
||||
@@ -414,44 +368,3 @@ func (v *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serve
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeTailReceiverResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) copyRemoteFile(ctx context.Context, sourceDataNode string, volumeID uint32, ext string, fileSize uint64, dialOption grpc.DialOption) error {
|
||||
path := v.filePath(volumeID, ext)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return operation.WithVolumeServerClient(true, pb.ServerAddress(sourceDataNode), dialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
stream, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
|
||||
VolumeId: volumeID,
|
||||
Ext: ext,
|
||||
StopOffset: fileSize,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if recvErr != nil {
|
||||
return recvErr
|
||||
}
|
||||
if len(resp.FileContent) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := file.Write(resp.FileContent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,8 +16,6 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const testVolumeDatSize = 1 * 1024 * 1024
|
||||
|
||||
func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
volumeID := uint32(303)
|
||||
|
||||
@@ -33,7 +31,6 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
|
||||
source := pluginworkers.NewVolumeServer(t, "")
|
||||
target := pluginworkers.NewVolumeServer(t, "")
|
||||
pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), volumeID, testVolumeDatSize)
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: fmt.Sprintf("balance-job-%d", volumeID),
|
||||
@@ -87,9 +84,6 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
|
||||
|
||||
// Build a batch job with 3 volume moves from source → target.
|
||||
volumeIDs := []uint32{401, 402, 403}
|
||||
for _, vid := range volumeIDs {
|
||||
pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), vid, testVolumeDatSize)
|
||||
}
|
||||
moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
|
||||
for i, vid := range volumeIDs {
|
||||
moves[i] = &worker_pb.BalanceMoveSpec{
|
||||
@@ -145,11 +139,10 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
|
||||
require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
|
||||
}
|
||||
|
||||
// Each move reads source status once before copy and once inside the
|
||||
// target's fake VolumeCopy implementation, then reads target status once
|
||||
// before deleting the source.
|
||||
require.Equal(t, len(volumeIDs)*2, source.ReadFileStatusCount(),
|
||||
"each move should read source volume status before copy and during target copy")
|
||||
// Pre-delete verification should have called ReadVolumeFileStatus on both
|
||||
// source and target for each volume.
|
||||
require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
|
||||
"each move should read source volume status before delete")
|
||||
require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
|
||||
"each move should read target volume status before delete")
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
@@ -22,12 +21,7 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
|
||||
datPath := filepath.Join(baseDir, volumeFilename(volumeID, ".dat"))
|
||||
idxPath := filepath.Join(baseDir, volumeFilename(volumeID, ".idx"))
|
||||
|
||||
// The idx entry records the needle's data size. The actual on-disk size
|
||||
// includes header + checksum + timestamp (GetActualSize). The .dat must
|
||||
// be large enough to hold the full needle.
|
||||
needleDataSize := types.Size(datSize)
|
||||
actualSize := needle.GetActualSize(needleDataSize, needle.Version3)
|
||||
data := make([]byte, actualSize)
|
||||
data := make([]byte, datSize)
|
||||
rng := rand.New(rand.NewSource(99))
|
||||
_, _ = rng.Read(data)
|
||||
if err := os.WriteFile(datPath, data, 0644); err != nil {
|
||||
@@ -41,7 +35,7 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
|
||||
|
||||
types.NeedleIdToBytes(entry[:idEnd], types.NeedleId(1))
|
||||
types.OffsetToBytes(entry[idEnd:offsetEnd], types.ToOffset(0))
|
||||
types.SizeToBytes(entry[offsetEnd:sizeEnd], needleDataSize)
|
||||
types.SizeToBytes(entry[offsetEnd:sizeEnd], types.Size(datSize))
|
||||
|
||||
if err := os.WriteFile(idxPath, entry, 0644); err != nil {
|
||||
t.Fatalf("write idx file: %v", err)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -12,8 +13,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
type TestEnvironment struct {
|
||||
@@ -97,26 +96,29 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
bindIP := findBindIP()
|
||||
|
||||
// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
|
||||
ports := testutil.MustAllocatePorts(t, 9)
|
||||
masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := mustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := mustFreePortPair(t, "S3")
|
||||
icebergPort := mustFreePort(t, "Iceberg")
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
masterPort: ports[0],
|
||||
masterGrpcPort: ports[1],
|
||||
volumePort: ports[2],
|
||||
volumeGrpcPort: ports[3],
|
||||
filerPort: ports[4],
|
||||
filerGrpcPort: ports[5],
|
||||
s3Port: ports[6],
|
||||
s3GrpcPort: ports[7],
|
||||
icebergPort: ports[8],
|
||||
dockerAvailable: testutil.HasDocker(),
|
||||
s3Port: s3Port,
|
||||
s3GrpcPort: s3GrpcPort,
|
||||
icebergPort: icebergPort,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
filerPort: filerPort,
|
||||
filerGrpcPort: filerGrpcPort,
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
dockerAvailable: hasDocker(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -331,6 +333,73 @@ func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
|
||||
}
|
||||
}
|
||||
|
||||
func hasDocker() bool {
|
||||
cmd := exec.Command("docker", "version")
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
|
||||
port, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for %s: %v", name, err)
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
func mustFreePortPair(t *testing.T, name string) (int, int) {
|
||||
t.Helper()
|
||||
|
||||
httpPort, grpcPort, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
|
||||
}
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func findAvailablePortPair() (int, int, error) {
|
||||
httpPort, err := getFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
grpcPort, err := getFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return httpPort, grpcPort, nil
|
||||
}
|
||||
|
||||
func getFreePort() (int, error) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, nil
|
||||
}
|
||||
|
||||
func findBindIP() string {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "127.0.0.1"
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
ipNet, ok := addr.(*net.IPNet)
|
||||
if !ok || ipNet.IP == nil {
|
||||
continue
|
||||
}
|
||||
ip := ipNet.IP.To4()
|
||||
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
return "127.0.0.1"
|
||||
}
|
||||
|
||||
func randomString(length int) string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
b := make([]byte, length)
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
# S3 delete regression test harness
|
||||
|
||||
.PHONY: all test help build-weed check-deps start-server stop-server test-delete test-with-server clean logs
|
||||
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
S3_PORT := 8333
|
||||
TEST_TIMEOUT := 10m
|
||||
SERVER_DIR := ./test-volume-data
|
||||
S3_CONFIG := ./test_s3.json
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
all: test
|
||||
|
||||
test: test-with-server
|
||||
|
||||
help:
|
||||
@echo "S3 delete regression test harness"
|
||||
@echo ""
|
||||
@echo "Available targets:"
|
||||
@echo " build-weed - Build the SeaweedFS binary"
|
||||
@echo " start-server - Start SeaweedFS with S3 enabled"
|
||||
@echo " stop-server - Stop the test server"
|
||||
@echo " test-delete - Run delete regression tests"
|
||||
@echo " test-with-server - Start server, run tests, stop server"
|
||||
@echo " logs - Show server logs"
|
||||
@echo " clean - Remove test artifacts"
|
||||
|
||||
build-weed:
|
||||
@echo "Building SeaweedFS binary..."
|
||||
@cd ../../../weed && go build -o weed_binary .
|
||||
@chmod +x $(WEED_BINARY)
|
||||
|
||||
check-deps: build-weed
|
||||
@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
|
||||
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
|
||||
@test -f $(S3_CONFIG) || (echo "S3 config not found at $(S3_CONFIG)" && exit 1)
|
||||
|
||||
start-server: check-deps
|
||||
@echo "Starting SeaweedFS server for delete regression tests..."
|
||||
@rm -f weed-server.pid
|
||||
@mkdir -p $(SERVER_DIR)
|
||||
@$(WEED_BINARY) mini \
|
||||
-dir=$(SERVER_DIR) \
|
||||
-s3 \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=$(S3_CONFIG) \
|
||||
-master.peers=none \
|
||||
> weed-test.log 2>&1 & echo $$! > weed-server.pid
|
||||
@for i in $$(seq 1 60); do \
|
||||
if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
|
||||
echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "SeaweedFS S3 server failed to start"; \
|
||||
echo "=== Server logs ==="; \
|
||||
test -f weed-test.log && cat weed-test.log || true; \
|
||||
exit 1
|
||||
|
||||
stop-server:
|
||||
@echo "Stopping SeaweedFS server..."
|
||||
@if [ -f weed-server.pid ]; then \
|
||||
SERVER_PID=$$(cat weed-server.pid); \
|
||||
kill -TERM $$SERVER_PID 2>/dev/null || true; \
|
||||
for i in $$(seq 1 20); do \
|
||||
if ! kill -0 $$SERVER_PID >/dev/null 2>&1; then \
|
||||
break; \
|
||||
fi; \
|
||||
sleep 0.1; \
|
||||
done; \
|
||||
if kill -0 $$SERVER_PID >/dev/null 2>&1; then \
|
||||
kill -KILL $$SERVER_PID 2>/dev/null || true; \
|
||||
fi; \
|
||||
rm -f weed-server.pid; \
|
||||
fi
|
||||
|
||||
test-delete: check-deps
|
||||
@echo "Running delete regression tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) .
|
||||
|
||||
test-with-server: start-server
|
||||
@trap "$(MAKE) stop-server" EXIT; \
|
||||
$(MAKE) test-delete || (echo "=== Last 100 lines of server logs ===" && tail -100 weed-test.log && exit 1)
|
||||
@$(MAKE) stop-server
|
||||
|
||||
logs:
|
||||
@if test -f weed-test.log; then \
|
||||
tail -f weed-test.log; \
|
||||
else \
|
||||
echo "No server log file found."; \
|
||||
fi
|
||||
|
||||
clean:
|
||||
@$(MAKE) stop-server
|
||||
@rm -f weed-test.log weed-server.pid
|
||||
@rm -rf $(SERVER_DIR)
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "admin",
|
||||
"secretKey": "admin"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -20,11 +20,8 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
@@ -116,7 +113,7 @@ func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
|
||||
require.NoError(t, os.MkdirAll(dir, 0o755), "create %s", dir)
|
||||
}
|
||||
|
||||
ports, err := testutil.AllocatePorts(12)
|
||||
ports, err := allocatePorts(12)
|
||||
require.NoError(t, err, "allocate ports")
|
||||
cluster.masterPort = ports[0]
|
||||
cluster.masterGrpcPort = ports[1]
|
||||
@@ -142,7 +139,6 @@ func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
|
||||
require.NoError(t, cluster.waitForTCP(cluster.filerGRPCAddress(i), 30*time.Second), "wait for filer %d grpc\n%s", i, cluster.tailLog(fmt.Sprintf("filer%d.log", i)))
|
||||
}
|
||||
require.NoError(t, cluster.waitForFilerCount(2, 30*time.Second), "wait for filer group registration")
|
||||
require.NoError(t, cluster.waitForLockRingConverged(30*time.Second), "wait for lock ring convergence")
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
require.NoError(t, cluster.startS3(i), "start s3 %d", i)
|
||||
@@ -415,130 +411,6 @@ func (c *distributedLockCluster) waitForFilerCount(expected int, timeout time.Du
|
||||
return fmt.Errorf("timed out waiting for %d filers in group %q", expected, c.filerGroup)
|
||||
}
|
||||
|
||||
// waitForLockRingConverged verifies that both filers have a consistent view of the
|
||||
// lock ring by acquiring the same lock through each filer and checking mutual exclusion.
|
||||
// This guards against the window between master seeing both filers and the filers
|
||||
// actually receiving the LockRingUpdate broadcast (delayed by the stabilization timer).
|
||||
//
|
||||
// A single arbitrary key could false-pass: if the key's real primary is filer0 and
|
||||
// filer0 has a stale ring (only sees itself), it still grants correctly because it IS
|
||||
// the primary. To catch the stale ring we must also test a key whose real primary is
|
||||
// filer1. So we generate one test key per primary filer using the same consistent-hash
|
||||
// ring as production, and require mutual exclusion for all of them.
|
||||
func (c *distributedLockCluster) waitForLockRingConverged(timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
owners := make([]pb.ServerAddress, 0, len(c.filerPorts))
|
||||
for i := range c.filerPorts {
|
||||
owners = append(owners, c.filerServerAddress(i))
|
||||
}
|
||||
ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
|
||||
ring.SetServers(owners)
|
||||
|
||||
attempt := 0
|
||||
for time.Now().Before(deadline) {
|
||||
// Generate one unique test key per primary filer
|
||||
testKeys := c.convergenceKeysPerPrimary(ring, owners, attempt)
|
||||
attempt++
|
||||
|
||||
allConverged := true
|
||||
for _, key := range testKeys {
|
||||
converged, _ := c.checkLockMutualExclusion(key)
|
||||
if !converged {
|
||||
allConverged = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allConverged {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("lock ring did not converge: both filers independently grant the same lock")
|
||||
}
|
||||
|
||||
// convergenceKeysPerPrimary returns one lock key per distinct primary filer,
|
||||
// using the same consistent-hash ring as production routing.
|
||||
func (c *distributedLockCluster) convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
|
||||
keysByOwner := make(map[pb.ServerAddress]string, len(owners))
|
||||
for i := 0; len(keysByOwner) < len(owners) && i < 1024; i++ {
|
||||
candidate := fmt.Sprintf("convergence-%d-%d", attempt, i)
|
||||
primary := ring.GetPrimary(candidate)
|
||||
if _, exists := keysByOwner[primary]; !exists {
|
||||
keysByOwner[primary] = candidate
|
||||
}
|
||||
}
|
||||
keys := make([]string, 0, len(keysByOwner))
|
||||
for _, k := range keysByOwner {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// checkLockMutualExclusion acquires a lock via filer0, then tries the same lock via filer1.
|
||||
// Returns true if mutual exclusion holds (second attempt is denied).
|
||||
func (c *distributedLockCluster) checkLockMutualExclusion(testKey string) (bool, error) {
|
||||
conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer conn0.Close()
|
||||
|
||||
conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer conn1.Close()
|
||||
|
||||
client0 := filer_pb.NewSeaweedFilerClient(conn0)
|
||||
client1 := filer_pb.NewSeaweedFilerClient(conn1)
|
||||
|
||||
// Acquire lock via filer0
|
||||
ctx0, cancel0 := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel0()
|
||||
resp0, err := client0.DistributedLock(ctx0, &filer_pb.LockRequest{
|
||||
Name: testKey,
|
||||
SecondsToLock: 5,
|
||||
Owner: "convergence-filer0",
|
||||
})
|
||||
if err != nil || resp0.RenewToken == "" {
|
||||
return false, fmt.Errorf("filer0 lock failed: err=%v resp=%v", err, resp0)
|
||||
}
|
||||
defer func() {
|
||||
// Always release the lock we acquired
|
||||
client0.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
|
||||
Name: testKey,
|
||||
RenewToken: resp0.RenewToken,
|
||||
})
|
||||
}()
|
||||
|
||||
// Try the same lock via filer1 - should be denied
|
||||
ctx1, cancel1 := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel1()
|
||||
resp1, err := client1.DistributedLock(ctx1, &filer_pb.LockRequest{
|
||||
Name: testKey,
|
||||
SecondsToLock: 5,
|
||||
Owner: "convergence-filer1",
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if resp1.RenewToken != "" {
|
||||
// Both filers granted the lock - ring not converged. Release the second lock.
|
||||
client1.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
|
||||
Name: testKey,
|
||||
RenewToken: resp1.RenewToken,
|
||||
})
|
||||
return false, nil
|
||||
}
|
||||
// Verify the denial is specifically because the lock is already held,
|
||||
// not due to a transient error that might give a false positive.
|
||||
if !strings.Contains(resp1.Error, "lock already owned") {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (c *distributedLockCluster) waitForHTTP(url string, timeout time.Duration) error {
|
||||
client := &net.Dialer{Timeout: time.Second}
|
||||
httpClient := &httpClientWithDialer{dialer: client}
|
||||
@@ -592,6 +464,25 @@ func (c *distributedLockCluster) tailLog(name string) string {
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func allocatePorts(count int) ([]int, error) {
|
||||
listeners := make([]net.Listener, 0, count)
|
||||
ports := make([]int, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
for _, openListener := range listeners {
|
||||
_ = openListener.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
for _, l := range listeners {
|
||||
_ = l.Close()
|
||||
}
|
||||
return ports, nil
|
||||
}
|
||||
|
||||
func stopProcess(cmd *exec.Cmd) {
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -14,9 +15,9 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -73,31 +74,33 @@ func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key str
|
||||
t.Helper()
|
||||
|
||||
start := make(chan struct{})
|
||||
results := make(chan putAttemptResult, len(clients))
|
||||
results := make(chan putAttemptResult, len(clients)*2)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, client := range clients {
|
||||
wg.Add(1)
|
||||
body := fmt.Sprintf("%s-race", client.name)
|
||||
go func(client s3RaceClient, body string) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
wg.Add(1)
|
||||
body := fmt.Sprintf("%s-attempt-%d", client.name, attempt)
|
||||
go func(client s3RaceClient, body string) {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := client.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte(body)),
|
||||
})
|
||||
results <- putAttemptResult{
|
||||
clientName: client.name,
|
||||
body: body,
|
||||
err: err,
|
||||
}
|
||||
}(client, body)
|
||||
_, err := client.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
IfNoneMatch: aws.String("*"),
|
||||
Body: bytes.NewReader([]byte(body)),
|
||||
})
|
||||
results <- putAttemptResult{
|
||||
clientName: client.name,
|
||||
body: body,
|
||||
err: err,
|
||||
}
|
||||
}(client, body)
|
||||
}
|
||||
}
|
||||
|
||||
close(start)
|
||||
@@ -124,7 +127,7 @@ func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key str
|
||||
|
||||
require.Empty(t, unexpectedErrors, "unexpected race errors")
|
||||
require.Equal(t, 1, successes, "exactly one write should win")
|
||||
require.Equal(t, len(clients)-1, preconditionFailures, "all losing writes should fail with 412")
|
||||
require.Equal(t, len(clients)*2-1, preconditionFailures, "all losing writes should fail with 412")
|
||||
|
||||
object, err := clients[0].client.GetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
@@ -148,15 +151,14 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
|
||||
for i := range c.filerPorts {
|
||||
owners = append(owners, c.filerServerAddress(i))
|
||||
}
|
||||
|
||||
ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
|
||||
ring.SetServers(owners)
|
||||
sort.Slice(owners, func(i, j int) bool {
|
||||
return owners[i] < owners[j]
|
||||
})
|
||||
|
||||
keysByOwner := make(map[pb.ServerAddress]string, len(owners))
|
||||
for i := 0; i < 1024 && len(keysByOwner) < len(owners); i++ {
|
||||
key := fmt.Sprintf("%s-%03d.txt", prefix, i)
|
||||
lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(key))
|
||||
lockOwner := ring.GetPrimary(lockKey)
|
||||
lockOwner := ownerForObjectLock(bucket, key, owners)
|
||||
if _, exists := keysByOwner[lockOwner]; !exists {
|
||||
keysByOwner[lockOwner] = key
|
||||
}
|
||||
@@ -164,6 +166,15 @@ func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb
|
||||
return keysByOwner
|
||||
}
|
||||
|
||||
func ownerForObjectLock(bucket, object string, owners []pb.ServerAddress) pb.ServerAddress {
|
||||
lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(object))
|
||||
hash := util.HashStringToLong(lockKey)
|
||||
if hash < 0 {
|
||||
hash = -hash
|
||||
}
|
||||
return owners[hash%int64(len(owners))]
|
||||
}
|
||||
|
||||
func lockOwnerLabel(owner pb.ServerAddress) string {
|
||||
replacer := strings.NewReplacer(":", "_", ".", "_")
|
||||
return "owner_" + replacer.Replace(string(owner))
|
||||
|
||||
@@ -321,13 +321,6 @@ func TestIAMPolicyManagement(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
|
||||
// List user policies before any inline policy is attached
|
||||
listResp, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, listResp.PolicyNames, "New user should have no inline policies")
|
||||
|
||||
policyName := "test-inline-policy"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`
|
||||
|
||||
@@ -339,14 +332,6 @@ func TestIAMPolicyManagement(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// List user policies after attaching inline policy
|
||||
listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, listResp.PolicyNames, "User should have inline policies after PutUserPolicy")
|
||||
assert.False(t, *listResp.IsTruncated)
|
||||
|
||||
// Get user policy
|
||||
getResp, err := iamClient.GetUserPolicy(&iam.GetUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
@@ -363,22 +348,5 @@ func TestIAMPolicyManagement(t *testing.T) {
|
||||
PolicyName: aws.String(policyName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// List user policies after deletion
|
||||
listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, listResp.PolicyNames, "User should have no inline policies after DeleteUserPolicy")
|
||||
})
|
||||
|
||||
t.Run("list_user_policies_nonexistent_user", func(t *testing.T) {
|
||||
_, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
|
||||
UserName: aws.String("nonexistent-user-for-list-policies"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
awsErr, ok := err.(awserr.Error)
|
||||
require.True(t, ok, "Expected AWS error type")
|
||||
assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -778,105 +778,6 @@ func TestIAMGroupRawAPI(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupInlinePolicy tests group inline policy operations:
|
||||
// PutGroupPolicy, GetGroupPolicy, ListGroupPolicies, DeleteGroupPolicy
|
||||
func TestIAMGroupInlinePolicy(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Skip if running against embedded IAM (which returns NotImplemented for group inline policies)
|
||||
_, probeErr := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{GroupName: aws.String("probe-group-inline-support")})
|
||||
if probeErr != nil {
|
||||
if awsErr, ok := probeErr.(awserr.Error); ok && awsErr.Code() == "NotImplemented" {
|
||||
t.Skip("Skipping: group inline policies not supported in embedded IAM mode")
|
||||
}
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-group-inline-policy"
|
||||
policyName := "TestInlinePolicy"
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
// Clean up inline policies before deleting the group
|
||||
iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyName: aws.String(policyName),
|
||||
})
|
||||
iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
|
||||
}()
|
||||
|
||||
t.Run("list_empty", func(t *testing.T) {
|
||||
resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, resp.PolicyNames)
|
||||
assert.False(t, *resp.IsTruncated)
|
||||
})
|
||||
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`
|
||||
|
||||
t.Run("put_group_policy", func(t *testing.T) {
|
||||
_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("list_after_put", func(t *testing.T) {
|
||||
resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, resp.PolicyNames)
|
||||
})
|
||||
|
||||
t.Run("get_group_policy", func(t *testing.T) {
|
||||
resp, err := iamClient.GetGroupPolicy(&iam.GetGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyName: aws.String(policyName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, groupName, *resp.GroupName)
|
||||
assert.Equal(t, policyName, *resp.PolicyName)
|
||||
assert.Contains(t, *resp.PolicyDocument, "s3:GetObject")
|
||||
})
|
||||
|
||||
t.Run("delete_group_policy", func(t *testing.T) {
|
||||
_, err := iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyName: aws.String(policyName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("list_after_delete", func(t *testing.T) {
|
||||
resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, resp.PolicyNames)
|
||||
})
|
||||
|
||||
t.Run("nonexistent_group", func(t *testing.T) {
|
||||
_, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
|
||||
GroupName: aws.String("nonexistent-group-for-policies"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
awsErr, ok := err.(awserr.Error)
|
||||
require.True(t, ok, "Expected AWS error type")
|
||||
assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
}
|
||||
|
||||
// createS3Client creates an S3 client with static credentials
|
||||
func createS3Client(t *testing.T, accessKey, secretKey string) *s3.S3 {
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
|
||||
@@ -1,511 +0,0 @@
|
||||
package iam
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// GetFederationTokenTestResponse represents the STS GetFederationToken response
|
||||
type GetFederationTokenTestResponse struct {
|
||||
XMLName xml.Name `xml:"GetFederationTokenResponse"`
|
||||
Result struct {
|
||||
Credentials struct {
|
||||
AccessKeyId string `xml:"AccessKeyId"`
|
||||
SecretAccessKey string `xml:"SecretAccessKey"`
|
||||
SessionToken string `xml:"SessionToken"`
|
||||
Expiration string `xml:"Expiration"`
|
||||
} `xml:"Credentials"`
|
||||
FederatedUser struct {
|
||||
FederatedUserId string `xml:"FederatedUserId"`
|
||||
Arn string `xml:"Arn"`
|
||||
} `xml:"FederatedUser"`
|
||||
} `xml:"GetFederationTokenResult"`
|
||||
}
|
||||
|
||||
func getTestCredentials() (string, string) {
|
||||
accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
|
||||
if accessKey == "" {
|
||||
accessKey = "admin"
|
||||
}
|
||||
secretKey := os.Getenv("STS_TEST_SECRET_KEY")
|
||||
if secretKey == "" {
|
||||
secretKey = "admin"
|
||||
}
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
// isGetFederationTokenImplemented checks if the running server supports GetFederationToken
|
||||
func isGetFederationTokenImplemented(t *testing.T) bool {
|
||||
accessKey, secretKey := getTestCredentials()
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"probe"},
|
||||
}, accessKey, secretKey)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
if xml.Unmarshal(body, &errResp) == nil {
|
||||
if errResp.Error.Code == "InvalidAction" || errResp.Error.Code == "NotImplemented" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// TestSTSGetFederationTokenValidation tests input validation for the GetFederationToken endpoint
|
||||
func TestSTSGetFederationTokenValidation(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
if !isSTSEndpointRunning(t) {
|
||||
t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
|
||||
}
|
||||
|
||||
if !isGetFederationTokenImplemented(t) {
|
||||
t.Fatal("GetFederationToken action is not implemented in the running server")
|
||||
}
|
||||
|
||||
accessKey, secretKey := getTestCredentials()
|
||||
|
||||
t.Run("missing_name", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
// Name is missing
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
|
||||
assert.Equal(t, "MissingParameter", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("name_too_short", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"A"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
|
||||
assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("name_too_long", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {strings.Repeat("A", 33)},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
|
||||
assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("name_invalid_characters", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"bad name"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
|
||||
assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("duration_too_short", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"TestApp"},
|
||||
"DurationSeconds": {"100"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
|
||||
assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("duration_too_long", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"TestApp"},
|
||||
"DurationSeconds": {"200000"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
|
||||
assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("malformed_policy", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"TestApp"},
|
||||
"Policy": {"not-valid-json"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var errResp STSErrorTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
|
||||
assert.Equal(t, "MalformedPolicyDocument", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("anonymous_rejected", func(t *testing.T) {
|
||||
// GetFederationToken requires SigV4, anonymous should fail
|
||||
resp, err := callSTSAPI(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"TestApp"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
assert.NotEqual(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
}
|
||||
|
||||
// TestSTSGetFederationTokenRejectTemporaryCredentials tests that temporary
|
||||
// credentials (session tokens) are rejected by GetFederationToken
|
||||
func TestSTSGetFederationTokenRejectTemporaryCredentials(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
if !isSTSEndpointRunning(t) {
|
||||
t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
|
||||
}
|
||||
|
||||
if !isGetFederationTokenImplemented(t) {
|
||||
t.Skip("GetFederationToken not implemented")
|
||||
}
|
||||
|
||||
accessKey, secretKey := getTestCredentials()
|
||||
|
||||
// First, obtain temporary credentials via AssumeRole
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"AssumeRole"},
|
||||
"Version": {"2011-06-15"},
|
||||
"RoleArn": {"arn:aws:iam::role/admin"},
|
||||
"RoleSessionName": {"temp-session"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Skipf("AssumeRole failed (may not be configured): status=%d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var assumeResp AssumeRoleTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &assumeResp), "Parse AssumeRole response: %s", string(body))
|
||||
|
||||
tempAccessKey := assumeResp.Result.Credentials.AccessKeyId
|
||||
tempSecretKey := assumeResp.Result.Credentials.SecretAccessKey
|
||||
tempSessionToken := assumeResp.Result.Credentials.SessionToken
|
||||
require.NotEmpty(t, tempAccessKey)
|
||||
require.NotEmpty(t, tempSessionToken)
|
||||
|
||||
// Now try GetFederationToken with the temporary credentials
|
||||
// Include X-Amz-Security-Token header which marks this as a temp credential call
|
||||
params := url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"ShouldFail"},
|
||||
}
|
||||
|
||||
reqBody := params.Encode()
|
||||
req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/", strings.NewReader(reqBody))
|
||||
require.NoError(t, err)
|
||||
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("X-Amz-Security-Token", tempSessionToken)
|
||||
|
||||
creds := credentials.NewStaticCredentials(tempAccessKey, tempSecretKey, tempSessionToken)
|
||||
signer := v4.NewSigner(creds)
|
||||
_, err = signer.Sign(req, strings.NewReader(reqBody), "sts", "us-east-1", time.Now())
|
||||
require.NoError(t, err)
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp2, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp2.Body.Close()
|
||||
|
||||
body2, _ := io.ReadAll(resp2.Body)
|
||||
assert.Equal(t, http.StatusForbidden, resp2.StatusCode,
|
||||
"GetFederationToken should reject temporary credentials: %s", string(body2))
|
||||
assert.Contains(t, string(body2), "temporary credentials",
|
||||
"Error should mention temporary credentials")
|
||||
}
|
||||
|
||||
// TestSTSGetFederationTokenSuccess tests a successful GetFederationToken call
|
||||
// and verifies the returned credentials can be used to access S3
|
||||
func TestSTSGetFederationTokenSuccess(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
if !isSTSEndpointRunning(t) {
|
||||
t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
|
||||
}
|
||||
|
||||
if !isGetFederationTokenImplemented(t) {
|
||||
t.Skip("GetFederationToken not implemented")
|
||||
}
|
||||
|
||||
accessKey, secretKey := getTestCredentials()
|
||||
|
||||
t.Run("basic_success", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"AppClient"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errResp STSErrorTestResponse
|
||||
_ = xml.Unmarshal(body, &errResp)
|
||||
t.Fatalf("GetFederationToken failed: code=%s message=%s", errResp.Error.Code, errResp.Error.Message)
|
||||
}
|
||||
|
||||
var stsResp GetFederationTokenTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &stsResp), "Parse response: %s", string(body))
|
||||
|
||||
creds := stsResp.Result.Credentials
|
||||
assert.NotEmpty(t, creds.AccessKeyId)
|
||||
assert.NotEmpty(t, creds.SecretAccessKey)
|
||||
assert.NotEmpty(t, creds.SessionToken)
|
||||
assert.NotEmpty(t, creds.Expiration)
|
||||
|
||||
fedUser := stsResp.Result.FederatedUser
|
||||
assert.Contains(t, fedUser.Arn, "federated-user/AppClient")
|
||||
assert.Contains(t, fedUser.FederatedUserId, "AppClient")
|
||||
})
|
||||
|
||||
t.Run("with_custom_duration", func(t *testing.T) {
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"DurationTest"},
|
||||
"DurationSeconds": {"3600"},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
|
||||
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
var stsResp GetFederationTokenTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &stsResp))
|
||||
assert.NotEmpty(t, stsResp.Result.Credentials.AccessKeyId)
|
||||
|
||||
// Verify expiration is roughly 1 hour from now
|
||||
expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
|
||||
require.NoError(t, err)
|
||||
diff := time.Until(expTime)
|
||||
assert.InDelta(t, 3600, diff.Seconds(), 60,
|
||||
"Expiration should be ~1 hour from now")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("with_36_hour_duration", func(t *testing.T) {
|
||||
// GetFederationToken allows up to 36 hours (unlike AssumeRole's 12h max)
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"LongDuration"},
|
||||
"DurationSeconds": {"129600"}, // 36 hours
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
var stsResp GetFederationTokenTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &stsResp))
|
||||
|
||||
expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
|
||||
require.NoError(t, err)
|
||||
diff := time.Until(expTime)
|
||||
assert.InDelta(t, 129600, diff.Seconds(), 60,
|
||||
"Expiration should be ~36 hours from now")
|
||||
} else {
|
||||
// Duration should not cause a rejection
|
||||
var errResp STSErrorTestResponse
|
||||
_ = xml.Unmarshal(body, &errResp)
|
||||
assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
|
||||
"36-hour duration should be accepted by GetFederationToken")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestSTSGetFederationTokenWithSessionPolicy tests that vended credentials
|
||||
// are scoped down by an inline session policy
|
||||
func TestSTSGetFederationTokenWithSessionPolicy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
if !isSTSEndpointRunning(t) {
|
||||
t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
|
||||
}
|
||||
|
||||
if !isGetFederationTokenImplemented(t) {
|
||||
t.Skip("GetFederationToken not implemented")
|
||||
}
|
||||
|
||||
accessKey, secretKey := getTestCredentials()
|
||||
|
||||
// Create a test bucket using admin credentials
|
||||
adminSess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
Endpoint: aws.String(TestSTSEndpoint),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
adminS3 := s3.New(adminSess)
|
||||
bucket := fmt.Sprintf("fed-token-test-%d", time.Now().UnixNano())
|
||||
|
||||
_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err)
|
||||
defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
|
||||
|
||||
_, err = adminS3.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("test.txt"),
|
||||
Body: strings.NewReader("hello"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("test.txt")})
|
||||
|
||||
// Get federated credentials with a session policy that only allows GetObject
|
||||
sessionPolicy := fmt.Sprintf(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["arn:aws:s3:::%s/*"]
|
||||
}]
|
||||
}`, bucket)
|
||||
|
||||
resp, err := callSTSAPIWithSigV4(t, url.Values{
|
||||
"Action": {"GetFederationToken"},
|
||||
"Version": {"2011-06-15"},
|
||||
"Name": {"ScopedClient"},
|
||||
"Policy": {sessionPolicy},
|
||||
}, accessKey, secretKey)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Logf("GetFederationToken response: status=%d body=%s", resp.StatusCode, string(body))
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Skipf("GetFederationToken failed (may need IAM policy config): %s", string(body))
|
||||
}
|
||||
|
||||
var stsResp GetFederationTokenTestResponse
|
||||
require.NoError(t, xml.Unmarshal(body, &stsResp))
|
||||
|
||||
fedCreds := stsResp.Result.Credentials
|
||||
require.NotEmpty(t, fedCreds.AccessKeyId)
|
||||
require.NotEmpty(t, fedCreds.SessionToken)
|
||||
|
||||
// Create S3 client with the federated credentials
|
||||
fedSess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
Endpoint: aws.String(TestSTSEndpoint),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
Credentials: credentials.NewStaticCredentials(
|
||||
fedCreds.AccessKeyId, fedCreds.SecretAccessKey, fedCreds.SessionToken),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
fedS3 := s3.New(fedSess)
|
||||
|
||||
// GetObject should succeed (allowed by session policy)
|
||||
getResp, err := fedS3.GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("test.txt"),
|
||||
})
|
||||
if err == nil {
|
||||
defer getResp.Body.Close()
|
||||
t.Log("GetObject with federated credentials succeeded (as expected)")
|
||||
} else {
|
||||
t.Logf("GetObject with federated credentials: %v (session policy enforcement may vary)", err)
|
||||
}
|
||||
|
||||
// PutObject should be denied (not allowed by session policy)
|
||||
_, err = fedS3.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("denied.txt"),
|
||||
Body: strings.NewReader("should fail"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Log("PutObject correctly denied with federated credentials")
|
||||
assert.Contains(t, err.Error(), "AccessDenied",
|
||||
"PutObject should be denied by session policy")
|
||||
} else {
|
||||
// Clean up if unexpectedly succeeded
|
||||
adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("denied.txt")})
|
||||
t.Log("PutObject unexpectedly succeeded — session policy enforcement may not be active")
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -24,7 +25,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/weed/command"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -112,16 +112,37 @@ func TestS3Integration(t *testing.T) {
|
||||
}
|
||||
|
||||
// findAvailablePort finds an available port by binding to port 0
|
||||
func findAvailablePort() (int, error) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, nil
|
||||
}
|
||||
|
||||
// startMiniCluster starts a weed mini instance directly without exec.
|
||||
// Extra flags (e.g. "-s3.allowDeleteBucketNotEmpty=false") can be appended via extraArgs.
|
||||
func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
|
||||
// Allocate non-colliding ports (including gRPC offsets) for weed mini.
|
||||
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
|
||||
masterPort := ports[0]
|
||||
volumePort := ports[1]
|
||||
filerPort := ports[2]
|
||||
s3Port := ports[3]
|
||||
// Find available ports
|
||||
masterPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find master port: %v", err)
|
||||
}
|
||||
volumePort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find volume port: %v", err)
|
||||
}
|
||||
filerPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find filer port: %v", err)
|
||||
}
|
||||
s3Port, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find s3 port: %v", err)
|
||||
}
|
||||
// Create temporary directory for test data
|
||||
testDir := t.TempDir()
|
||||
|
||||
@@ -146,7 +167,7 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
|
||||
|
||||
// Create empty security.toml to disable JWT authentication in tests
|
||||
securityToml := filepath.Join(testDir, "security.toml")
|
||||
err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
|
||||
err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to create security.toml: %v", err)
|
||||
@@ -212,9 +233,10 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
|
||||
}()
|
||||
|
||||
// Wait for S3 service to be ready
|
||||
if !testutil.WaitForService(cluster.s3Endpoint, 30*time.Second) {
|
||||
err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("S3 service failed to start at %s", cluster.s3Endpoint)
|
||||
return nil, fmt.Errorf("S3 service failed to start: %v", err)
|
||||
}
|
||||
|
||||
// If VOLUME_SERVER_IMPL=rust, start a Rust volume server alongside weed mini
|
||||
@@ -255,12 +277,14 @@ func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
|
||||
return fmt.Errorf("resolve rust volume binary: %v", err)
|
||||
}
|
||||
|
||||
rustPorts, err := testutil.AllocatePorts(2)
|
||||
rustVolumePort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return fmt.Errorf("find rust volume ports: %v", err)
|
||||
return fmt.Errorf("find rust volume port: %v", err)
|
||||
}
|
||||
rustVolumeGrpcPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return fmt.Errorf("find rust volume grpc port: %v", err)
|
||||
}
|
||||
rustVolumePort := rustPorts[0]
|
||||
rustVolumeGrpcPort := rustPorts[1]
|
||||
|
||||
rustVolumeDir := filepath.Join(c.dataDir, "rust-volume")
|
||||
if err := os.MkdirAll(rustVolumeDir, 0o755); err != nil {
|
||||
@@ -353,6 +377,25 @@ func (c *TestCluster) Stop() {
|
||||
}
|
||||
}
|
||||
|
||||
// waitForS3Ready waits for the S3 service to be ready
|
||||
func waitForS3Ready(endpoint string, timeout time.Duration) error {
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := client.Get(endpoint)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
// Wait a bit more to ensure service is fully ready
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
return fmt.Errorf("timeout waiting for S3 service at %s", endpoint)
|
||||
}
|
||||
|
||||
// Test functions
|
||||
|
||||
func testCreateBucket(t *testing.T, cluster *TestCluster) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -20,7 +21,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/weed/command"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -703,13 +703,43 @@ func uniqueName(prefix string) string {
|
||||
|
||||
// --- Test setup helpers ---
|
||||
|
||||
func findAvailablePort() (int, error) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer listener.Close()
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, nil
|
||||
}
|
||||
|
||||
// findAvailablePortPair finds an available http port P such that P and P+10000 (grpc) are both available
|
||||
func findAvailablePortPair() (int, int, error) {
|
||||
httpPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
grpcPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if grpcPort != httpPort {
|
||||
return httpPort, grpcPort, nil
|
||||
}
|
||||
}
|
||||
return 0, 0, fmt.Errorf("failed to find available port pair")
|
||||
}
|
||||
|
||||
func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
ports := testutil.MustAllocatePorts(t, 8)
|
||||
masterPort, masterGrpcPort := ports[0], ports[1]
|
||||
volumePort, volumeGrpcPort := ports[2], ports[3]
|
||||
filerPort, filerGrpcPort := ports[4], ports[5]
|
||||
s3Port, s3GrpcPort := ports[6], ports[7]
|
||||
masterPort, masterGrpcPort, err := findAvailablePortPair()
|
||||
require.NoError(t, err)
|
||||
volumePort, volumeGrpcPort, err := findAvailablePortPair()
|
||||
require.NoError(t, err)
|
||||
filerPort, filerGrpcPort, err := findAvailablePortPair()
|
||||
require.NoError(t, err)
|
||||
s3Port, s3GrpcPort, err := findAvailablePortPair()
|
||||
require.NoError(t, err)
|
||||
|
||||
testDir := t.TempDir()
|
||||
|
||||
@@ -730,7 +760,7 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
|
||||
// Disable authentication for tests
|
||||
securityToml := filepath.Join(testDir, "security.toml")
|
||||
err := os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
|
||||
err = os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure credential store for IAM tests
|
||||
@@ -789,9 +819,10 @@ enabled = true
|
||||
}()
|
||||
|
||||
// Wait for S3
|
||||
if !testutil.WaitForService(cluster.s3Endpoint, 60*time.Second) {
|
||||
err = waitForS3Ready(cluster.s3Endpoint, 60*time.Second)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("timeout waiting for S3 at %s", cluster.s3Endpoint)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If VOLUME_SERVER_IMPL=rust, start a Rust volume server alongside weed mini
|
||||
@@ -806,6 +837,19 @@ enabled = true
|
||||
return cluster, nil
|
||||
}
|
||||
|
||||
func waitForS3Ready(endpoint string, timeout time.Duration) error {
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := client.Get(endpoint)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("timeout waiting for S3")
|
||||
}
|
||||
|
||||
// startRustVolumeServer starts a Rust volume server that registers with the same master.
|
||||
func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
|
||||
@@ -816,12 +860,14 @@ func (c *TestCluster) startRustVolumeServer(t *testing.T) error {
|
||||
return fmt.Errorf("resolve rust volume binary: %v", err)
|
||||
}
|
||||
|
||||
rustPorts, err := testutil.AllocatePorts(2)
|
||||
rustVolumePort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return fmt.Errorf("find rust volume ports: %v", err)
|
||||
return fmt.Errorf("find rust volume port: %v", err)
|
||||
}
|
||||
rustVolumeGrpcPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return fmt.Errorf("find rust volume grpc port: %v", err)
|
||||
}
|
||||
rustVolumePort := rustPorts[0]
|
||||
rustVolumeGrpcPort := rustPorts[1]
|
||||
|
||||
rustVolumeDir := filepath.Join(c.dataDir, "rust-volume")
|
||||
if err := os.MkdirAll(rustVolumeDir, 0o755); err != nil {
|
||||
|
||||
@@ -70,8 +70,7 @@ print("WRITE_COUNT=" + str(count))
|
||||
"issue-8285/output/_temporary/0/",
|
||||
"issue-8285/output/_temporary/0/_temporary/",
|
||||
}
|
||||
// Empty folder cleanup has a 2m default delay + 30s processor interval
|
||||
lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 3*time.Minute)
|
||||
lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 35*time.Second)
|
||||
if len(lingering) > 0 {
|
||||
t.Fatalf("issue #8285 regression detected: lingering temporary directories: %v", lingering)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -14,7 +16,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
)
|
||||
|
||||
@@ -94,10 +96,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
t.Fatalf("failed to create temp directory: %v", err)
|
||||
}
|
||||
|
||||
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3"})
|
||||
env.masterPort = ports[0]
|
||||
env.filerPort = ports[1]
|
||||
env.s3Port = ports[2]
|
||||
env.masterPort = mustFreePort(t, "Master")
|
||||
env.filerPort = mustFreePort(t, "Filer")
|
||||
env.s3Port = mustFreePort(t, "S3")
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
|
||||
@@ -134,14 +135,14 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
}
|
||||
registerMiniProcess(env.masterProcess)
|
||||
|
||||
if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
|
||||
if !waitForPort(env.masterPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
|
||||
}
|
||||
if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
|
||||
if !waitForPort(env.filerPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
|
||||
}
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
|
||||
t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
|
||||
if !waitForPort(env.s3Port, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,6 +208,46 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
|
||||
for i := 0; i < 200; i++ {
|
||||
port := 20000 + rand.Intn(30000)
|
||||
listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = listener.Close()
|
||||
|
||||
grpcPort := port + 10000
|
||||
if grpcPort > 65535 {
|
||||
continue
|
||||
}
|
||||
grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_ = grpcListener.Close()
|
||||
return port
|
||||
}
|
||||
|
||||
t.Fatalf("failed to get free port for %s", name)
|
||||
return 0
|
||||
}
|
||||
|
||||
func waitForPort(port int, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -98,8 +98,8 @@ start-seaweedfs: check-binary
|
||||
# Create S3 configuration with SSE-KMS support
|
||||
@printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json
|
||||
|
||||
# Start weed mini (WEED_S3_SSE_KEY enables SSE-S3 encryption)
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key $(SEAWEEDFS_BINARY) mini \
|
||||
# Start weed mini
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(SEAWEEDFS_BINARY) mini \
|
||||
-dir=/tmp/seaweedfs-test-sse \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=/tmp/seaweedfs-sse-s3.json \
|
||||
@@ -337,9 +337,8 @@ start-seaweedfs-ci: check-binary
|
||||
s3-config-template.json > /tmp/seaweedfs-s3.json
|
||||
|
||||
# Start weed mini with embedded S3 using the JSON config (with verbose logging)
|
||||
# WEED_S3_SSE_KEY enables SSE-S3 encryption for testing (KEK derived via HKDF)
|
||||
@echo "Starting weed mini with embedded S3..."
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
|
||||
-dir=/tmp/seaweedfs-test-sse \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=/tmp/seaweedfs-s3.json \
|
||||
@@ -483,7 +482,7 @@ test-volume-encryption: build-weed
|
||||
-e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
|
||||
s3-config-template.json > /tmp/seaweedfs-s3.json
|
||||
@echo "Starting weed mini with S3 volume encryption..."
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) WEED_S3_SSE_KEY=test-sse-s3-key GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
|
||||
-dir=/tmp/seaweedfs-test-sse \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=/tmp/seaweedfs-s3.json \
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# S3 API Test Makefile
|
||||
# This Makefile provides comprehensive targets for running S3 versioning tests
|
||||
|
||||
.PHONY: all test help build-weed setup-server start-server stop-server test-versioning test-versioning-quick test-versioning-regressions test-versioning-comprehensive test-all clean logs check-deps
|
||||
.PHONY: help build-weed setup-server start-server stop-server test-versioning test-versioning-quick test-versioning-comprehensive test-all clean logs check-deps
|
||||
|
||||
# Configuration
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
@@ -12,12 +12,6 @@ FILER_PORT := 8888
|
||||
TEST_TIMEOUT := 10m
|
||||
TEST_PATTERN := TestVersioning
|
||||
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
all: test
|
||||
|
||||
test: test-with-server
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "S3 API Test Makefile"
|
||||
@@ -31,7 +25,6 @@ help:
|
||||
@echo " stop-server - Stop SeaweedFS server"
|
||||
@echo " test-versioning - Run all versioning tests"
|
||||
@echo " test-versioning-quick - Run core versioning tests only"
|
||||
@echo " test-versioning-regressions - Run targeted versioning regression tests"
|
||||
@echo " test-versioning-simple - Run tests without server management"
|
||||
@echo " test-versioning-comprehensive - Run comprehensive versioning tests"
|
||||
@echo " test-all - Run all S3 API tests"
|
||||
@@ -186,12 +179,6 @@ test-versioning-quick: check-deps
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestBucketListReturnDataVersioning|TestVersioningBasicWorkflow|TestVersioningDeleteMarkers" .
|
||||
@echo "✅ Core versioning tests completed"
|
||||
|
||||
# Targeted regressions for conditional mutation, multipart, copy, and suspended delete flows.
|
||||
test-versioning-regressions: check-deps
|
||||
@echo "Running targeted S3 versioning regression tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker" .
|
||||
@echo "✅ Targeted versioning regression tests completed"
|
||||
|
||||
# All versioning tests
|
||||
test-versioning: check-deps
|
||||
@echo "Running all S3 versioning tests..."
|
||||
@@ -361,4 +348,4 @@ compare-python-tests:
|
||||
@echo "Python equivalent: test_bucket_list_return_data_versioning"
|
||||
@echo ""
|
||||
@echo "Running Go version..."
|
||||
@time go test -v -run "TestBucketListReturnDataVersioning" . 2>&1 | grep -E "(PASS|FAIL|took)"
|
||||
@time go test -v -run "TestBucketListReturnDataVersioning" . 2>&1 | grep -E "(PASS|FAIL|took)"
|
||||
@@ -289,157 +289,6 @@ func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// TestVersioningPaginationDeepDirectoryHierarchy tests that paginated ListObjectVersions
|
||||
// correctly skips directory subtrees before the key-marker. This reproduces the
|
||||
// real-world scenario where Veeam backup objects are spread across many subdirectories
|
||||
// (e.g., Mailboxes/<uuid>/ItemsData/<file>) and pagination becomes exponentially
|
||||
// slower as the marker advances through the tree.
|
||||
//
|
||||
// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationDeepDirectoryHierarchy -timeout 10m
|
||||
func TestVersioningPaginationDeepDirectoryHierarchy(t *testing.T) {
|
||||
if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
|
||||
t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
|
||||
}
|
||||
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
|
||||
|
||||
// Create a deep directory structure mimicking Veeam 365 backup layout:
|
||||
// Backup/Organizations/<org>/Mailboxes/<mailbox>/ItemsData/<file>
|
||||
numMailboxes := 20
|
||||
filesPerMailbox := 5
|
||||
totalObjects := numMailboxes * filesPerMailbox
|
||||
orgPrefix := "Backup/Organizations/org-001/Mailboxes"
|
||||
|
||||
t.Logf("Creating %d objects across %d subdirectories (depth=6)...", totalObjects, numMailboxes)
|
||||
startTime := time.Now()
|
||||
|
||||
allKeys := make([]string, 0, totalObjects)
|
||||
for i := 0; i < numMailboxes; i++ {
|
||||
mailboxId := fmt.Sprintf("mbx-%03d", i)
|
||||
for j := 0; j < filesPerMailbox; j++ {
|
||||
key := fmt.Sprintf("%s/%s/ItemsData/file-%03d.dat", orgPrefix, mailboxId, j)
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(fmt.Sprintf("content-%d-%d", i, j)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
allKeys = append(allKeys, key)
|
||||
}
|
||||
}
|
||||
t.Logf("Created %d objects in %v", totalObjects, time.Since(startTime))
|
||||
|
||||
// Test 1: Paginate through all versions with a broad prefix and small maxKeys.
|
||||
// This forces multiple pages that must skip earlier subdirectory trees.
|
||||
t.Run("PaginateAcrossSubdirectories", func(t *testing.T) {
|
||||
maxKeys := int32(10) // Force many pages to exercise marker skipping
|
||||
var allVersions []types.ObjectVersion
|
||||
var keyMarker, versionIdMarker *string
|
||||
pageCount := 0
|
||||
pageStartTimes := make([]time.Duration, 0)
|
||||
|
||||
for {
|
||||
pageStart := time.Now()
|
||||
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(orgPrefix + "/"),
|
||||
MaxKeys: aws.Int32(maxKeys),
|
||||
KeyMarker: keyMarker,
|
||||
VersionIdMarker: versionIdMarker,
|
||||
})
|
||||
pageDuration := time.Since(pageStart)
|
||||
require.NoError(t, err)
|
||||
pageCount++
|
||||
pageStartTimes = append(pageStartTimes, pageDuration)
|
||||
|
||||
allVersions = append(allVersions, resp.Versions...)
|
||||
t.Logf("Page %d: %d versions in %v (marker: %v)",
|
||||
pageCount, len(resp.Versions), pageDuration,
|
||||
keyMarker)
|
||||
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker = resp.NextKeyMarker
|
||||
versionIdMarker = resp.NextVersionIdMarker
|
||||
}
|
||||
|
||||
assert.Greater(t, pageCount, 1, "Should require multiple pages")
|
||||
|
||||
// Verify listed keys exactly match the keys we created (same elements, same order)
|
||||
listedKeys := make([]string, 0, len(allVersions))
|
||||
for _, v := range allVersions {
|
||||
listedKeys = append(listedKeys, *v.Key)
|
||||
}
|
||||
assert.Equal(t, allKeys, listedKeys,
|
||||
"Listed version keys should exactly match created keys")
|
||||
|
||||
// Check that later pages don't take dramatically longer than earlier ones.
|
||||
// Before the fix, later pages were exponentially slower because they
|
||||
// re-traversed the entire tree. With the fix, all pages should be similar.
|
||||
if len(pageStartTimes) >= 4 {
|
||||
firstQuarter := pageStartTimes[0]
|
||||
lastQuarter := pageStartTimes[len(pageStartTimes)-1]
|
||||
t.Logf("First page: %v, Last page: %v, Ratio: %.1fx",
|
||||
firstQuarter, lastQuarter,
|
||||
float64(lastQuarter)/float64(firstQuarter))
|
||||
// Allow generous 10x ratio to avoid flakiness; before the fix
|
||||
// the ratio was 100x+ on large datasets
|
||||
if lastQuarter > firstQuarter*10 && lastQuarter > 500*time.Millisecond {
|
||||
t.Errorf("Last page took %.1fx longer than first page (%v vs %v) — possible pagination regression",
|
||||
float64(lastQuarter)/float64(firstQuarter), lastQuarter, firstQuarter)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Test 2: Paginate with delimiter to verify CommonPrefixes interaction
|
||||
t.Run("PaginateWithDelimiterAcrossSubdirs", func(t *testing.T) {
|
||||
maxKeys := int32(5)
|
||||
var allPrefixes []string
|
||||
var keyMarker *string
|
||||
pageCount := 0
|
||||
|
||||
for {
|
||||
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(orgPrefix + "/"),
|
||||
Delimiter: aws.String("/"),
|
||||
MaxKeys: aws.Int32(maxKeys),
|
||||
KeyMarker: keyMarker,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
pageCount++
|
||||
|
||||
for _, cp := range resp.CommonPrefixes {
|
||||
allPrefixes = append(allPrefixes, *cp.Prefix)
|
||||
}
|
||||
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker = resp.NextKeyMarker
|
||||
}
|
||||
|
||||
assert.Greater(t, pageCount, 1, "Should require multiple pages with maxKeys=%d", maxKeys)
|
||||
|
||||
// Build the exact expected prefixes
|
||||
expectedPrefixes := make([]string, 0, numMailboxes)
|
||||
for i := 0; i < numMailboxes; i++ {
|
||||
expectedPrefixes = append(expectedPrefixes,
|
||||
fmt.Sprintf("%s/mbx-%03d/", orgPrefix, i))
|
||||
}
|
||||
assert.Equal(t, expectedPrefixes, allPrefixes,
|
||||
"CommonPrefixes should exactly match expected mailbox prefixes")
|
||||
})
|
||||
}
|
||||
|
||||
// listAllVersions is a helper to list all versions of a specific object using pagination
|
||||
func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
|
||||
var allVersions []types.ObjectVersion
|
||||
|
||||
@@ -1,465 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
// oauthTestEnv holds a weed mini instance started with IAM credentials
|
||||
// so that the OAuth token endpoint is functional.
|
||||
type oauthTestEnv struct {
|
||||
seaweedDir string
|
||||
weedBinary string
|
||||
dataDir string
|
||||
bindIP string
|
||||
s3Port int
|
||||
s3GrpcPort int
|
||||
icebergPort int
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
filerPort int
|
||||
filerGrpcPort int
|
||||
volumePort int
|
||||
volumeGrpcPort int
|
||||
webdavPort int
|
||||
weedProcess *exec.Cmd
|
||||
weedCancel context.CancelFunc
|
||||
accessKey string
|
||||
secretKey string
|
||||
}
|
||||
|
||||
func newOAuthTestEnv(t *testing.T) *oauthTestEnv {
|
||||
t.Helper()
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("get working directory: %v", err)
|
||||
}
|
||||
|
||||
seaweedDir := wd
|
||||
for i := 0; i < 6; i++ {
|
||||
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
|
||||
break
|
||||
}
|
||||
seaweedDir = filepath.Dir(seaweedDir)
|
||||
}
|
||||
|
||||
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
|
||||
if info, err := os.Stat(weedBinary); err != nil || info.IsDir() {
|
||||
weedBinary = "weed"
|
||||
if _, err := exec.LookPath(weedBinary); err != nil {
|
||||
t.Skip("weed binary not found, skipping integration test")
|
||||
}
|
||||
}
|
||||
|
||||
dataDir, err := os.MkdirTemp("", "seaweed-oauth-test-*")
|
||||
if err != nil {
|
||||
t.Fatalf("create temp dir: %v", err)
|
||||
}
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
ports := testutil.MustAllocatePorts(t, 10)
|
||||
|
||||
return &oauthTestEnv{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
masterPort: ports[0],
|
||||
masterGrpcPort: ports[1],
|
||||
volumePort: ports[2],
|
||||
volumeGrpcPort: ports[3],
|
||||
filerPort: ports[4],
|
||||
filerGrpcPort: ports[5],
|
||||
s3Port: ports[6],
|
||||
s3GrpcPort: ports[7],
|
||||
icebergPort: ports[8],
|
||||
webdavPort: ports[9],
|
||||
accessKey: "AKIAIOSFODNN7EXAMPLE",
|
||||
secretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
}
|
||||
}
|
||||
|
||||
func (env *oauthTestEnv) start(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
|
||||
if err != nil {
|
||||
t.Fatalf("write IAM config: %v", err)
|
||||
}
|
||||
|
||||
securityToml := filepath.Join(env.dataDir, "security.toml")
|
||||
if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
|
||||
t.Fatalf("write security.toml: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
|
||||
"-master.port", fmt.Sprintf("%d", env.masterPort),
|
||||
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
|
||||
"-volume.port", fmt.Sprintf("%d", env.volumePort),
|
||||
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
|
||||
"-filer.port", fmt.Sprintf("%d", env.filerPort),
|
||||
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
|
||||
"-s3.port", fmt.Sprintf("%d", env.s3Port),
|
||||
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
|
||||
"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
|
||||
"-webdav.port", fmt.Sprintf("%d", env.webdavPort),
|
||||
"-s3.config", iamConfigPath,
|
||||
"-ip", env.bindIP,
|
||||
"-ip.bind", "0.0.0.0",
|
||||
"-dir", env.dataDir,
|
||||
)
|
||||
cmd.Dir = env.dataDir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Env = append(os.Environ(),
|
||||
"AWS_ACCESS_KEY_ID="+env.accessKey,
|
||||
"AWS_SECRET_ACCESS_KEY="+env.secretKey,
|
||||
)
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
cancel()
|
||||
t.Fatalf("start weed mini: %v", err)
|
||||
}
|
||||
env.weedProcess = cmd
|
||||
|
||||
icebergURL := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
|
||||
if !testutil.WaitForService(icebergURL, 30*time.Second) {
|
||||
cancel()
|
||||
cmd.Wait()
|
||||
t.Fatalf("Iceberg REST API did not become ready at %s", icebergURL)
|
||||
}
|
||||
}
|
||||
|
||||
func (env *oauthTestEnv) cleanup(t *testing.T) {
|
||||
t.Helper()
|
||||
if env.weedCancel != nil {
|
||||
env.weedCancel()
|
||||
}
|
||||
if env.weedProcess != nil {
|
||||
env.weedProcess.Wait()
|
||||
}
|
||||
if env.dataDir != "" {
|
||||
os.RemoveAll(env.dataDir)
|
||||
}
|
||||
}
|
||||
|
||||
func (env *oauthTestEnv) icebergURL() string {
|
||||
return fmt.Sprintf("http://%s:%d", env.bindIP, env.icebergPort)
|
||||
}
|
||||
|
||||
// TestOAuthTokenEndpoint tests the /v1/oauth/tokens endpoint directly via HTTP.
|
||||
func TestOAuthTokenEndpoint(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := newOAuthTestEnv(t)
|
||||
defer env.cleanup(t)
|
||||
env.start(t)
|
||||
|
||||
t.Run("valid credentials", func(t *testing.T) {
|
||||
token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
|
||||
if token == "" {
|
||||
t.Fatal("expected non-empty token")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid secret", func(t *testing.T) {
|
||||
resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
|
||||
"grant_type": {"client_credentials"},
|
||||
"client_id": {env.accessKey},
|
||||
"client_secret": {"wrong-secret"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("POST /v1/oauth/tokens: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("expected 401, got %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing grant_type", func(t *testing.T) {
|
||||
resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
|
||||
"client_id": {env.accessKey},
|
||||
"client_secret": {env.secretKey},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("POST /v1/oauth/tokens: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("expected 400, got %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bearer token auth on catalog endpoint", func(t *testing.T) {
|
||||
token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
|
||||
|
||||
// Use the token to call the catalog
|
||||
req, err := http.NewRequest(http.MethodGet, env.icebergURL()+"/v1/namespaces", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("create request: %v", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("GET /v1/namespaces with Bearer: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Auth should pass. We accept 200 (success) or 500 (missing warehouse bucket
|
||||
// is an internal error, not an auth error). Reject 401/403/404/405.
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK, http.StatusInternalServerError:
|
||||
t.Logf("Bearer token auth succeeded, status=%d", resp.StatusCode)
|
||||
default:
|
||||
t.Fatalf("Bearer auth failed unexpectedly: status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestDuckDBOAuthIntegration tests that DuckDB can connect to the Iceberg REST
|
||||
// catalog using the OAuth2 client_credentials flow (CREATE SECRET with client_id
|
||||
// and client_secret). This is the scenario reported in issue #9015.
|
||||
func TestDuckDBOAuthIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("Docker not available, skipping DuckDB OAuth integration test")
|
||||
}
|
||||
|
||||
env := newOAuthTestEnv(t)
|
||||
defer env.cleanup(t)
|
||||
env.start(t)
|
||||
|
||||
// Create a table bucket and namespace so DuckDB has something to query
|
||||
bucketName := "duckdb-oauth-" + randomSuffix()
|
||||
createTableBucketViaShell(t, env, bucketName)
|
||||
|
||||
// Create a namespace via the Iceberg REST API using OAuth token
|
||||
token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
|
||||
createNamespaceWithToken(t, env, token, bucketName, "testns")
|
||||
|
||||
sqlContent := fmt.Sprintf(`
|
||||
INSTALL iceberg;
|
||||
LOAD iceberg;
|
||||
|
||||
CREATE SECRET iceberg_secret (
|
||||
TYPE ICEBERG,
|
||||
ENDPOINT 'http://host.docker.internal:%d',
|
||||
CLIENT_ID '%s',
|
||||
CLIENT_SECRET '%s',
|
||||
SCOPE 's3://%s/'
|
||||
);
|
||||
|
||||
CREATE SECRET s3_secret (
|
||||
TYPE S3,
|
||||
KEY_ID '%s',
|
||||
SECRET '%s',
|
||||
ENDPOINT 'host.docker.internal:%d',
|
||||
URL_STYLE 'path',
|
||||
USE_SSL false,
|
||||
SCOPE 's3://%s/'
|
||||
);
|
||||
|
||||
SELECT 'OAuth token obtained successfully' as status;
|
||||
|
||||
-- Try listing namespaces via the Iceberg catalog
|
||||
SELECT * FROM iceberg_scan('iceberg_secret', ALLOW_MOVED_PATHS => TRUE) LIMIT 0;
|
||||
`, env.icebergPort, env.accessKey, env.secretKey, bucketName,
|
||||
env.accessKey, env.secretKey, env.s3Port, bucketName)
|
||||
|
||||
sqlFile := filepath.Join(env.dataDir, "duckdb_oauth_test.sql")
|
||||
if err := os.WriteFile(sqlFile, []byte(sqlContent), 0644); err != nil {
|
||||
t.Fatalf("write SQL file: %v", err)
|
||||
}
|
||||
|
||||
// Run DuckDB in Docker.
|
||||
// We use a simple test: get a token, create a secret, and verify the
|
||||
// Iceberg extension can communicate with the catalog.
|
||||
// The simpler fallback test just verifies CREATE SECRET succeeds (no 404).
|
||||
fallbackSQL := fmt.Sprintf(`
|
||||
INSTALL iceberg;
|
||||
LOAD iceberg;
|
||||
|
||||
CREATE SECRET (
|
||||
TYPE ICEBERG,
|
||||
ENDPOINT 'http://host.docker.internal:%d',
|
||||
CLIENT_ID '%s',
|
||||
CLIENT_SECRET '%s'
|
||||
);
|
||||
|
||||
SELECT 'DuckDB Iceberg OAuth secret created successfully' as result;
|
||||
`, env.icebergPort, env.accessKey, env.secretKey)
|
||||
|
||||
fallbackFile := filepath.Join(env.dataDir, "duckdb_oauth_fallback.sql")
|
||||
if err := os.WriteFile(fallbackFile, []byte(fallbackSQL), 0644); err != nil {
|
||||
t.Fatalf("write fallback SQL file: %v", err)
|
||||
}
|
||||
|
||||
// Try the full SQL script first (CREATE SECRET + catalog access).
|
||||
// Fall back to the simple CREATE SECRET test if iceberg_scan isn't supported.
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"-v", fmt.Sprintf("%s:/test", env.dataDir),
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"--entrypoint", "duckdb",
|
||||
"duckdb/duckdb:latest",
|
||||
"-init", "/test/duckdb_oauth_test.sql",
|
||||
"-c", "SELECT 1",
|
||||
)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
outputStr := string(output)
|
||||
t.Logf("DuckDB output:\n%s", outputStr)
|
||||
|
||||
if err != nil {
|
||||
if strings.Contains(outputStr, "iceberg extension is not available") ||
|
||||
strings.Contains(outputStr, "Failed to load") {
|
||||
t.Skip("Skipping: Iceberg extension not available in DuckDB Docker image")
|
||||
}
|
||||
// The key check: the old error was "HTTP NotFound_404" on /v1/oauth/tokens.
|
||||
// With our fix, this should no longer happen.
|
||||
if strings.Contains(outputStr, "NotFound_404") && strings.Contains(outputStr, "/v1/oauth/tokens") {
|
||||
t.Fatal("OAuth token endpoint returned 404 - the fix is not working")
|
||||
}
|
||||
// If iceberg_scan failed but CREATE SECRET worked, fall back to simpler test
|
||||
if strings.Contains(outputStr, "OAuth token obtained successfully") {
|
||||
t.Logf("Full SQL had partial success (token obtained), iceberg_scan may not be supported. Continuing.")
|
||||
} else {
|
||||
// Try the fallback script that only tests CREATE SECRET
|
||||
t.Logf("Full SQL failed, trying fallback CREATE SECRET test...")
|
||||
fallbackCmd := exec.Command("docker", "run", "--rm",
|
||||
"-v", fmt.Sprintf("%s:/test", env.dataDir),
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"--entrypoint", "duckdb",
|
||||
"duckdb/duckdb:latest",
|
||||
"-init", "/test/duckdb_oauth_fallback.sql",
|
||||
"-c", "SELECT 1",
|
||||
)
|
||||
fallbackOutput, fallbackErr := fallbackCmd.CombinedOutput()
|
||||
fallbackStr := string(fallbackOutput)
|
||||
t.Logf("DuckDB fallback output:\n%s", fallbackStr)
|
||||
|
||||
if fallbackErr != nil {
|
||||
if strings.Contains(fallbackStr, "NotFound_404") && strings.Contains(fallbackStr, "/v1/oauth/tokens") {
|
||||
t.Fatal("OAuth token endpoint returned 404 - the fix is not working")
|
||||
}
|
||||
t.Fatalf("DuckDB fallback also failed: %v\nOutput: %s", fallbackErr, fallbackStr)
|
||||
}
|
||||
if !strings.Contains(fallbackStr, "DuckDB Iceberg OAuth secret created successfully") {
|
||||
t.Errorf("expected success message in fallback output, got:\n%s", fallbackStr)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(outputStr, "OAuth token obtained successfully") {
|
||||
t.Logf("DuckDB OAuth CREATE SECRET succeeded")
|
||||
} else {
|
||||
t.Errorf("expected OAuth success message in output, got:\n%s", outputStr)
|
||||
}
|
||||
}
|
||||
|
||||
// requestOAuthToken obtains a bearer token from the OAuth endpoint.
|
||||
func requestOAuthToken(t *testing.T, env *oauthTestEnv, accessKey, secretKey string) string {
|
||||
t.Helper()
|
||||
|
||||
resp, err := http.PostForm(env.icebergURL()+"/v1/oauth/tokens", url.Values{
|
||||
"grant_type": {"client_credentials"},
|
||||
"client_id": {accessKey},
|
||||
"client_secret": {secretKey},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("POST /v1/oauth/tokens: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("OAuth token request failed: status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
var tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &tokenResp); err != nil {
|
||||
t.Fatalf("decode token response: %v", err)
|
||||
}
|
||||
|
||||
if tokenResp.AccessToken == "" {
|
||||
t.Fatal("got empty access_token")
|
||||
}
|
||||
if tokenResp.TokenType != "bearer" {
|
||||
t.Errorf("expected token_type=bearer, got %s", tokenResp.TokenType)
|
||||
}
|
||||
return tokenResp.AccessToken
|
||||
}
|
||||
|
||||
// createTableBucketViaShell creates a table bucket using weed shell,
|
||||
// which bypasses S3 auth. This is the same approach used by the Trino tests.
|
||||
func createTableBucketViaShell(t *testing.T, env *oauthTestEnv, bucketName string) {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command(env.weedBinary, "shell",
|
||||
fmt.Sprintf("-master=%s:%d.%d", env.bindIP, env.masterPort, env.masterGrpcPort),
|
||||
)
|
||||
cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, output)
|
||||
}
|
||||
t.Logf("Created table bucket %s", bucketName)
|
||||
}
|
||||
|
||||
// createNamespaceWithToken creates a namespace using a Bearer token.
|
||||
func createNamespaceWithToken(t *testing.T, env *oauthTestEnv, token, bucketName, namespace string) {
|
||||
t.Helper()
|
||||
|
||||
path := fmt.Sprintf("/v1/%s/namespaces", bucketName)
|
||||
body := fmt.Sprintf(`{"namespace":["%s"]}`, namespace)
|
||||
req, err := http.NewRequest(http.MethodPost, env.icebergURL()+path, strings.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("create request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("create namespace: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
|
||||
t.Fatalf("create namespace failed: status=%d body=%s", resp.StatusCode, respBody)
|
||||
}
|
||||
t.Logf("Created namespace %s in bucket %s", namespace, bucketName)
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -16,8 +17,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
// sharedEnv is the single TestEnvironment shared across all tests in this package.
|
||||
@@ -70,6 +69,23 @@ type TestEnvironment struct {
|
||||
dockerAvailable bool
|
||||
}
|
||||
|
||||
// hasDocker checks if Docker is available
|
||||
func hasDocker() bool {
|
||||
cmd := exec.Command("docker", "version")
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
// getFreePort returns an available ephemeral port and its listener
|
||||
func getFreePort() (int, net.Listener, error) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, listener, nil
|
||||
}
|
||||
|
||||
// newTestEnvironmentForMain creates a TestEnvironment without calling t.Fatalf so it
|
||||
// can be used from TestMain (which has no *testing.T).
|
||||
func newTestEnvironmentForMain() (*TestEnvironment, error) {
|
||||
@@ -102,27 +118,95 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) {
|
||||
return nil, fmt.Errorf("create temp dir: %w", err)
|
||||
}
|
||||
|
||||
// Allocate 9 unique ports atomically: s3, iceberg, s3Grpc, master, masterGrpc,
|
||||
// filer, filerGrpc, volume, volumeGrpc
|
||||
ports, err := testutil.AllocatePorts(9)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("allocate ports: %w", err)
|
||||
// Allocate free ephemeral ports for each service
|
||||
var listeners []net.Listener
|
||||
closeListeners := func() {
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
}
|
||||
|
||||
var l net.Listener
|
||||
s3Port, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for S3: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
icebergPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Iceberg: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
s3GrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for S3 gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
masterPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Master: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
masterGrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Master gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
filerPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Filer: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
filerGrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Filer gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
volumePort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Volume: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
volumeGrpcPort, l, err := getFreePort()
|
||||
if err != nil {
|
||||
closeListeners()
|
||||
return nil, fmt.Errorf("get free port for Volume gRPC: %w", err)
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
|
||||
// Release the port reservations so weed mini can bind to them
|
||||
closeListeners()
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
s3Port: ports[0],
|
||||
s3GrpcPort: ports[1],
|
||||
icebergPort: ports[2],
|
||||
masterPort: ports[3],
|
||||
masterGrpcPort: ports[4],
|
||||
filerPort: ports[5],
|
||||
filerGrpcPort: ports[6],
|
||||
volumePort: ports[7],
|
||||
volumeGrpcPort: ports[8],
|
||||
dockerAvailable: testutil.HasDocker(),
|
||||
s3Port: s3Port,
|
||||
s3GrpcPort: s3GrpcPort,
|
||||
icebergPort: icebergPort,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
filerPort: filerPort,
|
||||
filerGrpcPort: filerGrpcPort,
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
dockerAvailable: hasDocker(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -18,7 +19,7 @@ import (
|
||||
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -114,12 +115,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
t.Fatalf("failed to create temp directory: %v", err)
|
||||
}
|
||||
|
||||
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "RisingWave"})
|
||||
env.masterPort = ports[0]
|
||||
env.filerPort = ports[1]
|
||||
env.s3Port = ports[2]
|
||||
env.icebergRestPort = ports[3]
|
||||
env.risingwavePort = ports[4]
|
||||
env.masterPort = mustFreePort(t, "Master")
|
||||
env.filerPort = mustFreePort(t, "Filer")
|
||||
env.s3Port = mustFreePort(t, "S3")
|
||||
env.icebergRestPort = mustFreePort(t, "Iceberg")
|
||||
env.risingwavePort = mustFreePort(t, "RisingWave")
|
||||
|
||||
env.bindIP = testutil.FindBindIP()
|
||||
|
||||
@@ -161,20 +161,62 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
registerMiniProcess(env.masterProcess)
|
||||
|
||||
// Wait for all services to be ready
|
||||
if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
|
||||
if !waitForPort(env.masterPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
|
||||
}
|
||||
if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
|
||||
if !waitForPort(env.filerPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
|
||||
}
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
|
||||
t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
|
||||
if !waitForPort(env.s3Port, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
|
||||
}
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
|
||||
t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
|
||||
if !waitForPort(env.icebergRestPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
|
||||
}
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
minPort := 10000
|
||||
maxPort := 55000 // Ensure port+10000 < 65535
|
||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
port := minPort + r.Intn(maxPort-minPort)
|
||||
|
||||
// Check http port
|
||||
ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ln.Close()
|
||||
|
||||
// Check grpc port (weed mini uses port+10000)
|
||||
ln2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ln2.Close()
|
||||
|
||||
return port
|
||||
}
|
||||
t.Fatalf("failed to find a free port < %d for %s after 1000 attempts", maxPort, name)
|
||||
return 0
|
||||
}
|
||||
|
||||
func waitForPort(port int, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
return true
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) StartRisingWave(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
@@ -211,7 +253,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) {
|
||||
}
|
||||
|
||||
// Wait for RisingWave port to be open on host
|
||||
if !testutil.WaitForPort(env.risingwavePort, 120*time.Second) {
|
||||
if !waitForPort(env.risingwavePort, 120*time.Second) {
|
||||
t.Fatalf("timed out waiting for RisingWave port %d to be open", env.risingwavePort)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -16,7 +17,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
)
|
||||
|
||||
@@ -90,11 +91,10 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
t.Fatalf("failed to create temp directory: %v", err)
|
||||
}
|
||||
|
||||
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"})
|
||||
env.masterPort = ports[0]
|
||||
env.filerPort = ports[1]
|
||||
env.s3Port = ports[2]
|
||||
env.icebergRestPort = ports[3]
|
||||
env.masterPort = mustFreePort(t, "Master")
|
||||
env.filerPort = mustFreePort(t, "Filer")
|
||||
env.s3Port = mustFreePort(t, "S3")
|
||||
env.icebergRestPort = mustFreePort(t, "Iceberg")
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
@@ -127,20 +127,58 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
registerMiniProcess(env.masterProcess)
|
||||
|
||||
// Wait for all services to be ready
|
||||
if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
|
||||
if !waitForPort(env.masterPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
|
||||
}
|
||||
if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
|
||||
if !waitForPort(env.filerPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
|
||||
}
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
|
||||
t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
|
||||
if !waitForPort(env.s3Port, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
|
||||
}
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
|
||||
t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
|
||||
if !waitForPort(env.icebergRestPort, 15*time.Second) {
|
||||
t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
|
||||
}
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
|
||||
for i := 0; i < 200; i++ {
|
||||
port := 20000 + rand.Intn(30000)
|
||||
listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
listener.Close()
|
||||
grpcPort := port + 10000
|
||||
if grpcPort > 65535 {
|
||||
continue
|
||||
}
|
||||
grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
grpcListener.Close()
|
||||
return port
|
||||
}
|
||||
t.Fatalf("failed to get free port for %s", name)
|
||||
return 0
|
||||
}
|
||||
|
||||
func waitForPort(port int, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
return true
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) writeSparkConfig(t *testing.T, catalogBucket string) string {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -277,80 +277,3 @@ print(f"Count at snapshot: {count}")
|
||||
|
||||
t.Logf(">>> Time travel test passed")
|
||||
}
|
||||
|
||||
// TestSparkMultiLevelNamespace tests that multi-level namespaces produce correct
|
||||
// S3 paths (dot-separated) so that Spark can read back the data it writes.
|
||||
// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
|
||||
func TestSparkMultiLevelNamespace(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env, _, _ := setupSparkTestEnv(t)
|
||||
|
||||
// Use a two-level namespace like "analytics.daily"
|
||||
nsLevel1 := "analytics_" + randomString(4)
|
||||
nsLevel2 := "daily_" + randomString(4)
|
||||
multiNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
|
||||
tableName := "events_" + randomString(4)
|
||||
|
||||
// Create multi-level namespace
|
||||
t.Logf(">>> Creating multi-level namespace: %s", multiNs)
|
||||
createNsSQL := fmt.Sprintf(`
|
||||
spark.sql("CREATE NAMESPACE iceberg.%s")
|
||||
print("Namespace created")
|
||||
`, multiNs)
|
||||
output := runSparkPySQL(t, env.sparkContainer, createNsSQL, env.icebergRestPort, env.s3Port)
|
||||
if !strings.Contains(output, "Namespace created") {
|
||||
t.Fatalf("multi-level namespace creation failed, output: %s", output)
|
||||
}
|
||||
|
||||
// Create table under multi-level namespace
|
||||
t.Logf(">>> Creating table under multi-level namespace")
|
||||
createTableSQL := fmt.Sprintf(`
|
||||
spark.sql("""
|
||||
CREATE TABLE iceberg.%s.%s (
|
||||
id INT,
|
||||
event STRING,
|
||||
ts TIMESTAMP
|
||||
)
|
||||
USING iceberg
|
||||
""")
|
||||
print("Table created")
|
||||
`, multiNs, tableName)
|
||||
output = runSparkPySQL(t, env.sparkContainer, createTableSQL, env.icebergRestPort, env.s3Port)
|
||||
if !strings.Contains(output, "Table created") {
|
||||
t.Fatalf("table creation under multi-level namespace failed, output: %s", output)
|
||||
}
|
||||
|
||||
// Insert data
|
||||
t.Logf(">>> Inserting data into multi-level namespace table")
|
||||
insertSQL := fmt.Sprintf(`
|
||||
spark.sql("""
|
||||
INSERT INTO iceberg.%s.%s VALUES
|
||||
(1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
|
||||
(2, 'view', TIMESTAMP '2025-01-01 01:00:00'),
|
||||
(3, 'click', TIMESTAMP '2025-01-02 00:00:00')
|
||||
""")
|
||||
print("Data inserted")
|
||||
`, multiNs, tableName)
|
||||
output = runSparkPySQL(t, env.sparkContainer, insertSQL, env.icebergRestPort, env.s3Port)
|
||||
if !strings.Contains(output, "Data inserted") {
|
||||
t.Fatalf("data insertion failed, output: %s", output)
|
||||
}
|
||||
|
||||
// Query data back — this is the key test: if the namespace path separator
|
||||
// was wrong (\x1F instead of "."), Spark would not find the data files.
|
||||
t.Logf(">>> Querying data from multi-level namespace table")
|
||||
querySQL := fmt.Sprintf(`
|
||||
result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
|
||||
count = result.collect()[0]['count']
|
||||
print(f"Row count: {count}")
|
||||
`, multiNs, tableName)
|
||||
output = runSparkPySQL(t, env.sparkContainer, querySQL, env.icebergRestPort, env.s3Port)
|
||||
if !strings.Contains(output, "Row count: 3") {
|
||||
t.Errorf("expected row count 3 from multi-level namespace table, got output: %s", output)
|
||||
}
|
||||
|
||||
t.Logf(">>> Multi-level namespace test passed")
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
)
|
||||
|
||||
type TestEnvironment struct {
|
||||
@@ -40,6 +40,7 @@ type TestEnvironment struct {
|
||||
dockerAvailable bool
|
||||
accessKey string
|
||||
secretKey string
|
||||
closers []io.Closer
|
||||
}
|
||||
|
||||
func TestTrinoIcebergCatalog(t *testing.T) {
|
||||
@@ -81,101 +82,6 @@ func TestTrinoIcebergCatalog(t *testing.T) {
|
||||
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SHOW TABLES FROM iceberg.%s", schemaName))
|
||||
}
|
||||
|
||||
// TestTrinoMultiLevelNamespace tests that multi-level namespaces (dot-separated)
|
||||
// produce correct S3 paths so Trino can read back data it writes.
|
||||
// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
|
||||
func TestTrinoMultiLevelNamespace(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
if !env.dockerAvailable {
|
||||
t.Skip("Docker not available, skipping Trino integration test")
|
||||
}
|
||||
|
||||
t.Logf(">>> Starting SeaweedFS...")
|
||||
env.StartSeaweedFS(t)
|
||||
|
||||
tableBucket := "iceberg-tables"
|
||||
createTableBucket(t, env, tableBucket)
|
||||
|
||||
configDir := env.writeTrinoConfig(t, tableBucket, withNestedNamespace())
|
||||
env.startTrinoContainer(t, configDir)
|
||||
waitForTrino(t, env.trinoContainer, 60*time.Second)
|
||||
|
||||
// Use a two-level namespace: "analytics.daily"
|
||||
nsLevel1 := "analytics_" + randomString(4)
|
||||
nsLevel2 := "daily_" + randomString(4)
|
||||
flatNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
|
||||
// Trino uses double-quoted schema names for multi-level namespaces
|
||||
multiNs := fmt.Sprintf(`"%s"`, flatNs)
|
||||
tableName := "events_" + randomString(4)
|
||||
|
||||
// Create multi-level namespace (schema)
|
||||
t.Logf(">>> Creating multi-level schema: %s", flatNs)
|
||||
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", multiNs))
|
||||
|
||||
// Verify the schema shows up
|
||||
output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
|
||||
if !strings.Contains(output, flatNs) {
|
||||
t.Fatalf("Expected schema %s in output:\n%s", flatNs, output)
|
||||
}
|
||||
|
||||
// Create table with explicit location to avoid non-empty location conflict.
|
||||
// The location uses the dot-separated namespace — if #8959 regresses
|
||||
// (unit separator instead of dot), data would be written to the wrong path.
|
||||
tableLocation := fmt.Sprintf("s3://%s/%s/%s_%s", tableBucket, flatNs, tableName, randomString(6))
|
||||
t.Logf(">>> Creating table at location: %s", tableLocation)
|
||||
createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s (
|
||||
id INTEGER,
|
||||
event VARCHAR,
|
||||
ts TIMESTAMP(6)
|
||||
) WITH (
|
||||
format = 'PARQUET',
|
||||
location = '%s'
|
||||
)`, multiNs, tableName, tableLocation)
|
||||
runTrinoSQLAllowExists(t, env.trinoContainer, createSQL)
|
||||
|
||||
// Insert data
|
||||
t.Logf(">>> Inserting data into multi-level namespace table")
|
||||
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`
|
||||
INSERT INTO iceberg.%s.%s VALUES
|
||||
(1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
|
||||
(2, 'view', TIMESTAMP '2025-01-01 01:00:00'),
|
||||
(3, 'click', TIMESTAMP '2025-01-02 00:00:00')
|
||||
`, multiNs, tableName))
|
||||
|
||||
// Query data back — if the namespace path separator were wrong (\x1F
|
||||
// instead of "."), the metadata location would point to a non-existent
|
||||
// S3 path and this query would fail.
|
||||
t.Logf(">>> Querying data from multi-level namespace table")
|
||||
countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
|
||||
"SELECT count(*) FROM iceberg.%s.%s", multiNs, tableName))
|
||||
rowCount := mustParseCSVInt64(t, countOutput)
|
||||
if rowCount != 3 {
|
||||
t.Fatalf("expected row count 3, got %d", rowCount)
|
||||
}
|
||||
|
||||
// Verify the S3 file path contains the dot-separated namespace, not \x1F.
|
||||
filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
|
||||
`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, multiNs, tableName))
|
||||
filePath := strings.TrimSpace(filesOutput)
|
||||
if filePath == "" {
|
||||
t.Fatalf("expected at least one data file, got empty output")
|
||||
}
|
||||
if !strings.Contains(filePath, flatNs+"/") {
|
||||
t.Errorf("expected file path to contain dot-separated namespace %q, got: %s", flatNs, filePath)
|
||||
}
|
||||
if strings.Contains(filePath, "\x1F") {
|
||||
t.Errorf("file path contains unit separator (\\x1F), expected dot separator: %s", filePath)
|
||||
}
|
||||
|
||||
t.Logf(">>> Trino multi-level namespace test passed")
|
||||
}
|
||||
|
||||
func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Helper()
|
||||
|
||||
@@ -213,25 +119,20 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
|
||||
ports := testutil.MustAllocatePorts(t, 9)
|
||||
|
||||
env := &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
masterPort: ports[0],
|
||||
masterGrpcPort: ports[1],
|
||||
volumePort: ports[2],
|
||||
volumeGrpcPort: ports[3],
|
||||
filerPort: ports[4],
|
||||
filerGrpcPort: ports[5],
|
||||
s3Port: ports[6],
|
||||
s3GrpcPort: ports[7],
|
||||
icebergPort: ports[8],
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
closers: []io.Closer{},
|
||||
}
|
||||
|
||||
env.masterPort, env.masterGrpcPort = env.mustFreePortPair("Master")
|
||||
env.volumePort, env.volumeGrpcPort = env.mustFreePortPair("Volume")
|
||||
env.filerPort, env.filerGrpcPort = env.mustFreePortPair("Filer")
|
||||
env.s3Port, env.s3GrpcPort = env.mustFreePortPair("S3")
|
||||
env.icebergPort = env.mustFreePort("Iceberg")
|
||||
|
||||
env.dockerAvailable = hasDocker()
|
||||
env.accessKey = "AKIAIOSFODNN7EXAMPLE"
|
||||
env.secretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
@@ -239,6 +140,47 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
return env
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) mustFreePort(name string) int {
|
||||
port, closer, err := getFreePort()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to get free port for %s: %v", name, err))
|
||||
}
|
||||
env.closers = append(env.closers, closer)
|
||||
return port
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) mustFreePortPair(name string) (int, int) {
|
||||
httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to get free port pair for %s: %v", name, err))
|
||||
}
|
||||
env.closers = append(env.closers, httpCloser, grpcCloser)
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
|
||||
port, closer, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for %s: %v", name, err)
|
||||
}
|
||||
closer.Close()
|
||||
return port
|
||||
}
|
||||
|
||||
func mustFreePortPair(t *testing.T, name string) (int, int) {
|
||||
t.Helper()
|
||||
|
||||
httpPort, httpCloser, grpcPort, grpcCloser, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
|
||||
}
|
||||
httpCloser.Close()
|
||||
grpcCloser.Close()
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
@@ -256,6 +198,12 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
// Close all port listeners right before starting the weed process
|
||||
for _, closer := range env.closers {
|
||||
closer.Close()
|
||||
}
|
||||
env.closers = nil
|
||||
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
|
||||
"-master.port", fmt.Sprintf("%d", env.masterPort),
|
||||
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
|
||||
@@ -384,32 +332,18 @@ func testIcebergRestAPI(t *testing.T, env *TestEnvironment) {
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string, opts ...func(*trinoConfigOptions)) string {
|
||||
func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string) string {
|
||||
t.Helper()
|
||||
|
||||
o := trinoConfigOptions{}
|
||||
for _, fn := range opts {
|
||||
fn(&o)
|
||||
}
|
||||
|
||||
dirName := "trino"
|
||||
if o.nestedNamespace {
|
||||
dirName = "trino-nested"
|
||||
}
|
||||
configDir := filepath.Join(env.dataDir, dirName)
|
||||
configDir := filepath.Join(env.dataDir, "trino")
|
||||
if err := os.MkdirAll(configDir, 0755); err != nil {
|
||||
t.Fatalf("Failed to create Trino config dir: %v", err)
|
||||
}
|
||||
|
||||
nestedLine := ""
|
||||
if o.nestedNamespace {
|
||||
nestedLine = "\niceberg.rest-catalog.nested-namespace-enabled=true"
|
||||
}
|
||||
|
||||
config := fmt.Sprintf(`connector.name=iceberg
|
||||
iceberg.catalog.type=rest
|
||||
iceberg.rest-catalog.uri=http://host.docker.internal:%d
|
||||
iceberg.rest-catalog.warehouse=s3://%s%s
|
||||
iceberg.rest-catalog.warehouse=s3://%s
|
||||
iceberg.file-format=PARQUET
|
||||
iceberg.unique-table-location=true
|
||||
|
||||
@@ -424,7 +358,7 @@ s3.region=us-west-2
|
||||
|
||||
# REST catalog authentication
|
||||
iceberg.rest-catalog.security=SIGV4
|
||||
`, env.icebergPort, warehouseBucket, nestedLine, env.s3Port, env.accessKey, env.secretKey)
|
||||
`, env.icebergPort, warehouseBucket, env.s3Port, env.accessKey, env.secretKey)
|
||||
|
||||
if err := os.WriteFile(filepath.Join(configDir, "iceberg.properties"), []byte(config), 0644); err != nil {
|
||||
t.Fatalf("Failed to write Trino config: %v", err)
|
||||
@@ -433,14 +367,6 @@ iceberg.rest-catalog.security=SIGV4
|
||||
return configDir
|
||||
}
|
||||
|
||||
type trinoConfigOptions struct {
|
||||
nestedNamespace bool
|
||||
}
|
||||
|
||||
func withNestedNamespace() func(*trinoConfigOptions) {
|
||||
return func(o *trinoConfigOptions) { o.nestedNamespace = true }
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -588,6 +514,29 @@ func hasDocker() bool {
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
func findAvailablePortPair() (int, io.Closer, int, io.Closer, error) {
|
||||
httpPort, httpCloser, err := getFreePort()
|
||||
if err != nil {
|
||||
return 0, nil, 0, nil, err
|
||||
}
|
||||
grpcPort, grpcCloser, err := getFreePort()
|
||||
if err != nil {
|
||||
httpCloser.Close()
|
||||
return 0, nil, 0, nil, err
|
||||
}
|
||||
return httpPort, httpCloser, grpcPort, grpcCloser, nil
|
||||
}
|
||||
|
||||
func getFreePort() (int, io.Closer, error) {
|
||||
listener, err := net.Listen("tcp", "0.0.0.0:0")
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, listener, nil
|
||||
}
|
||||
|
||||
func randomString(length int) string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
b := make([]byte, length)
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sts"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
@@ -115,13 +115,10 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
// Allocate all ports in a single batch to prevent the OS from recycling
|
||||
// a released port, which can cause two services to get the same port.
|
||||
ports := testutil.MustAllocatePorts(t, 8)
|
||||
masterPort, masterGrpcPort := ports[0], ports[1]
|
||||
volumePort, volumeGrpcPort := ports[2], ports[3]
|
||||
filerPort, filerGrpcPort := ports[4], ports[5]
|
||||
s3Port, s3GrpcPort := ports[6], ports[7]
|
||||
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@@ -39,7 +40,6 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/command"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -92,7 +92,7 @@ func TestMain(m *testing.M) {
|
||||
}
|
||||
|
||||
func startCluster(testDir string, extraArgs []string) (*testCluster, error) {
|
||||
ports, err := testutil.AllocatePorts(10)
|
||||
ports, err := findPorts(10)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -201,6 +201,26 @@ func (c *testCluster) filerConn(t *testing.T) (*grpc.ClientConn, filer_pb.Seawee
|
||||
return conn, filer_pb.NewSeaweedFilerClient(conn)
|
||||
}
|
||||
|
||||
func findPorts(n int) ([]int, error) {
|
||||
ls := make([]*net.TCPListener, n)
|
||||
ps := make([]int, n)
|
||||
for i := 0; i < n; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
for j := 0; j < i; j++ {
|
||||
ls[j].Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
ls[i] = l.(*net.TCPListener)
|
||||
ps[i] = ls[i].Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
for _, l := range ls {
|
||||
l.Close()
|
||||
}
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
func waitReady(endpoint string, timeout time.Duration) error {
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -89,14 +89,11 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
// Allocate all ports in a single batch to prevent the OS from recycling
|
||||
// a released port, which can cause two services to get the same port.
|
||||
ports := testutil.MustAllocatePorts(t, 10)
|
||||
masterPort, masterGrpcPort := ports[0], ports[1]
|
||||
volumePort, volumeGrpcPort := ports[2], ports[3]
|
||||
filerPort, filerGrpcPort := ports[4], ports[5]
|
||||
s3Port, s3GrpcPort := ports[6], ports[7]
|
||||
polarisPort, polarisAdminPort := ports[8], ports[9]
|
||||
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
|
||||
polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
)
|
||||
|
||||
// TestEnvironment mirrors the one in trino_catalog_test.go but simplified
|
||||
@@ -93,13 +93,10 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
// Allocate all ports in a single batch to prevent the OS from recycling
|
||||
// a released port, which can cause two services to get the same port.
|
||||
ports := testutil.MustAllocatePorts(t, 8)
|
||||
masterPort, masterGrpcPort := ports[0], ports[1]
|
||||
volumePort, volumeGrpcPort := ports[2], ports[3]
|
||||
filerPort, filerGrpcPort := ports[4], ports[5]
|
||||
s3Port, s3GrpcPort := ports[6], ports[7]
|
||||
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3") // Changed to use testutil.MustFreePortPair
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
|
||||
@@ -3,6 +3,7 @@ package s3tables
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -17,7 +18,6 @@ import (
|
||||
|
||||
"flag"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/command"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
@@ -590,12 +590,40 @@ func testTargetOperations(t *testing.T, client *S3TablesClient) {
|
||||
|
||||
// Helper functions
|
||||
|
||||
// findAvailablePorts finds n available ports by binding to port 0 multiple times
|
||||
// It keeps the listeners open until all ports are found to ensure uniqueness
|
||||
func findAvailablePorts(n int) ([]int, error) {
|
||||
listeners := make([]*net.TCPListener, n)
|
||||
ports := make([]int, n)
|
||||
|
||||
// Open all listeners to ensure we get unique ports
|
||||
for i := 0; i < n; i++ {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
// Close valid listeners before returning error
|
||||
for j := 0; j < i; j++ {
|
||||
listeners[j].Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
listeners[i] = listener.(*net.TCPListener)
|
||||
ports[i] = listeners[i].Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
// Close all listeners
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
|
||||
return ports, nil
|
||||
}
|
||||
|
||||
// startMiniClusterInDir starts a weed mini instance using testDir as the data
|
||||
// directory. It does not require a *testing.T so it can be called from TestMain.
|
||||
// extraArgs are appended to the default mini command flags.
|
||||
func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) {
|
||||
// We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2)
|
||||
ports, err := testutil.AllocatePorts(10)
|
||||
ports, err := findAvailablePorts(10)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find available ports: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func HasDocker() bool {
|
||||
cmd := exec.Command("docker", "version")
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
func MustFreePortPair(t *testing.T, name string) (int, int) {
|
||||
httpPort, grpcPort, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
|
||||
}
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func findAvailablePortPair() (int, int, error) {
|
||||
httpPort, err := GetFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
grpcPort, err := GetFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return httpPort, grpcPort, nil
|
||||
}
|
||||
|
||||
func GetFreePort() (int, error) {
|
||||
listener, err := net.Listen("tcp", "0.0.0.0:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer listener.Close()
|
||||
return listener.Addr().(*net.TCPAddr).Port, nil
|
||||
}
|
||||
|
||||
func WaitForService(url string, timeout time.Duration) bool {
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-ticker.C:
|
||||
resp, err := client.Get(url)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func FindBindIP() string {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "127.0.0.1"
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
ipNet, ok := addr.(*net.IPNet)
|
||||
if !ok || ipNet.IP == nil {
|
||||
continue
|
||||
}
|
||||
ip := ipNet.IP.To4()
|
||||
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
return "127.0.0.1"
|
||||
}
|
||||
|
||||
func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
|
||||
iamConfigPath := filepath.Join(dir, "iam_config.json")
|
||||
iamConfig := fmt.Sprintf(`{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "%s",
|
||||
"secretKey": "%s"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, accessKey, secretKey)
|
||||
|
||||
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return iamConfigPath, nil
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const SeaweedMiniStartupTimeout = 45 * time.Second
|
||||
|
||||
func HasDocker() bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "docker", "version")
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
func FindBindIP() string {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "127.0.0.1"
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
ipNet, ok := addr.(*net.IPNet)
|
||||
if !ok || ipNet.IP == nil {
|
||||
continue
|
||||
}
|
||||
ip := ipNet.IP.To4()
|
||||
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
return "127.0.0.1"
|
||||
}
|
||||
|
||||
func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
|
||||
iamConfigPath := filepath.Join(dir, "iam_config.json")
|
||||
iamConfig := fmt.Sprintf(`{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "%s",
|
||||
"secretKey": "%s"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}`, accessKey, secretKey)
|
||||
|
||||
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return iamConfigPath, nil
|
||||
}
|
||||
|
||||
func WaitForService(url string, timeout time.Duration) bool {
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-ticker.C:
|
||||
resp, err := client.Get(url)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func WaitForPort(port int, timeout time.Duration) bool {
|
||||
deadline := time.Now().Add(timeout)
|
||||
address := fmt.Sprintf("127.0.0.1:%d", port)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
|
||||
if err == nil {
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
// Package testutil provides shared test utilities for SeaweedFS integration tests.
|
||||
package testutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// GrpcPortOffset is the offset weed mini uses to derive gRPC ports from HTTP ports.
|
||||
const GrpcPortOffset = 10000
|
||||
|
||||
// AllocatePorts allocates count unique free ports atomically.
|
||||
// All listeners are held open until every port is obtained, preventing
|
||||
// the OS from recycling a port between successive allocations.
|
||||
func AllocatePorts(count int) ([]int, error) {
|
||||
listeners := make([]net.Listener, 0, count)
|
||||
ports := make([]int, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
for _, ll := range listeners {
|
||||
_ = ll.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
for _, l := range listeners {
|
||||
_ = l.Close()
|
||||
}
|
||||
return ports, nil
|
||||
}
|
||||
|
||||
// MustAllocatePorts is a testing wrapper for AllocatePorts.
|
||||
func MustAllocatePorts(t *testing.T, count int) []int {
|
||||
t.Helper()
|
||||
ports, err := AllocatePorts(count)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to allocate %d free ports: %v", count, err)
|
||||
}
|
||||
return ports
|
||||
}
|
||||
|
||||
// AllocateMiniPorts allocates n free ports where each port and its gRPC
|
||||
// counterpart (port + GrpcPortOffset) are available and don't collide
|
||||
// with any other allocated port or its gRPC counterpart. All listeners
|
||||
// are held open until the entire batch is allocated, preventing the OS
|
||||
// from recycling ports between allocations. Use this when ports will be
|
||||
// passed to weed mini without explicit gRPC port flags, so mini will
|
||||
// derive gRPC ports as HTTP + 10000.
|
||||
func AllocateMiniPorts(count int) ([]int, error) {
|
||||
const (
|
||||
minPort = 10000
|
||||
maxPort = 55000
|
||||
)
|
||||
reserved := make(map[int]bool)
|
||||
ports := make([]int, 0, count)
|
||||
var listeners []net.Listener
|
||||
defer func() {
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
for idx := 0; idx < count; idx++ {
|
||||
found := false
|
||||
for i := 0; i < 1000; i++ {
|
||||
port := minPort + rand.Intn(maxPort-minPort)
|
||||
grpcPort := port + GrpcPortOffset
|
||||
|
||||
if reserved[port] || reserved[grpcPort] {
|
||||
continue
|
||||
}
|
||||
|
||||
l1, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
l2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
|
||||
if err != nil {
|
||||
l1.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
listeners = append(listeners, l1, l2)
|
||||
reserved[port] = true
|
||||
reserved[grpcPort] = true
|
||||
ports = append(ports, port)
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("failed to allocate mini port %d of %d", idx+1, count)
|
||||
}
|
||||
}
|
||||
|
||||
return ports, nil
|
||||
}
|
||||
|
||||
// MustFreeMiniPorts allocates n ports suitable for weed mini, ensuring
|
||||
// each port's gRPC offset (port + 10000) doesn't collide with any other
|
||||
// allocated port. names is used only for error messages.
|
||||
func MustFreeMiniPorts(t *testing.T, names []string) []int {
|
||||
t.Helper()
|
||||
ports, err := AllocateMiniPorts(len(names))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to allocate mini ports for %v: %v", names, err)
|
||||
}
|
||||
return ports
|
||||
}
|
||||
|
||||
// MustFreeMiniPort allocates a single weed mini port.
|
||||
func MustFreeMiniPort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
return MustFreeMiniPorts(t, []string{name})[0]
|
||||
}
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
)
|
||||
|
||||
@@ -85,14 +84,12 @@ func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
|
||||
t.Fatalf("write security config: %v", err)
|
||||
}
|
||||
|
||||
miniPorts, err := testutil.AllocateMiniPorts(1)
|
||||
masterPort, masterGrpcPort, err := allocateMasterPortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("allocate master port pair: %v", err)
|
||||
}
|
||||
masterPort := miniPorts[0]
|
||||
masterGrpcPort := masterPort + testutil.GrpcPortOffset
|
||||
|
||||
ports, err := testutil.AllocatePorts(3)
|
||||
ports, err := allocatePorts(3)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate ports: %v", err)
|
||||
}
|
||||
@@ -272,6 +269,44 @@ func stopProcess(cmd *exec.Cmd) {
|
||||
}
|
||||
}
|
||||
|
||||
func allocatePorts(count int) ([]int, error) {
|
||||
listeners := make([]net.Listener, 0, count)
|
||||
ports := make([]int, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
for _, ll := range listeners {
|
||||
_ = ll.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
listeners = append(listeners, l)
|
||||
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
for _, l := range listeners {
|
||||
_ = l.Close()
|
||||
}
|
||||
return ports, nil
|
||||
}
|
||||
|
||||
func allocateMasterPortPair() (int, int, error) {
|
||||
for masterPort := 10000; masterPort <= 55535; masterPort++ {
|
||||
masterGrpcPort := masterPort + 10000
|
||||
l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterGrpcPort)))
|
||||
if err != nil {
|
||||
_ = l1.Close()
|
||||
continue
|
||||
}
|
||||
_ = l2.Close()
|
||||
_ = l1.Close()
|
||||
return masterPort, masterGrpcPort, nil
|
||||
}
|
||||
return 0, 0, errors.New("unable to find available master port pair")
|
||||
}
|
||||
|
||||
func newWorkDir() (dir string, keepLogs bool, err error) {
|
||||
keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1"
|
||||
|
||||
@@ -1,371 +0,0 @@
|
||||
package framework
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
)
|
||||
|
||||
// MixedVolumeCluster wraps a Go master + a mix of Go and Rust volume servers
|
||||
// for integration testing. Go servers occupy indices [0, goCount) and Rust
|
||||
// servers occupy indices [goCount, goCount+rustCount).
|
||||
type MixedVolumeCluster struct {
|
||||
testingTB testing.TB
|
||||
profile matrix.Profile
|
||||
|
||||
weedBinary string // Go weed binary (master + Go volume servers)
|
||||
rustVolumeBinary string // Rust volume binary
|
||||
|
||||
baseDir string
|
||||
configDir string
|
||||
logsDir string
|
||||
keepLogs bool
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
|
||||
volumePorts []int
|
||||
volumeGrpcPorts []int
|
||||
volumePubPorts []int
|
||||
isRust []bool // which servers are Rust
|
||||
|
||||
masterCmd *exec.Cmd
|
||||
volumeCmds []*exec.Cmd
|
||||
|
||||
cleanupOnce sync.Once
|
||||
}
|
||||
|
||||
// StartMixedVolumeCluster starts a cluster with 1 Go master, goCount Go volume
|
||||
// servers, and rustCount Rust volume servers. Go servers come first in the index.
|
||||
func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rustCount int) *MixedVolumeCluster {
|
||||
t.Helper()
|
||||
|
||||
if goCount < 0 || rustCount < 0 {
|
||||
t.Fatalf("goCount and rustCount must be non-negative, got go=%d rust=%d", goCount, rustCount)
|
||||
}
|
||||
total := goCount + rustCount
|
||||
if total < 2 {
|
||||
t.Fatalf("need at least 2 volume servers, got %d", total)
|
||||
}
|
||||
|
||||
weedBinary, err := FindOrBuildWeedBinary()
|
||||
if err != nil {
|
||||
t.Fatalf("resolve weed binary: %v", err)
|
||||
}
|
||||
|
||||
// Only build the Rust binary when Rust servers are requested.
|
||||
var rustBinary string
|
||||
if rustCount > 0 {
|
||||
rustBinary, err = FindOrBuildRustBinary()
|
||||
if err != nil {
|
||||
t.Skipf("skipping mixed cluster test: rust binary unavailable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
baseDir, keepLogs, err := newWorkDir()
|
||||
if err != nil {
|
||||
t.Fatalf("create temp test directory: %v", err)
|
||||
}
|
||||
|
||||
configDir := filepath.Join(baseDir, "config")
|
||||
logsDir := filepath.Join(baseDir, "logs")
|
||||
masterDataDir := filepath.Join(baseDir, "master")
|
||||
|
||||
dirs := []string{configDir, logsDir, masterDataDir}
|
||||
for i := 0; i < total; i++ {
|
||||
dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
|
||||
t.Fatalf("create %s: %v", dir, mkErr)
|
||||
}
|
||||
}
|
||||
|
||||
if err = writeSecurityConfig(configDir, profile); err != nil {
|
||||
t.Fatalf("write security config: %v", err)
|
||||
}
|
||||
|
||||
miniPorts, err := testutil.AllocateMiniPorts(1)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate master port pair: %v", err)
|
||||
}
|
||||
masterPort := miniPorts[0]
|
||||
masterGrpcPort := masterPort + testutil.GrpcPortOffset
|
||||
|
||||
// 2 ports per server (admin, grpc); add 1 more when public port is split out.
|
||||
portsPerServer := 2
|
||||
if profile.SplitPublicPort {
|
||||
portsPerServer = 3
|
||||
}
|
||||
ports, err := testutil.AllocatePorts(total * portsPerServer)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate volume ports: %v", err)
|
||||
}
|
||||
|
||||
isRust := make([]bool, total)
|
||||
for i := goCount; i < total; i++ {
|
||||
isRust[i] = true
|
||||
}
|
||||
|
||||
c := &MixedVolumeCluster{
|
||||
testingTB: t,
|
||||
profile: profile,
|
||||
weedBinary: weedBinary,
|
||||
rustVolumeBinary: rustBinary,
|
||||
baseDir: baseDir,
|
||||
configDir: configDir,
|
||||
logsDir: logsDir,
|
||||
keepLogs: keepLogs,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
volumePorts: make([]int, total),
|
||||
volumeGrpcPorts: make([]int, total),
|
||||
volumePubPorts: make([]int, total),
|
||||
isRust: isRust,
|
||||
volumeCmds: make([]*exec.Cmd, total),
|
||||
}
|
||||
|
||||
for i := 0; i < total; i++ {
|
||||
baseIdx := i * portsPerServer
|
||||
c.volumePorts[i] = ports[baseIdx]
|
||||
c.volumeGrpcPorts[i] = ports[baseIdx+1]
|
||||
if profile.SplitPublicPort {
|
||||
c.volumePubPorts[i] = ports[baseIdx+2]
|
||||
} else {
|
||||
c.volumePubPorts[i] = c.volumePorts[i] // reuse admin port
|
||||
}
|
||||
}
|
||||
|
||||
// Start master
|
||||
if err = c.startMaster(masterDataDir); err != nil {
|
||||
c.Stop()
|
||||
t.Fatalf("start master: %v", err)
|
||||
}
|
||||
helper := &Cluster{logsDir: logsDir}
|
||||
if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
|
||||
masterLog := helper.tailLog("master.log")
|
||||
c.Stop()
|
||||
t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
|
||||
}
|
||||
|
||||
// Start volume servers
|
||||
for i := 0; i < total; i++ {
|
||||
volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
|
||||
if isRust[i] {
|
||||
err = c.startRustVolume(i, volumeDataDir)
|
||||
} else {
|
||||
err = c.startGoVolume(i, volumeDataDir)
|
||||
}
|
||||
if err != nil {
|
||||
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
|
||||
c.Stop()
|
||||
t.Fatalf("start volume server %d (rust=%v): %v\nlog tail:\n%s", i, isRust[i], err, logTail)
|
||||
}
|
||||
|
||||
// Rust uses /healthz, Go uses /status
|
||||
healthURL := c.VolumeAdminURL(i) + "/status"
|
||||
if isRust[i] {
|
||||
healthURL = c.VolumeAdminURL(i) + "/healthz"
|
||||
}
|
||||
if err = helper.waitForHTTP(healthURL); err != nil {
|
||||
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
|
||||
c.Stop()
|
||||
t.Fatalf("wait for volume server %d readiness: %v\nlog tail:\n%s", i, err, logTail)
|
||||
}
|
||||
if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
|
||||
logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
|
||||
c.Stop()
|
||||
t.Fatalf("wait for volume server %d grpc readiness: %v\nlog tail:\n%s", i, err, logTail)
|
||||
}
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
c.Stop()
|
||||
})
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) Stop() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.cleanupOnce.Do(func() {
|
||||
for i := len(c.volumeCmds) - 1; i >= 0; i-- {
|
||||
stopProcess(c.volumeCmds[i])
|
||||
}
|
||||
stopProcess(c.masterCmd)
|
||||
if !c.keepLogs && !c.testingTB.Failed() {
|
||||
_ = os.RemoveAll(c.baseDir)
|
||||
} else if c.baseDir != "" {
|
||||
c.testingTB.Logf("mixed volume server integration logs kept at %s", c.baseDir)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) startMaster(dataDir string) error {
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-config_dir=" + c.configDir,
|
||||
"master",
|
||||
"-ip=127.0.0.1",
|
||||
"-port=" + strconv.Itoa(c.masterPort),
|
||||
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
|
||||
"-mdir=" + dataDir,
|
||||
"-peers=none",
|
||||
"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
|
||||
"-defaultReplication=000",
|
||||
}
|
||||
|
||||
c.masterCmd = exec.Command(c.weedBinary, args...)
|
||||
c.masterCmd.Dir = c.baseDir
|
||||
c.masterCmd.Stdout = logFile
|
||||
c.masterCmd.Stderr = logFile
|
||||
if err = c.masterCmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return err
|
||||
}
|
||||
logFile.Close() // child inherited the fd
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
|
||||
logName := fmt.Sprintf("volume%d.log", index)
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-config_dir=" + c.configDir,
|
||||
"volume",
|
||||
"-ip=127.0.0.1",
|
||||
"-port=" + strconv.Itoa(c.volumePorts[index]),
|
||||
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
|
||||
"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
|
||||
"-dir=" + dataDir,
|
||||
"-max=16",
|
||||
"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
|
||||
"-readMode=" + c.profile.ReadMode,
|
||||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
|
||||
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
|
||||
}
|
||||
if c.profile.InflightUploadTimeout > 0 {
|
||||
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
|
||||
}
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
|
||||
cmd := exec.Command(c.weedBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
if err = cmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return err
|
||||
}
|
||||
logFile.Close() // child inherited the fd
|
||||
c.volumeCmds[index] = cmd
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) startRustVolume(index int, dataDir string) error {
|
||||
logName := fmt.Sprintf("volume%d.log", index)
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, logName))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
args := rustVolumeArgs(
|
||||
c.profile,
|
||||
c.configDir,
|
||||
c.masterPort,
|
||||
c.volumePorts[index],
|
||||
c.volumeGrpcPorts[index],
|
||||
c.volumePubPorts[index],
|
||||
dataDir,
|
||||
)
|
||||
|
||||
cmd := exec.Command(c.rustVolumeBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
if err = cmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return err
|
||||
}
|
||||
logFile.Close() // child inherited the fd
|
||||
c.volumeCmds[index] = cmd
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- accessor methods (mirror MultiVolumeCluster) ---
|
||||
|
||||
func (c *MixedVolumeCluster) MasterAddress() string {
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) MasterURL() string {
|
||||
return "http://" + c.MasterAddress()
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumeAdminAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumePorts) {
|
||||
return ""
|
||||
}
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumePublicAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumePubPorts) {
|
||||
return ""
|
||||
}
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumeGRPCAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumeGrpcPorts) {
|
||||
return ""
|
||||
}
|
||||
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumeAdminURL(index int) string {
|
||||
return "http://" + c.VolumeAdminAddress(index)
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) VolumePublicURL(index int) string {
|
||||
return "http://" + c.VolumePublicAddress(index)
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) BaseDir() string {
|
||||
return c.baseDir
|
||||
}
|
||||
|
||||
// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
|
||||
func (c *MixedVolumeCluster) VolumeServerAddress(index int) string {
|
||||
if index < 0 || index >= len(c.volumePorts) {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(index), c.volumeGrpcPorts[index])
|
||||
}
|
||||
|
||||
func (c *MixedVolumeCluster) IsRust(index int) bool {
|
||||
if index < 0 || index >= len(c.isRust) {
|
||||
return false
|
||||
}
|
||||
return c.isRust[index]
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
)
|
||||
|
||||
@@ -75,12 +74,10 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
|
||||
t.Fatalf("write security config: %v", err)
|
||||
}
|
||||
|
||||
miniPorts, err := testutil.AllocateMiniPorts(1)
|
||||
masterPort, masterGrpcPort, err := allocateMasterPortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("allocate master port pair: %v", err)
|
||||
}
|
||||
masterPort := miniPorts[0]
|
||||
masterGrpcPort := masterPort + testutil.GrpcPortOffset
|
||||
|
||||
// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
|
||||
// If SplitPublicPort is true, we need an additional port per server
|
||||
@@ -89,7 +86,7 @@ func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount i
|
||||
portsPerServer = 4
|
||||
}
|
||||
totalPorts := serverCount * portsPerServer
|
||||
ports, err := testutil.AllocatePorts(totalPorts)
|
||||
ports, err := allocatePorts(totalPorts)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate volume ports: %v", err)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
)
|
||||
|
||||
@@ -86,12 +85,10 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
|
||||
t.Fatalf("write security config: %v", err)
|
||||
}
|
||||
|
||||
miniPorts, err := testutil.AllocateMiniPorts(1)
|
||||
masterPort, masterGrpcPort, err := allocateMasterPortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("allocate master port pair: %v", err)
|
||||
}
|
||||
masterPort := miniPorts[0]
|
||||
masterGrpcPort := masterPort + testutil.GrpcPortOffset
|
||||
|
||||
// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
|
||||
// If SplitPublicPort is true, we need an additional port per server
|
||||
@@ -100,7 +97,7 @@ func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCou
|
||||
portsPerServer = 4
|
||||
}
|
||||
totalPorts := serverCount * portsPerServer
|
||||
ports, err := testutil.AllocatePorts(totalPorts)
|
||||
ports, err := allocatePorts(totalPorts)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate volume ports: %v", err)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
)
|
||||
|
||||
@@ -80,14 +79,12 @@ func StartRustVolumeCluster(t testing.TB, profile matrix.Profile) *RustCluster {
|
||||
t.Fatalf("write security config: %v", err)
|
||||
}
|
||||
|
||||
miniPorts, err := testutil.AllocateMiniPorts(1)
|
||||
masterPort, masterGrpcPort, err := allocateMasterPortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("allocate master port pair: %v", err)
|
||||
}
|
||||
masterPort := miniPorts[0]
|
||||
masterGrpcPort := masterPort + testutil.GrpcPortOffset
|
||||
|
||||
ports, err := testutil.AllocatePorts(3)
|
||||
ports, err := allocatePorts(3)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate ports: %v", err)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
)
|
||||
|
||||
@@ -26,7 +25,7 @@ func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *Cl
|
||||
|
||||
baseCluster := StartSingleVolumeCluster(t, profile)
|
||||
|
||||
ports, err := testutil.AllocatePorts(2)
|
||||
ports, err := allocatePorts(2)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate filer ports: %v", err)
|
||||
}
|
||||
|
||||
@@ -80,42 +80,6 @@ func EnableMaintenanceMode(t testing.TB, ctx context.Context, client volume_serv
|
||||
}
|
||||
}
|
||||
|
||||
// CorruptDatFile truncates a volume's .dat file to just the superblock (8 bytes)
|
||||
// so that needle reads fail during a full scrub due to data file size mismatch.
|
||||
func CorruptDatFile(t testing.TB, baseDir string, volumeID uint32) {
|
||||
t.Helper()
|
||||
datPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.dat", volumeID))
|
||||
// Truncate to superblock size only, removing all needle data.
|
||||
if err := os.Truncate(datPath, 8); err != nil {
|
||||
t.Fatalf("truncate dat file for corruption: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// CorruptEcxFile appends garbage bytes to a volume's .ecx file on disk so
|
||||
// that CheckIndexFile detects a size mismatch during EC index scrub.
|
||||
func CorruptEcxFile(t testing.TB, baseDir string, volumeID uint32) {
|
||||
t.Helper()
|
||||
ecxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ecx", volumeID))
|
||||
f, err := os.OpenFile(ecxPath, os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
t.Fatalf("open ecx file for corruption: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
|
||||
t.Fatalf("corrupt ecx file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// CorruptEcShardFile truncates an EC shard file to 1 byte so that local shard
|
||||
// reads fail during an EC scrub.
|
||||
func CorruptEcShardFile(t testing.TB, baseDir string, volumeID uint32, shardID int) {
|
||||
t.Helper()
|
||||
shardPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ec%02d", volumeID, shardID))
|
||||
if err := os.Truncate(shardPath, 1); err != nil {
|
||||
t.Fatalf("truncate EC shard file %s: %v", shardPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -1,294 +0,0 @@
|
||||
package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
|
||||
// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
|
||||
// decoded volume produced by VolumeEcShardsToVolume.
|
||||
func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const (
|
||||
volumeID = uint32(140)
|
||||
keyA = uint64(990020)
|
||||
cookieA = uint32(0xDA001122)
|
||||
keyB = uint64(990021)
|
||||
cookieB = uint32(0xDA003344)
|
||||
)
|
||||
|
||||
framework.AllocateVolume(t, client, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fidA := framework.NewFileID(volumeID, keyA, cookieA)
|
||||
fidB := framework.NewFileID(volumeID, keyB, cookieB)
|
||||
payloadA := []byte("needle-A-should-be-deleted-after-decode")
|
||||
payloadB := []byte("needle-B-should-survive-decode")
|
||||
|
||||
// Upload two needles.
|
||||
for _, tc := range []struct {
|
||||
fid string
|
||||
payload []byte
|
||||
}{
|
||||
{fidA, payloadA},
|
||||
{fidB, payloadB},
|
||||
} {
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// EC encode.
|
||||
_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate: %v", err)
|
||||
}
|
||||
|
||||
// Mount all data shards so the EC volume is usable.
|
||||
_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount: %v", err)
|
||||
}
|
||||
|
||||
// Delete needle A via EC path (writes to .ecj).
|
||||
_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
|
||||
}
|
||||
|
||||
// Unmount the normal volume so decode writes fresh files.
|
||||
_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeUnmount: %v", err)
|
||||
}
|
||||
|
||||
// Decode EC shards back to a normal volume.
|
||||
_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsToVolume: %v", err)
|
||||
}
|
||||
|
||||
// Re-mount the decoded volume.
|
||||
_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeMount: %v", err)
|
||||
}
|
||||
|
||||
// Needle A should be gone (deleted via .ecj before decode).
|
||||
respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
|
||||
bodyA := framework.ReadAllAndClose(t, respA)
|
||||
if respA.StatusCode >= 500 {
|
||||
t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
|
||||
}
|
||||
if respA.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
|
||||
}
|
||||
|
||||
// Needle B should still be readable.
|
||||
respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
|
||||
bodyB := framework.ReadAllAndClose(t, respB)
|
||||
if respB.StatusCode != http.StatusOK {
|
||||
t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
|
||||
}
|
||||
if string(bodyB) != string(payloadB) {
|
||||
t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
|
||||
// peer server that contributes no new data shards are still collected during
|
||||
// decode. This is the regression test for the fix in collectEcShards that
|
||||
// always copies .ecj from every shard location.
|
||||
//
|
||||
// Scenario:
|
||||
// - Server 0 holds all 10 data shards (decode target).
|
||||
// - Server 1 holds a copy of shard 0 (no new shards for server 0).
|
||||
// - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
|
||||
// - During decode on server 0, server 1's .ecj must be collected and applied.
|
||||
func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
|
||||
conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
const (
|
||||
volumeID = uint32(141)
|
||||
keyA = uint64(990030)
|
||||
cookieA = uint32(0xDB001122)
|
||||
keyB = uint64(990031)
|
||||
cookieB = uint32(0xDB003344)
|
||||
)
|
||||
|
||||
// Allocate and upload on server 0.
|
||||
framework.AllocateVolume(t, client0, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fidA := framework.NewFileID(volumeID, keyA, cookieA)
|
||||
fidB := framework.NewFileID(volumeID, keyB, cookieB)
|
||||
payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
|
||||
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
|
||||
}
|
||||
resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// EC encode on server 0.
|
||||
_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
|
||||
}
|
||||
|
||||
// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
|
||||
sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
|
||||
strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
|
||||
|
||||
// Copy shard 0 + ecx + ecj from server 0 → server 1.
|
||||
_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
SourceDataNode: sourceDataNode,
|
||||
ShardIds: []uint32{0},
|
||||
CopyEcxFile: true,
|
||||
CopyEcjFile: true,
|
||||
CopyVifFile: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
|
||||
}
|
||||
|
||||
// Mount shard 0 on server 1 so the EC volume can accept deletions.
|
||||
_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
ShardIds: []uint32{0},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
|
||||
}
|
||||
|
||||
// Delete needle A on server 1 only (creates .ecj entry on server 1).
|
||||
_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
|
||||
}
|
||||
|
||||
// Mount all data shards on server 0 (the decode target).
|
||||
_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
|
||||
}
|
||||
|
||||
// Collect .ecj from server 1 → server 0 with NO new shard IDs.
|
||||
// This is the critical path: server 1 has shard 0 which server 0 already
|
||||
// has, so needToCopyShardsInfo would be empty. Before the fix in
|
||||
// collectEcShards, this copy would be skipped entirely, losing server 1's
|
||||
// deletion entries.
|
||||
server1DataNode := cluster.VolumeAdminAddress(1) + "." +
|
||||
strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
|
||||
|
||||
_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
SourceDataNode: server1DataNode,
|
||||
ShardIds: []uint32{}, // No new shards — just .ecj.
|
||||
CopyEcxFile: false,
|
||||
CopyEcjFile: true,
|
||||
CopyVifFile: false,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
|
||||
}
|
||||
|
||||
// Unmount the normal volume before decode.
|
||||
_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeUnmount on server 0: %v", err)
|
||||
}
|
||||
|
||||
// Decode on server 0. RebuildEcxFile should see needle A's deletion from
|
||||
// the .ecj that was collected from server 1.
|
||||
_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
|
||||
VolumeId: volumeID, Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
|
||||
}
|
||||
|
||||
// Re-mount the decoded normal volume.
|
||||
_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeMount on server 0: %v", err)
|
||||
}
|
||||
|
||||
// Needle A should be gone — its deletion was in server 1's .ecj.
|
||||
respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
|
||||
bodyA := framework.ReadAllAndClose(t, respA)
|
||||
if respA.StatusCode >= 500 {
|
||||
t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
|
||||
}
|
||||
if respA.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
|
||||
}
|
||||
|
||||
// Needle B should still be readable.
|
||||
respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
|
||||
bodyB := framework.ReadAllAndClose(t, respB)
|
||||
if respB.StatusCode != http.StatusOK {
|
||||
t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
|
||||
}
|
||||
if string(bodyB) != string(payloadB) {
|
||||
t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
|
||||
}
|
||||
}
|
||||
@@ -752,87 +752,6 @@ func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcIndexConsistencyAfterEncode verifies that every needle indexed in .ecx
|
||||
// can be read back correctly from EC shards after VolumeEcShardsGenerate.
|
||||
// This catches the race condition fixed in this PR where .ecx could reference
|
||||
// data not present in EC shards.
|
||||
func TestEcIndexConsistencyAfterEncode(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(130)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
// Upload multiple needles of varying sizes
|
||||
type testNeedle struct {
|
||||
fid string
|
||||
payload []byte
|
||||
}
|
||||
needles := []testNeedle{
|
||||
{framework.NewFileID(volumeID, 1001, 0xAABB0001), []byte("small-needle-1")},
|
||||
{framework.NewFileID(volumeID, 1002, 0xAABB0002), make([]byte, 1024)}, // 1KB
|
||||
{framework.NewFileID(volumeID, 1003, 0xAABB0003), make([]byte, 64*1024)}, // 64KB
|
||||
{framework.NewFileID(volumeID, 1004, 0xAABB0004), make([]byte, 256*1024)}, // 256KB
|
||||
{framework.NewFileID(volumeID, 1005, 0xAABB0005), []byte("small-needle-2")},
|
||||
}
|
||||
|
||||
// Fill larger payloads with recognizable data
|
||||
for i := range needles {
|
||||
for j := range needles[i].payload {
|
||||
needles[i].payload[j] = byte(i*37 + j%251)
|
||||
}
|
||||
}
|
||||
|
||||
for _, n := range needles {
|
||||
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// EC encode
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
// Mount all data shards so reads go through the EC path
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
// Read every needle back from EC shards and verify payload
|
||||
for _, n := range needles {
|
||||
readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
|
||||
readBody := framework.ReadAllAndClose(t, readResp)
|
||||
if readResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("EC read %s expected 200, got %d", n.fid, readResp.StatusCode)
|
||||
}
|
||||
if string(readBody) != string(n.payload) {
|
||||
t.Fatalf("EC read %s payload mismatch: got %d bytes, want %d bytes", n.fid, len(readBody), len(n.payload))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -16,7 +17,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
@@ -24,6 +24,29 @@ import (
|
||||
)
|
||||
|
||||
// findAvailablePort finds a free TCP port on localhost.
|
||||
func findAvailablePort() (int, error) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
l.Close()
|
||||
return port, nil
|
||||
}
|
||||
|
||||
// waitForPort waits until a TCP port is listening, up to timeout.
|
||||
func waitForPort(port int, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 500*time.Millisecond)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("port %d not listening after %v", port, timeout)
|
||||
}
|
||||
|
||||
// startWeedMini starts a weed mini subprocess and returns the S3 endpoint and cleanup func.
|
||||
func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
|
||||
@@ -37,11 +60,10 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
|
||||
}
|
||||
}
|
||||
|
||||
miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
|
||||
miniMasterPort := miniPorts[0]
|
||||
miniVolumePort := miniPorts[1]
|
||||
miniFilerPort := miniPorts[2]
|
||||
miniS3Port := miniPorts[3]
|
||||
miniMasterPort, _ := findAvailablePort()
|
||||
miniVolumePort, _ := findAvailablePort()
|
||||
miniFilerPort, _ := findAvailablePort()
|
||||
miniS3Port, _ := findAvailablePort()
|
||||
miniDir := t.TempDir()
|
||||
os.WriteFile(filepath.Join(miniDir, "security.toml"), []byte("# empty\n"), 0644)
|
||||
|
||||
@@ -67,11 +89,11 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
|
||||
t.Fatalf("start weed mini: %v", err)
|
||||
}
|
||||
|
||||
if !testutil.WaitForPort(miniS3Port, 30*time.Second) {
|
||||
if err := waitForPort(miniS3Port, 30*time.Second); err != nil {
|
||||
cancel()
|
||||
miniCmd.Wait()
|
||||
logFile.Close()
|
||||
t.Fatalf("weed mini S3 not ready on port %d", miniS3Port)
|
||||
t.Fatalf("weed mini S3 not ready: %v", err)
|
||||
}
|
||||
t.Logf("weed mini S3 ready on port %d", miniS3Port)
|
||||
|
||||
|
||||
@@ -1,567 +0,0 @@
|
||||
package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
)
|
||||
|
||||
// deleteAndWaitForTombstone issues an HTTP DELETE for the given fid on
|
||||
// volumeURL, asserts a successful response, then polls with GET until
|
||||
// the file returns 404 (tombstone visible) or the timeout elapses.
|
||||
func deleteAndWaitForTombstone(t *testing.T, httpClient *http.Client, volumeURL, fid string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("build delete request for %s: %v", fid, err)
|
||||
}
|
||||
resp := framework.DoRequest(t, httpClient, req)
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("delete %s: expected 200 or 202, got %d", fid, resp.StatusCode)
|
||||
}
|
||||
// Poll until GET returns 404 (tombstone flushed to disk).
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
getResp := framework.ReadBytes(t, httpClient, volumeURL, fid)
|
||||
status := getResp.StatusCode
|
||||
framework.ReadAllAndClose(t, getResp)
|
||||
if status == http.StatusNotFound {
|
||||
return
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("delete %s: tombstone not visible after 5s", fid)
|
||||
}
|
||||
|
||||
// TestMixedBalanceCopyGoToRust verifies that VolumeCopy works from a Go volume
|
||||
// server to a Rust volume server. This is the core operation behind volume
|
||||
// balancing in a mixed Go+Rust cluster.
|
||||
func TestMixedBalanceCopyGoToRust(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
// server 0 = Go, server 1 = Rust
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(50)
|
||||
|
||||
// Allocate volume on Go server and upload test data
|
||||
framework.AllocateVolume(t, goClient, volumeID, "")
|
||||
|
||||
testFiles := []struct {
|
||||
key uint64
|
||||
cookie uint32
|
||||
data []byte
|
||||
}{
|
||||
{1, 0xAABBCCDD, []byte("hello from Go server")},
|
||||
{2, 0x11223344, []byte("second file for balance test")},
|
||||
{3, 0xDEADBEEF, make([]byte, 4096)}, // larger file
|
||||
}
|
||||
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, f.data)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Read source volume status before copy
|
||||
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
|
||||
}
|
||||
t.Logf("Source: dat=%d idx=%d files=%d version=%d",
|
||||
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
|
||||
sourceStatus.GetFileCount(), sourceStatus.GetVersion())
|
||||
|
||||
// Copy volume from Go (server 0) to Rust (server 1)
|
||||
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy call failed: %v", err)
|
||||
}
|
||||
|
||||
var lastAppendAtNs uint64
|
||||
for {
|
||||
resp, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv error: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
if resp.GetLastAppendAtNs() != 0 {
|
||||
lastAppendAtNs = resp.GetLastAppendAtNs()
|
||||
}
|
||||
}
|
||||
t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
|
||||
|
||||
// Verify: read volume status from Rust server
|
||||
targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
|
||||
}
|
||||
t.Logf("Target: dat=%d idx=%d files=%d version=%d",
|
||||
targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
|
||||
targetStatus.GetFileCount(), targetStatus.GetVersion())
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
|
||||
t.Fatalf("dat file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
|
||||
t.Fatalf("idx file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
|
||||
}
|
||||
if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
|
||||
t.Fatalf("file count mismatch: source=%d target=%d",
|
||||
sourceStatus.GetFileCount(), targetStatus.GetFileCount())
|
||||
}
|
||||
|
||||
// Verify data can be read from Rust server
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read %s from Rust server: expected 200, got %d", fid, resp.StatusCode)
|
||||
}
|
||||
if !bytes.Equal(body, f.data) {
|
||||
t.Fatalf("read %s from Rust server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMixedBalanceCopyRustToGo verifies that VolumeCopy works from a Rust
|
||||
// volume server to a Go volume server.
|
||||
func TestMixedBalanceCopyRustToGo(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(51)
|
||||
|
||||
// Allocate volume on Rust server and upload test data
|
||||
framework.AllocateVolume(t, rustClient, volumeID, "")
|
||||
|
||||
testFiles := []struct {
|
||||
key uint64
|
||||
cookie uint32
|
||||
data []byte
|
||||
}{
|
||||
{1, 0xAABBCCDD, []byte("hello from Rust server")},
|
||||
{2, 0x11223344, []byte("second file for reverse balance")},
|
||||
}
|
||||
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid, f.data)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
sourceStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
|
||||
}
|
||||
|
||||
// Copy volume from Rust (server 1) to Go (server 0)
|
||||
copyStream, err := goClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy call failed: %v", err)
|
||||
}
|
||||
for {
|
||||
_, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv error: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
targetStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
|
||||
}
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
|
||||
t.Fatalf("dat file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
|
||||
t.Fatalf("idx file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
|
||||
}
|
||||
if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
|
||||
t.Fatalf("file count mismatch: source=%d target=%d",
|
||||
sourceStatus.GetFileCount(), targetStatus.GetFileCount())
|
||||
}
|
||||
|
||||
// Verify data can be read from Go server
|
||||
for _, f := range testFiles {
|
||||
fid := framework.NewFileID(volumeID, f.key, f.cookie)
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read %s from Go server: expected 200, got %d", fid, resp.StatusCode)
|
||||
}
|
||||
if !bytes.Equal(body, f.data) {
|
||||
t.Fatalf("read %s from Go server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMixedBalanceCopyWithDeletes verifies that VolumeCopy correctly handles
|
||||
// volumes that have both active and deleted needles.
|
||||
func TestMixedBalanceCopyWithDeletes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(52)
|
||||
framework.AllocateVolume(t, goClient, volumeID, "")
|
||||
|
||||
// Upload files
|
||||
for i := uint64(1); i <= 5; i++ {
|
||||
fid := framework.NewFileID(volumeID, i, 0x12340000+uint32(i))
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("file-%d", i)))
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete some files and wait for tombstones to be visible
|
||||
for _, key := range []uint64{2, 4} {
|
||||
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
|
||||
deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0), fid)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus: %v", err)
|
||||
}
|
||||
t.Logf("Source after deletes: dat=%d idx=%d files=%d",
|
||||
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
|
||||
sourceStatus.GetFileCount())
|
||||
|
||||
// Copy volume from Go to Rust
|
||||
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy: %v", err)
|
||||
}
|
||||
var lastAppendAtNs uint64
|
||||
for {
|
||||
resp, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv error: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
if resp.GetLastAppendAtNs() != 0 {
|
||||
lastAppendAtNs = resp.GetLastAppendAtNs()
|
||||
}
|
||||
}
|
||||
if lastAppendAtNs == 0 {
|
||||
t.Fatalf("VolumeCopy did not return a lastAppendAtNs timestamp")
|
||||
}
|
||||
t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
|
||||
|
||||
targetStatusAfterCopy, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus on Rust: %v", err)
|
||||
}
|
||||
t.Logf("Target after copy: dat=%d idx=%d files=%d",
|
||||
targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterCopy.GetIdxFileSize(),
|
||||
targetStatusAfterCopy.GetFileCount())
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
|
||||
t.Fatalf("dat file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatusAfterCopy.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
|
||||
t.Fatalf("idx file size mismatch: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatusAfterCopy.GetIdxFileSize())
|
||||
}
|
||||
|
||||
// Tail from the copy checkpoint — source is unchanged (deletes happened
|
||||
// before copy), so tailing should not append any data.
|
||||
_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
|
||||
VolumeId: volumeID,
|
||||
SinceNs: lastAppendAtNs,
|
||||
IdleTimeoutSeconds: 3,
|
||||
SourceVolumeServer: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeTailReceiver: %v", err)
|
||||
}
|
||||
|
||||
targetStatusAfterTail, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadVolumeFileStatus after tail: %v", err)
|
||||
}
|
||||
if targetStatusAfterTail.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
|
||||
t.Fatalf("dat grew after tail: before=%d after=%d",
|
||||
targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterTail.GetDatFileSize())
|
||||
}
|
||||
if targetStatusAfterTail.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
|
||||
t.Fatalf("idx grew after tail: before=%d after=%d",
|
||||
targetStatusAfterCopy.GetIdxFileSize(), targetStatusAfterTail.GetIdxFileSize())
|
||||
}
|
||||
|
||||
// Verify surviving files are readable from Rust
|
||||
for _, key := range []uint64{1, 3, 5} {
|
||||
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read surviving file %d from Rust: expected 200, got %d", key, resp.StatusCode)
|
||||
}
|
||||
expected := fmt.Sprintf("file-%d", key)
|
||||
if string(body) != expected {
|
||||
t.Fatalf("data mismatch for file %d: got %q want %q", key, body, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify deleted files return 404
|
||||
for _, key := range []uint64{2, 4} {
|
||||
fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("read deleted file %d from Rust: expected 404, got %d", key, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMixedBalanceFullMoveGoToRust exercises the complete volume balance move
|
||||
// flow: mark readonly → copy → tail → verify sizes → delete source.
|
||||
// This mirrors the steps in balance_task.go Execute().
|
||||
func TestMixedBalanceFullMoveGoToRust(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
|
||||
|
||||
conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer conn0.Close()
|
||||
conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer conn1.Close()
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
const volumeID = uint32(53)
|
||||
framework.AllocateVolume(t, goClient, volumeID, "")
|
||||
|
||||
// Upload test data
|
||||
for i := uint64(1); i <= 5; i++ {
|
||||
fid := framework.NewFileID(volumeID, i, 0xABCD0000+uint32(i))
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("balance-move-file-%d", i)))
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Step 1: Copy volume to Rust server (source stays writable so we can
|
||||
// delete after copy to exercise the tail tombstone path)
|
||||
copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceDataNode: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeCopy: %v", err)
|
||||
}
|
||||
var lastAppendAtNs uint64
|
||||
for {
|
||||
resp, recvErr := copyStream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr != io.EOF {
|
||||
t.Fatalf("VolumeCopy recv: %v", recvErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
if resp.GetLastAppendAtNs() != 0 {
|
||||
lastAppendAtNs = resp.GetLastAppendAtNs()
|
||||
}
|
||||
}
|
||||
t.Logf("Copy done, lastAppendAtNs=%d", lastAppendAtNs)
|
||||
|
||||
// Step 2: Delete file 3 on the source AFTER copy. This creates a
|
||||
// tombstone needle that the tail step must propagate to the Rust server.
|
||||
deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0),
|
||||
framework.NewFileID(volumeID, 3, 0xABCD0003))
|
||||
|
||||
// Step 3: Mark source readonly so no further writes arrive during tail
|
||||
_, err = goClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("mark readonly: %v", err)
|
||||
}
|
||||
|
||||
// Read source status (the reference for post-tail verification)
|
||||
sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read source status: %v", err)
|
||||
}
|
||||
t.Logf("Source: dat=%d idx=%d files=%d",
|
||||
sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
|
||||
sourceStatus.GetFileCount())
|
||||
|
||||
// Step 4: Tail for updates — this must pick up the delete tombstone
|
||||
_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
|
||||
VolumeId: volumeID,
|
||||
SinceNs: lastAppendAtNs,
|
||||
IdleTimeoutSeconds: 5,
|
||||
SourceVolumeServer: cluster.VolumeServerAddress(0),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeTailReceiver: %v", err)
|
||||
}
|
||||
|
||||
// Step 5: Verify dat/idx sizes match after tail.
|
||||
// We compare file sizes (byte-level correctness) rather than file_count
|
||||
// because the tail writes the tombstone via the write path (which
|
||||
// increments file_count) while the source used the delete path (which
|
||||
// only increments deletion_count). This is consistent with Go behavior.
|
||||
targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("read target status: %v", err)
|
||||
}
|
||||
t.Logf("Target: dat=%d idx=%d files=%d",
|
||||
targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
|
||||
targetStatus.GetFileCount())
|
||||
|
||||
if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
|
||||
t.Fatalf("dat size mismatch after tail: source=%d target=%d",
|
||||
sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
|
||||
}
|
||||
if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
|
||||
t.Fatalf("idx size mismatch after tail: source=%d target=%d",
|
||||
sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
|
||||
}
|
||||
|
||||
// Step 6: Delete volume from source
|
||||
_, err = goClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("delete source volume: %v", err)
|
||||
}
|
||||
|
||||
// Verify source volume is gone
|
||||
_, err = goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error reading deleted source volume, got nil")
|
||||
}
|
||||
t.Logf("Source volume deleted successfully (error as expected: %v)", err)
|
||||
|
||||
// Verify all surviving data is readable from Rust (the new home)
|
||||
for _, key := range []uint64{1, 2, 4, 5} {
|
||||
fid := framework.NewFileID(volumeID, key, 0xABCD0000+uint32(key))
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("read file %d from Rust after move: expected 200, got %d", key, resp.StatusCode)
|
||||
}
|
||||
expected := fmt.Sprintf("balance-move-file-%d", key)
|
||||
if string(body) != expected {
|
||||
t.Fatalf("data mismatch for file %d after move: got %q want %q", key, body, expected)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify deleted file is 404 on Rust (tombstone propagated via tail)
|
||||
{
|
||||
fid := framework.NewFileID(volumeID, 3, 0xABCD0003)
|
||||
resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
|
||||
framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("deleted file 3 on Rust: expected 404, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Full balance move completed: volume %d moved from Go to Rust, source purged", volumeID)
|
||||
}
|
||||
@@ -1,416 +0,0 @@
|
||||
package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
)
|
||||
|
||||
// --- Normal volume scrub tests ---
|
||||
|
||||
func TestScrubVolumeFullHealthy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(200)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")))
|
||||
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")))
|
||||
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")))
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
|
||||
VolumeIds: []uint32{volumeID},
|
||||
Mode: volume_server_pb.VolumeScrubMode_FULL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubVolume FULL on healthy volume failed: %v", err)
|
||||
}
|
||||
if resp.GetTotalVolumes() != 1 {
|
||||
t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
|
||||
}
|
||||
if resp.GetTotalFiles() != 3 {
|
||||
t.Fatalf("expected total_files=3, got %d", resp.GetTotalFiles())
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) != 0 {
|
||||
t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubVolumeFullCorruptData(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(201)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")))
|
||||
|
||||
framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
|
||||
VolumeIds: []uint32{volumeID},
|
||||
Mode: volume_server_pb.VolumeScrubMode_FULL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubVolume FULL on corrupt volume failed: %v", err)
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) == 0 {
|
||||
t.Fatalf("expected broken volume after data corruption, got none")
|
||||
}
|
||||
if len(resp.GetDetails()) == 0 {
|
||||
t.Fatalf("expected error details for corrupt volume")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubVolumeMixedHealthy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const healthyVol = uint32(202)
|
||||
const corruptVol = uint32(203)
|
||||
framework.AllocateVolume(t, grpcClient, healthyVol, "")
|
||||
framework.AllocateVolume(t, grpcClient, corruptVol, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")))
|
||||
framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")))
|
||||
|
||||
framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
|
||||
VolumeIds: []uint32{healthyVol, corruptVol},
|
||||
Mode: volume_server_pb.VolumeScrubMode_INDEX,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubVolume INDEX on mixed volumes failed: %v", err)
|
||||
}
|
||||
if resp.GetTotalVolumes() != 2 {
|
||||
t.Fatalf("expected total_volumes=2, got %d", resp.GetTotalVolumes())
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) != 1 {
|
||||
t.Fatalf("expected exactly 1 broken volume, got %v", resp.GetBrokenVolumeIds())
|
||||
}
|
||||
if resp.GetBrokenVolumeIds()[0] != corruptVol {
|
||||
t.Fatalf("expected broken volume %d, got %d", corruptVol, resp.GetBrokenVolumeIds()[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubVolumeMissingVolumeReturnsError(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
|
||||
VolumeIds: []uint32{99999},
|
||||
Mode: volume_server_pb.VolumeScrubMode_FULL,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("ScrubVolume should fail for missing volume")
|
||||
}
|
||||
}
|
||||
|
||||
// --- EC volume scrub tests ---
|
||||
|
||||
// ecSetup creates a volume, uploads data, generates EC shards, and mounts all of them.
|
||||
func ecSetup(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, httpClient *http.Client, volumeURL string, volumeID uint32) {
|
||||
t.Helper()
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, volumeURL, fid, []byte("ec-scrub-test-data-payload"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
allShards := make([]uint32, erasure_coding.TotalShardsCount)
|
||||
for i := range allShards {
|
||||
allShards[i] = uint32(i)
|
||||
}
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: allShards,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount all shards failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeIndexHealthy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(210)
|
||||
httpClient := framework.NewHTTPClient()
|
||||
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
|
||||
VolumeIds: []uint32{volumeID},
|
||||
Mode: volume_server_pb.VolumeScrubMode_INDEX,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubEcVolume INDEX on healthy volume failed: %v", err)
|
||||
}
|
||||
if resp.GetTotalVolumes() != 1 {
|
||||
t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) != 0 {
|
||||
t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeLocalHealthy(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(211)
|
||||
httpClient := framework.NewHTTPClient()
|
||||
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
|
||||
VolumeIds: []uint32{volumeID},
|
||||
Mode: volume_server_pb.VolumeScrubMode_LOCAL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubEcVolume LOCAL on healthy volume failed: %v", err)
|
||||
}
|
||||
if resp.GetTotalVolumes() != 1 {
|
||||
t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
|
||||
}
|
||||
if resp.GetTotalFiles() != 1 {
|
||||
t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles())
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) != 0 {
|
||||
t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
|
||||
}
|
||||
if len(resp.GetBrokenShardInfos()) != 0 {
|
||||
t.Fatalf("expected no broken shards, got %v", resp.GetBrokenShardInfos())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeLocalCorruptShard(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(212)
|
||||
httpClient := framework.NewHTTPClient()
|
||||
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
|
||||
|
||||
// Corrupt shard 0 by truncating it.
|
||||
framework.CorruptEcShardFile(t, clusterHarness.BaseDir(), volumeID, 0)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
|
||||
VolumeIds: []uint32{volumeID},
|
||||
Mode: volume_server_pb.VolumeScrubMode_LOCAL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubEcVolume LOCAL on corrupt shard failed: %v", err)
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) == 0 {
|
||||
t.Fatalf("expected broken volume after shard corruption")
|
||||
}
|
||||
if len(resp.GetBrokenShardInfos()) == 0 {
|
||||
t.Fatalf("expected broken shard info after shard corruption")
|
||||
}
|
||||
// Verify all reported broken shards belong to the corrupted volume.
|
||||
for _, si := range resp.GetBrokenShardInfos() {
|
||||
if si.GetVolumeId() != volumeID {
|
||||
t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeIDA = uint32(213)
|
||||
const volumeIDB = uint32(214)
|
||||
httpClient := framework.NewHTTPClient()
|
||||
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDA)
|
||||
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDB)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Auto-select (empty VolumeIds) should find both EC volumes.
|
||||
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
|
||||
Mode: volume_server_pb.VolumeScrubMode_INDEX,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
|
||||
}
|
||||
if resp.GetTotalVolumes() < 2 {
|
||||
t.Fatalf("expected at least 2 EC volumes via auto-select, got %d", resp.GetTotalVolumes())
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) != 0 {
|
||||
t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeUnsupportedMode(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(215)
|
||||
httpClient := framework.NewHTTPClient()
|
||||
ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
|
||||
VolumeIds: []uint32{volumeID},
|
||||
Mode: volume_server_pb.VolumeScrubMode(99),
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("ScrubEcVolume should fail for unsupported mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubEcVolumeIndexCorruptEcx(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(216)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-ecx-corrupt-test"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Generate EC shards (this creates the .ecx file on disk).
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
// Corrupt the .ecx file BEFORE mounting, so the corrupted size is loaded.
|
||||
framework.CorruptEcxFile(t, clusterHarness.BaseDir(), volumeID)
|
||||
|
||||
// Now mount shards - the ecx file size will reflect the corruption.
|
||||
allShards := make([]uint32, erasure_coding.TotalShardsCount)
|
||||
for i := range allShards {
|
||||
allShards[i] = uint32(i)
|
||||
}
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: allShards,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
|
||||
VolumeIds: []uint32{volumeID},
|
||||
Mode: volume_server_pb.VolumeScrubMode_INDEX,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ScrubEcVolume INDEX on corrupt ecx failed: %v", err)
|
||||
}
|
||||
if len(resp.GetBrokenVolumeIds()) == 0 {
|
||||
t.Fatalf("expected broken volume after ECX corruption")
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,6 @@ type ObjectStoreUser struct {
|
||||
SecretKey string `json:"secret_key"`
|
||||
Permissions []string `json:"permissions"`
|
||||
PolicyNames []string `json:"policy_names"`
|
||||
IsStatic bool `json:"is_static"` // loaded from static config file, not editable
|
||||
}
|
||||
|
||||
type ObjectStoreUsersData struct {
|
||||
|
||||
+158
-24
@@ -867,24 +867,6 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
|
||||
})
|
||||
}
|
||||
|
||||
// IsStaticUser checks if a user is a static identity by loading the
|
||||
// configuration from the credential manager and checking the IsStatic flag.
|
||||
func (s *AdminServer) IsStaticUser(username string) bool {
|
||||
if s.credentialManager == nil {
|
||||
return false
|
||||
}
|
||||
s3cfg, err := s.credentialManager.LoadConfiguration(context.Background())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, ident := range s3cfg.Identities {
|
||||
if ident.Name == username {
|
||||
return ident.IsStatic
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetObjectStoreUsers retrieves object store users from identity.json
|
||||
func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) {
|
||||
if s.credentialManager == nil {
|
||||
@@ -908,7 +890,6 @@ func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUse
|
||||
user := ObjectStoreUser{
|
||||
Username: identity.Name,
|
||||
Permissions: identity.Actions,
|
||||
IsStatic: identity.IsStatic,
|
||||
}
|
||||
|
||||
// Set email from account if available
|
||||
@@ -962,7 +943,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
|
||||
leaderCount++
|
||||
}
|
||||
|
||||
masterMap[masterInfo.Address] = masterInfo
|
||||
masterMap[master.Address] = masterInfo
|
||||
}
|
||||
|
||||
// Then, get additional master information from Raft cluster
|
||||
@@ -974,11 +955,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
|
||||
|
||||
// Process each raft server
|
||||
for _, server := range resp.ClusterServers {
|
||||
// Raft stores gRPC addresses, convert to HTTP address
|
||||
httpAddress := pb.GrpcAddressToServerAddress(server.Address)
|
||||
address := server.Address
|
||||
httpAddress := pb.ServerAddress(address).ToHttpAddress()
|
||||
|
||||
// Update existing master info or create new one
|
||||
if masterInfo, exists := masterMap[httpAddress]; exists {
|
||||
if masterInfo, exists := masterMap[address]; exists {
|
||||
// Update existing master with raft data
|
||||
masterInfo.IsLeader = server.IsLeader
|
||||
masterInfo.Suffrage = server.Suffrage
|
||||
@@ -989,7 +970,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
|
||||
IsLeader: server.IsLeader,
|
||||
Suffrage: server.Suffrage,
|
||||
}
|
||||
masterMap[httpAddress] = masterInfo
|
||||
masterMap[address] = masterInfo
|
||||
}
|
||||
|
||||
if server.IsLeader {
|
||||
@@ -1665,6 +1646,159 @@ func (as *AdminServer) GetConfigPersistence() *ConfigPersistence {
|
||||
return as.configPersistence
|
||||
}
|
||||
|
||||
// convertJSONToMaintenanceConfig converts JSON map to protobuf MaintenanceConfig
|
||||
func convertJSONToMaintenanceConfig(jsonConfig map[string]interface{}) (*maintenance.MaintenanceConfig, error) {
|
||||
config := &maintenance.MaintenanceConfig{}
|
||||
|
||||
// Helper function to get int32 from interface{}
|
||||
getInt32 := func(key string) (int32, error) {
|
||||
if val, ok := jsonConfig[key]; ok {
|
||||
switch v := val.(type) {
|
||||
case int:
|
||||
return int32(v), nil
|
||||
case int32:
|
||||
return v, nil
|
||||
case int64:
|
||||
return int32(v), nil
|
||||
case float64:
|
||||
return int32(v), nil
|
||||
default:
|
||||
return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Helper function to get bool from interface{}
|
||||
getBool := func(key string) bool {
|
||||
if val, ok := jsonConfig[key]; ok {
|
||||
if b, ok := val.(bool); ok {
|
||||
return b
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// Convert basic fields
|
||||
config.Enabled = getBool("enabled")
|
||||
|
||||
if config.ScanIntervalSeconds, err = getInt32("scan_interval_seconds"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.WorkerTimeoutSeconds, err = getInt32("worker_timeout_seconds"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.TaskTimeoutSeconds, err = getInt32("task_timeout_seconds"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.RetryDelaySeconds, err = getInt32("retry_delay_seconds"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.MaxRetries, err = getInt32("max_retries"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.CleanupIntervalSeconds, err = getInt32("cleanup_interval_seconds"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.TaskRetentionSeconds, err = getInt32("task_retention_seconds"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert policy if present
|
||||
if policyData, ok := jsonConfig["policy"]; ok {
|
||||
if policyMap, ok := policyData.(map[string]interface{}); ok {
|
||||
policy := &maintenance.MaintenancePolicy{}
|
||||
|
||||
if globalMaxConcurrent, err := getInt32FromMap(policyMap, "global_max_concurrent"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
policy.GlobalMaxConcurrent = globalMaxConcurrent
|
||||
}
|
||||
|
||||
if defaultRepeatIntervalSeconds, err := getInt32FromMap(policyMap, "default_repeat_interval_seconds"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
policy.DefaultRepeatIntervalSeconds = defaultRepeatIntervalSeconds
|
||||
}
|
||||
|
||||
if defaultCheckIntervalSeconds, err := getInt32FromMap(policyMap, "default_check_interval_seconds"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
policy.DefaultCheckIntervalSeconds = defaultCheckIntervalSeconds
|
||||
}
|
||||
|
||||
// Convert task policies if present
|
||||
if taskPoliciesData, ok := policyMap["task_policies"]; ok {
|
||||
if taskPoliciesMap, ok := taskPoliciesData.(map[string]interface{}); ok {
|
||||
policy.TaskPolicies = make(map[string]*maintenance.TaskPolicy)
|
||||
|
||||
for taskType, taskPolicyData := range taskPoliciesMap {
|
||||
if taskPolicyMap, ok := taskPolicyData.(map[string]interface{}); ok {
|
||||
taskPolicy := &maintenance.TaskPolicy{}
|
||||
|
||||
taskPolicy.Enabled = getBoolFromMap(taskPolicyMap, "enabled")
|
||||
|
||||
if maxConcurrent, err := getInt32FromMap(taskPolicyMap, "max_concurrent"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
taskPolicy.MaxConcurrent = maxConcurrent
|
||||
}
|
||||
|
||||
if repeatIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "repeat_interval_seconds"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
taskPolicy.RepeatIntervalSeconds = repeatIntervalSeconds
|
||||
}
|
||||
|
||||
if checkIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "check_interval_seconds"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
taskPolicy.CheckIntervalSeconds = checkIntervalSeconds
|
||||
}
|
||||
|
||||
policy.TaskPolicies[taskType] = taskPolicy
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
config.Policy = policy
|
||||
}
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// Helper functions for map conversion
|
||||
func getInt32FromMap(m map[string]interface{}, key string) (int32, error) {
|
||||
if val, ok := m[key]; ok {
|
||||
switch v := val.(type) {
|
||||
case int:
|
||||
return int32(v), nil
|
||||
case int32:
|
||||
return v, nil
|
||||
case int64:
|
||||
return int32(v), nil
|
||||
case float64:
|
||||
return int32(v), nil
|
||||
default:
|
||||
return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
|
||||
}
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func getBoolFromMap(m map[string]interface{}, key string) bool {
|
||||
if val, ok := m[key]; ok {
|
||||
if b, ok := val.(bool); ok {
|
||||
return b
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type collectionStats struct {
|
||||
PhysicalSize int64
|
||||
LogicalSize int64
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user