mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-13 10:05:50 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Copilot
|
91b834bd9d | ||
|
Copilot
|
2f84fdfc8d | ||
|
Copilot
|
c820280996 | ||
|
Copilot
|
2033ec28d7 | ||
|
Copilot
|
e3ef3a5b56 | ||
|
Copilot
|
0a4ec5ba15 |
@@ -4,10 +4,6 @@ on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -22,7 +18,6 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
|
||||
@@ -9,96 +9,17 @@ permissions:
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -109,40 +30,40 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v7.1.0
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v7.1.0
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -23,22 +23,15 @@ on:
|
||||
- all
|
||||
- standard
|
||||
- large_disk
|
||||
publish:
|
||||
description: 'Publish images and manifests'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
variants: ${{ steps.set-variants.outputs.variants }}
|
||||
publish: ${{ steps.set-publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Select variants for this run
|
||||
id: set-variants
|
||||
@@ -49,89 +42,9 @@ jobs:
|
||||
variants='["standard","large_disk"]'
|
||||
fi
|
||||
echo "variants=$variants" >> "$GITHUB_OUTPUT"
|
||||
- name: Select publish mode
|
||||
id: set-publish
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
needs: [setup]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -166,32 +79,11 @@ jobs:
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
@@ -219,25 +111,25 @@ jobs:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v7.1.0
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ needs.setup.outputs.publish == 'true' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
@@ -257,151 +149,10 @@ jobs:
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Build local scan image tarball
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/build-push-action@v7.1.0
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64
|
||||
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
# since they all use the same alpine base, so a single-arch scan
|
||||
# provides sufficient coverage without multiplying CI time.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
- name: Trivy gate (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, trivy-scan]
|
||||
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
|
||||
needs: [setup, build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
@@ -429,12 +180,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
fi
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v7.1.0
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -39,82 +39,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
|
||||
# emulated cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
# ── Build Docker containers ─────────────────────────────────────────
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# Build sequentially to avoid rate limits
|
||||
@@ -127,23 +52,20 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk - multi-arch
|
||||
|
||||
# Large disk - multi-arch
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# Full tags - multi-arch
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- variant: large_disk_full
|
||||
@@ -151,42 +73,19 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
rust_variant: large-disk
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -236,14 +135,14 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -251,7 +150,7 @@ jobs:
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/build-push-action@v7.1.0
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
@@ -299,14 +198,14 @@ jobs:
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -88,13 +88,13 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
@@ -1,69 +0,0 @@
|
||||
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run P2P integration tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-p2p-test-logs
|
||||
path: /tmp/seaweedfs-fuse-p2p-logs/
|
||||
retention-days: 3
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v5
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
name: "helm: release"
|
||||
name: "helm: manual release"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
name: "NFS Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/nfs-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
nfs-integration:
|
||||
name: NFS Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run NFS Integration Tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running NFS integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run all NFS tests
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "NFS integration tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Each test boots its own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
|
||||
echo "stack on loopback and drives it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
|
||||
echo "go-nfs-client. No kernel NFS mount or privileged port is required." >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1,118 +0,0 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-e2e
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build pjdfstest image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: test/pjdfstest
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-harness
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-harness
|
||||
|
||||
- name: Tag pjdfstest image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
@@ -156,7 +156,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -169,7 +169,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ jobs:
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@@ -48,7 +47,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -84,7 +83,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
@@ -119,7 +118,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -155,7 +154,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
|
||||
@@ -39,19 +39,11 @@ jobs:
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -88,8 +80,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
@@ -97,15 +88,6 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -130,7 +112,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -165,8 +147,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
@@ -174,15 +155,6 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -198,7 +170,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -234,20 +206,10 @@ jobs:
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
|
||||
@@ -245,61 +245,6 @@ jobs:
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Checksum Tests
|
||||
timeout-minutes: 16
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-checksum-test-logs
|
||||
path: test/s3/checksum/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
name: "S3 Mutation Regression Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/delete/**'
|
||||
- 'test/s3/distributed_lock/**'
|
||||
- 'test/s3/versioning/**'
|
||||
- 'test/volume_server/framework/**'
|
||||
- 'docker/compose/s3.json'
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-mutation-regression-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload versioning logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-regression-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload delete logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-delete-regression-logs
|
||||
path: test/s3/delete/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run distributed lock regressions
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
||||
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
||||
|
||||
- name: Upload distributed lock logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-distributed-lock-regression-logs
|
||||
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
||||
retention-days: 3
|
||||
@@ -37,8 +37,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -341,8 +339,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -511,8 +507,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -736,8 +730,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "Vacuum Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run Vacuum Integration Tests
|
||||
working-directory: test/vacuum
|
||||
run: |
|
||||
go test -v -timeout 10m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/vacuum-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vacuum-integration-test-logs
|
||||
path: /tmp/vacuum-test-logs/
|
||||
retention-days: 14
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"setup": [],
|
||||
"teardown": [],
|
||||
"run": []
|
||||
}
|
||||
@@ -42,7 +42,8 @@ test: admin-generate
|
||||
|
||||
# Admin component targets
|
||||
admin-generate:
|
||||
@cd $(ADMIN_DIR) && $(MAKE) generate
|
||||
@echo "Generating admin component templates..."
|
||||
@cd $(ADMIN_DIR) && templ generate ./view
|
||||
|
||||
admin-build: admin-generate
|
||||
@echo "Building admin component..."
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you find a security issue in SeaweedFS, please report it privately:
|
||||
|
||||
- Email: support@seaweedfs.com
|
||||
- Do not open a public GitHub issue
|
||||
|
||||
Please include:
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce (if possible)
|
||||
- Affected versions
|
||||
|
||||
## Response
|
||||
|
||||
- We will respond as soon as possible (usually within 1 business day)
|
||||
- We will investigate and work on a fix
|
||||
- We may coordinate disclosure with you
|
||||
|
||||
## Notes
|
||||
|
||||
- Please allow time for a fix before public disclosure
|
||||
- If you’re unsure whether something is a security issue, feel free to reach out
|
||||
@@ -7,8 +7,8 @@ LABEL author="Chris Lu"
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
RUN apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y \
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
|
||||
@@ -16,22 +16,15 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
# Rust volume server: use pre-built binary from CI when available (placed in
|
||||
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
|
||||
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
|
||||
# for non-native architectures.
|
||||
FROM alpine:3.23 as rust_builder
|
||||
# Rust volume server builder (amd64/arm64 only)
|
||||
FROM rust:1-alpine as rust_builder
|
||||
ARG TARGETARCH
|
||||
ARG TAGS
|
||||
COPY weed-volume-prebuilt/ /prebuilt/
|
||||
RUN apk add musl-dev protobuf-dev git
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/proto /build/proto
|
||||
WORKDIR /build/seaweed-volume
|
||||
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
|
||||
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
|
||||
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
|
||||
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
|
||||
ARG TAGS
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
if [ "$TAGS" = "5BytesOffset" ]; then \
|
||||
cargo build --release; \
|
||||
else \
|
||||
@@ -56,8 +49,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -7,8 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -34,8 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -17,8 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "steward",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "steward-key",
|
||||
"secretKey": "steward-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "le001",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "le001-key",
|
||||
"secretKey": "le001-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -34,12 +34,12 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.0
|
||||
github.com/jackc/pgx/v5 v5.8.0
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
@@ -83,62 +83,63 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/viant/ptrie v1.0.1
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.etcd.io/etcd/client/v3 v3.6.7
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.50.0
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
|
||||
golang.org/x/image v0.38.0
|
||||
golang.org/x/net v0.53.0
|
||||
golang.org/x/image v0.36.0
|
||||
golang.org/x/net v0.51.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.43.0
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.274.0
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/api v0.267.0
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.46.1
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.26.0
|
||||
cloud.google.com/go/kms v1.25.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.977
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
|
||||
github.com/apache/iceberg-go v0.5.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.6.2
|
||||
github.com/getsentry/sentry-go v0.43.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.28.0
|
||||
@@ -153,12 +154,10 @@ require (
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.3
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.0
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
@@ -182,7 +181,7 @@ require (
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -212,6 +211,7 @@ require (
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
@@ -258,7 +258,6 @@ require (
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
@@ -285,14 +284,14 @@ require (
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth v0.18.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
@@ -324,22 +323,22 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
@@ -372,7 +371,7 @@ require (
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
@@ -388,7 +387,7 @@ require (
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
@@ -479,7 +478,7 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@@ -487,22 +486,22 @@ require (
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.42.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
|
||||
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
|
||||
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
||||
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
|
||||
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
|
||||
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
|
||||
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
|
||||
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
|
||||
cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
|
||||
cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
|
||||
cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
|
||||
cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
|
||||
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
|
||||
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
|
||||
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
|
||||
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA=
|
||||
cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
@@ -692,8 +692,8 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 h1:r/gg2gzUXiXoy72VU3jnODh8l/5rL0aslnTAbtmai/U=
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87/go.mod h1:IJTMBTlHe7cDOhRh0ioGuEKBl5iTR6xPfl5BN4AgirU=
|
||||
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq/RrkjCcxCnqUxhwf8=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0 h1:Omnzb1Z/P90Dr2TbVNu54ICQL7TKVIIsJO231w484HU=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
|
||||
github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
@@ -714,48 +714,48 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
|
||||
github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
@@ -1076,8 +1076,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
|
||||
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
|
||||
@@ -1383,8 +1383,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
|
||||
github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
|
||||
github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
|
||||
github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
|
||||
github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
@@ -1403,8 +1403,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.0 h1:T/dI+2TvmI2H8s/KH1/lXIbz1CUFk3gn5oTjr0/mBsE=
|
||||
github.com/jackc/pgx/v5 v5.9.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
|
||||
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jaegertracing/jaeger v1.47.0 h1:XXxTMO+GxX930gxKWsg90rFr6RswkCRIW0AgWFnTYsg=
|
||||
@@ -1783,8 +1783,6 @@ github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3Adq
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
|
||||
@@ -1840,6 +1838,8 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
@@ -2028,10 +2028,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/willscott/go-nfs v0.0.3 h1:Z5fHVxMsppgEucdkKBN26Vou19MtEM875NmRwj156RE=
|
||||
github.com/willscott/go-nfs v0.0.3/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
@@ -2042,8 +2038,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@@ -2064,14 +2060,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 h1:avIdi8eGXjKbn1WLokNR1Ofnz1k8t7tJ88YQLD/iCi8=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.0 h1:Voog4d56wPNT8JQvbgN6QPdZ1gKG36YM5615PJB/XpM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.0/go.mod h1:VYUUkRJkKuQPkIpgtZJj6+58Fa2g8ccAqdmaaK6HP5k=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3 h1:hTwpF+PdbuR7vcixN+4AC6yu4asaUIAljQCxL51eDII=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
|
||||
@@ -2107,12 +2103,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
|
||||
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
@@ -2134,8 +2130,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
|
||||
@@ -2150,14 +2146,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
@@ -2214,8 +2210,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -2246,8 +2242,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2281,8 +2277,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2350,8 +2346,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -2509,11 +2505,11 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc=
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2529,8 +2525,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
|
||||
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -2551,15 +2547,15 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -2630,8 +2626,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -2709,8 +2705,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
||||
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -2844,12 +2840,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.21"
|
||||
appVersion: "4.17"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.21.0
|
||||
version: 4.17.0
|
||||
|
||||
@@ -49,35 +49,6 @@ CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
#### RocksDB variant
|
||||
|
||||
The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
|
||||
To use this image with the Helm chart, override the image on all three components and disable
|
||||
the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
|
||||
override the image's built-in RocksDB configuration:
|
||||
|
||||
```yaml
|
||||
# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
|
||||
master:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
volume:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
filer:
|
||||
enablePVC: true
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
extraEnvironmentVars:
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
* `master` and `volume` use the same image tag so that all components share a consistent
|
||||
SeaweedFS build; RocksDB itself is only used by the filer.
|
||||
* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
|
||||
so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.admin.image" . }}
|
||||
image: {{ template "admin.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
{{- $adminAuthEnabled := or .Values.admin.secret.existingSecret .Values.admin.secret.adminPassword }}
|
||||
{{- $urlPrefix := .Values.admin.urlPrefix }}
|
||||
@@ -242,7 +242,7 @@ spec:
|
||||
securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.admin.data.type "hostPath" }}
|
||||
@@ -303,7 +303,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.admin.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.admin.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "admin.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.admin.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
{{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
|
||||
@@ -418,7 +418,7 @@ spec:
|
||||
{{- omit .Values.allInOne.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: data
|
||||
|
||||
@@ -164,7 +164,7 @@ spec:
|
||||
securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.cosi.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: socket
|
||||
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.filer.image" . }}
|
||||
image: {{ template "filer.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -317,7 +317,7 @@ spec:
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
@@ -413,7 +413,7 @@ spec:
|
||||
storageClassName: {{ .Values.filer.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.filer.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "filer.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -80,7 +80,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -251,7 +251,7 @@ spec:
|
||||
securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
@@ -312,7 +312,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.master.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "master.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.master.data.type "persistentVolumeClaim"}}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.s3.image" . }}
|
||||
image: {{ template "s3.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -226,7 +226,7 @@ spec:
|
||||
securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.s3.enableAuth }}
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
{{- $access_key_admin := $adminCreds.accessKey -}}
|
||||
{{- $secret_key_admin := $adminCreds.secretKey -}}
|
||||
{{- if not (and $access_key_admin $secret_key_admin) -}}
|
||||
{{- $access_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
{{- $readCreds := $creds.read | default dict -}}
|
||||
{{- $access_key_read := $readCreds.accessKey -}}
|
||||
{{- $secret_key_read := $readCreds.secretKey -}}
|
||||
{{- if not (and $access_key_read $secret_key_read) -}}
|
||||
{{- $access_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- /*
|
||||
The seaweedfs-s3 Service only gains a "metrics" port when the standalone S3
|
||||
gateway is enabled. With only the embedded filer S3 gateway, metrics live on
|
||||
the filer process and are already scraped by the filer ServiceMonitor.
|
||||
*/ -}}
|
||||
{{- if and .Values.s3.enabled .Values.s3.metricsPort .Values.global.seaweedfs.monitoring.enabled }}
|
||||
{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
|
||||
{{- if .Values.s3.metricsPort }}
|
||||
{{- if .Values.global.seaweedfs.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
@@ -34,3 +31,5 @@ spec:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.sftp.image" . }}
|
||||
image: {{ template "sftp.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -233,7 +233,7 @@ spec:
|
||||
securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.sftp.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.sftp.enableAuth }}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{{- if or .Values.sftp.enabled .Values.allInOne.enabled }}
|
||||
{{- $secretName := printf "%s-sftp-secret" (include "seaweedfs.fullname" .) }}
|
||||
{{- $admin_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" "length" 20) -}}
|
||||
{{- $read_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" "length" 20) -}}
|
||||
{{- $public_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" "length" 20) -}}
|
||||
{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" 20) -}}
|
||||
{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" 20) -}}
|
||||
{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" 20) -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
type: Opaque
|
||||
|
||||
@@ -72,77 +72,77 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper filer image */}}
|
||||
{{- define "seaweedfs.filer.image" -}}
|
||||
{{- define "filer.image" -}}
|
||||
{{- if .Values.filer.imageOverride -}}
|
||||
{{- $imageOverride := .Values.filer.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper master image */}}
|
||||
{{- define "seaweedfs.master.image" -}}
|
||||
{{- define "master.image" -}}
|
||||
{{- if .Values.master.imageOverride -}}
|
||||
{{- $imageOverride := .Values.master.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper s3 image */}}
|
||||
{{- define "seaweedfs.s3.image" -}}
|
||||
{{- define "s3.image" -}}
|
||||
{{- if .Values.s3.imageOverride -}}
|
||||
{{- $imageOverride := .Values.s3.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper sftp image */}}
|
||||
{{- define "seaweedfs.sftp.image" -}}
|
||||
{{- define "sftp.image" -}}
|
||||
{{- if .Values.sftp.imageOverride -}}
|
||||
{{- $imageOverride := .Values.sftp.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper admin image */}}
|
||||
{{- define "seaweedfs.admin.image" -}}
|
||||
{{- define "admin.image" -}}
|
||||
{{- if .Values.admin.imageOverride -}}
|
||||
{{- $imageOverride := .Values.admin.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper worker image */}}
|
||||
{{- define "seaweedfs.worker.image" -}}
|
||||
{{- define "worker.image" -}}
|
||||
{{- if .Values.worker.imageOverride -}}
|
||||
{{- $imageOverride := .Values.worker.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper volume image */}}
|
||||
{{- define "seaweedfs.volume.image" -}}
|
||||
{{- define "volume.image" -}}
|
||||
{{- if .Values.volume.imageOverride -}}
|
||||
{{- $imageOverride := .Values.volume.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Computes the container image name for all components (if they are not overridden) */}}
|
||||
{{- define "seaweedfs.image" -}}
|
||||
{{- define "common.image" -}}
|
||||
{{- $registryName := default .Values.image.registry .Values.global.imageRegistry | toString -}}
|
||||
{{- $repositoryName := default .Values.image.repository .Values.global.seaweedfs.image.repository | toString -}}
|
||||
{{- $name := .Values.global.seaweedfs.image.name | toString -}}
|
||||
@@ -160,7 +160,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Volume PVC exists */}}
|
||||
{{- define "seaweedfs.volume.pvc_exists" -}}
|
||||
{{- define "volume.pvc_exists" -}}
|
||||
{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -169,7 +169,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Filer PVC exists */}}
|
||||
{{- define "seaweedfs.filer.pvc_exists" -}}
|
||||
{{- define "filer.pvc_exists" -}}
|
||||
{{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -178,7 +178,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Master PVC exists */}}
|
||||
{{- define "seaweedfs.master.pvc_exists" -}}
|
||||
{{- define "master.pvc_exists" -}}
|
||||
{{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -187,7 +187,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Admin PVC exists */}}
|
||||
{{- define "seaweedfs.admin.pvc_exists" -}}
|
||||
{{- define "admin.pvc_exists" -}}
|
||||
{{- if or (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -196,7 +196,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any InitContainers exist for Volumes */}}
|
||||
{{- define "seaweedfs.volume.initContainers_exists" -}}
|
||||
{{- define "volume.initContainers_exists" -}}
|
||||
{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -225,10 +225,10 @@ imagePullSecrets:
|
||||
{{/*
|
||||
Renders a value that contains template perhaps with scope if the scope is present.
|
||||
Usage:
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.tplvalues.render" -}}
|
||||
{{- define "common.tplvalues.render" -}}
|
||||
{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
|
||||
{{- if contains "{{" (toJson .value) }}
|
||||
{{- if .scope }}
|
||||
@@ -245,9 +245,9 @@ Usage:
|
||||
Converts a Kubernetes quantity like "256Mi" or "2G" to a float64 in base units,
|
||||
handling both binary (Ki, Mi, Gi) and decimal (m, k, M) suffixes; numeric inputs
|
||||
Usage:
|
||||
{{ include "seaweedfs.resource-quantity" "10Gi" }}
|
||||
{{ include "common.resource-quantity" "10Gi" }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.resource-quantity" -}}
|
||||
{{- define "common.resource-quantity" -}}
|
||||
{{- $value := . -}}
|
||||
{{- $unit := 1.0 -}}
|
||||
{{- if typeIs "string" . -}}
|
||||
@@ -267,7 +267,7 @@ Usage:
|
||||
getOrGeneratePassword will check if a password exists in a secret and return it,
|
||||
or generate a new random password if it doesn't exist.
|
||||
*/}}
|
||||
{{- define "seaweedfs.getOrGeneratePassword" -}}
|
||||
{{- define "getOrGeneratePassword" -}}
|
||||
{{- $params := . -}}
|
||||
{{- $namespace := $params.namespace -}}
|
||||
{{- $secretName := $params.secretName -}}
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
{{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
|
||||
containers:
|
||||
- name: post-install-job
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
env:
|
||||
- name: WEED_CLUSTER_DEFAULT
|
||||
|
||||
@@ -35,8 +35,8 @@
|
||||
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
|
||||
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
|
||||
{{- if and $currentPVC }}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
|
||||
{{- $oldSize := include "common.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "common.resource-quantity" $desiredSize }}
|
||||
{{- if gt $newSize $oldSize }}
|
||||
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
|
||||
{{- end }}
|
||||
|
||||
@@ -71,12 +71,12 @@ spec:
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
|
||||
{{- $initContainers_exists := include "seaweedfs.volume.initContainers_exists" $ -}}
|
||||
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
|
||||
{{- if $initContainers_exists }}
|
||||
initContainers:
|
||||
{{- if $volume.idx }}
|
||||
- name: seaweedfs-vol-move-idx
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
command: [ '/bin/sh', '-c' ]
|
||||
args: [ '{{range $dir := $volume.dataDirs }}if ls /{{$dir.name}}/*.idx >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
|
||||
@@ -104,7 +104,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" $.Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_NAME
|
||||
@@ -274,7 +274,7 @@ spec:
|
||||
securityContext: {{- omit $volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if $volume.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.worker.image" . }}
|
||||
image: {{ template "worker.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -219,7 +219,7 @@ spec:
|
||||
securityContext: {{- omit .Values.worker.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.worker.data.type "hostPath" }}
|
||||
|
||||
@@ -1339,14 +1339,9 @@ worker:
|
||||
# Admin server to connect to
|
||||
adminServer: ""
|
||||
|
||||
# Worker job types - categories or comma-separated list of names/aliases
|
||||
# Categories: all, default, heavy
|
||||
# default: vacuum, volume_balance, ec_balance, admin_script
|
||||
# heavy: erasure_coding, iceberg_maintenance
|
||||
# all: every registered job type
|
||||
# Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
|
||||
# Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
|
||||
jobType: "all"
|
||||
# Worker job types - comma-separated list
|
||||
# Available: vacuum, volume_balance, erasure_coding
|
||||
jobType: "vacuum,volume_balance,erasure_coding"
|
||||
|
||||
# Maximum number of concurrent detection requests
|
||||
maxDetect: 1
|
||||
|
||||
@@ -84,15 +84,6 @@ service SeaweedFiler {
|
||||
// distributed lock management internal use only
|
||||
rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
|
||||
}
|
||||
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
|
||||
}
|
||||
|
||||
// Peer chunk sharing — tier 1: mount-server registry.
|
||||
// See design-weed-mount-peer-chunk-sharing.md for details.
|
||||
rpc MountRegister (MountRegisterRequest) returns (MountRegisterResponse) {
|
||||
}
|
||||
rpc MountList (MountListRequest) returns (MountListResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
@@ -204,10 +195,6 @@ message FuseAttributes {
|
||||
bytes md5 = 14;
|
||||
uint32 rdev = 16;
|
||||
uint64 inode = 17;
|
||||
int64 ctime = 18; // unix time in seconds, inode change time
|
||||
int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
|
||||
int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
|
||||
int32 crtime_ns = 21; // nanosecond component of crtime (0-999999999)
|
||||
}
|
||||
|
||||
message CreateEntryRequest {
|
||||
@@ -305,7 +292,6 @@ message AssignVolumeRequest {
|
||||
string rack = 7;
|
||||
string data_node = 9;
|
||||
string disk_type = 8;
|
||||
uint64 expected_data_size = 10; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message AssignVolumeResponse {
|
||||
@@ -536,7 +522,6 @@ message LockResponse {
|
||||
string lock_owner = 2;
|
||||
string lock_host_moved_to = 3;
|
||||
string error = 4;
|
||||
int64 generation = 5;
|
||||
}
|
||||
message UnlockRequest {
|
||||
string name = 1;
|
||||
@@ -559,26 +544,12 @@ message Lock {
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_backup = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message TransferLocksRequest {
|
||||
repeated Lock locks = 1;
|
||||
}
|
||||
message TransferLocksResponse {
|
||||
}
|
||||
message ReplicateLockRequest {
|
||||
string name = 1;
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_unlock = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message ReplicateLockResponse {
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
|
||||
@@ -608,31 +579,3 @@ message StreamMutateEntryResponse {
|
||||
string error = 7; // human-readable error message when the operation failed
|
||||
int32 errno = 8; // POSIX errno (e.g. ENOENT=2, ENOTEMPTY=66) for direct FUSE status mapping
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Peer chunk sharing — mount-server registry
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
message MountRegisterRequest {
|
||||
string peer_addr = 1; // host:port where this mount serves peer chunk requests
|
||||
string rack = 2; // locality label (rack); used for peer ranking
|
||||
int32 ttl_seconds = 3; // how long the filer should keep this entry without a heartbeat
|
||||
string data_center = 4; // locality label (data center); coarser than rack
|
||||
}
|
||||
|
||||
message MountRegisterResponse {
|
||||
}
|
||||
|
||||
message MountListRequest {
|
||||
}
|
||||
|
||||
message MountListResponse {
|
||||
repeated MountInfo mounts = 1;
|
||||
}
|
||||
|
||||
message MountInfo {
|
||||
string peer_addr = 1;
|
||||
string rack = 2;
|
||||
int64 last_seen_ns = 3;
|
||||
string data_center = 4;
|
||||
}
|
||||
|
||||
Generated
+310
-150
File diff suppressed because it is too large
Load Diff
@@ -20,7 +20,7 @@ default = ["5bytes"]
|
||||
[dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
tokio-stream = "0.1"
|
||||
tokio-io-timeout = "1"
|
||||
|
||||
# gRPC + protobuf
|
||||
@@ -65,6 +65,8 @@ reed-solomon-erasure = "6"
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
pprof = { version = "0.15", features = ["prost-codec"] }
|
||||
|
||||
# Config
|
||||
toml = "0.8"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -82,7 +84,7 @@ memmap2 = "0.9"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
# HTTP client (for proxying, remote fetch)
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
reqwest = { version = "0.12", features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
|
||||
# Content hashing
|
||||
md-5 = "0.10"
|
||||
@@ -125,10 +127,6 @@ aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4
|
||||
aws-credential-types = "1"
|
||||
aws-types = "1"
|
||||
|
||||
# pprof is Unix-only (requires libc/nix APIs not available on Windows)
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
pprof = { version = "0.15", features = ["prost-codec"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
|
||||
@@ -145,11 +145,6 @@ message VolumeEcShardInformationMessage {
|
||||
uint64 expire_at_sec = 5; // used to record the destruction time of ec volume
|
||||
uint32 disk_id = 6;
|
||||
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
|
||||
uint64 file_count = 8; // total needles in the .ecx index (live + tombstoned)
|
||||
uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
|
||||
// fields 10-19 reserved for future upstream open-source additions.
|
||||
// fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
|
||||
// and must not be used here without coordination.
|
||||
}
|
||||
|
||||
message StorageBackend {
|
||||
@@ -236,7 +231,6 @@ message AssignRequest {
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
uint32 writable_volume_count = 9;
|
||||
string disk_type = 10;
|
||||
uint64 expected_data_size = 11; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message VolumeGrowRequest {
|
||||
|
||||
+37
-85
@@ -10,11 +10,9 @@ use seaweed_volume::security::tls::{
|
||||
GrpcClientAuthPolicy, TlsPolicy,
|
||||
};
|
||||
use seaweed_volume::security::{Guard, SigningKey};
|
||||
#[cfg(unix)]
|
||||
use seaweed_volume::server::debug::build_debug_router;
|
||||
use seaweed_volume::server::grpc_client::load_outgoing_grpc_tls;
|
||||
use seaweed_volume::server::grpc_server::VolumeGrpcService;
|
||||
#[cfg(unix)]
|
||||
use seaweed_volume::server::profiling::CpuProfileSession;
|
||||
use seaweed_volume::server::request_id::GrpcRequestIdLayer;
|
||||
use seaweed_volume::server::volume_server::{
|
||||
@@ -26,11 +24,6 @@ use seaweed_volume::storage::types::DiskType;
|
||||
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
|
||||
#[cfg(unix)]
|
||||
type CpuProfileParam = Option<CpuProfileSession>;
|
||||
#[cfg(not(unix))]
|
||||
type CpuProfileParam = Option<()>;
|
||||
|
||||
const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
|
||||
const GRPC_KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
const GRPC_KEEPALIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
@@ -49,7 +42,6 @@ fn main() {
|
||||
|
||||
let config = config::parse_cli();
|
||||
seaweed_volume::server::server_stats::init_process_start();
|
||||
#[cfg(unix)]
|
||||
let cpu_profile = match CpuProfileSession::start(&config) {
|
||||
Ok(session) => session,
|
||||
Err(e) => {
|
||||
@@ -57,8 +49,6 @@ fn main() {
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let cpu_profile: Option<()> = None;
|
||||
info!(
|
||||
"SeaweedFS Volume Server (Rust) v{}",
|
||||
seaweed_volume::version::full_version()
|
||||
@@ -267,7 +257,7 @@ where
|
||||
|
||||
async fn run(
|
||||
config: VolumeServerConfig,
|
||||
#[allow(unused_variables)] cpu_profile: CpuProfileParam,
|
||||
cpu_profile: Option<CpuProfileSession>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Initialize the store
|
||||
let mut store = Store::new(config.index_type);
|
||||
@@ -441,12 +431,10 @@ async fn run(
|
||||
}
|
||||
|
||||
// Build HTTP routers
|
||||
#[allow(unused_mut)]
|
||||
let mut admin_router = seaweed_volume::server::volume_server::build_admin_router_with_ui(
|
||||
state.clone(),
|
||||
config.ui_enabled,
|
||||
);
|
||||
#[cfg(unix)]
|
||||
if config.pprof {
|
||||
admin_router = admin_router.merge(build_debug_router());
|
||||
}
|
||||
@@ -600,35 +588,31 @@ async fn run(
|
||||
})
|
||||
};
|
||||
|
||||
// Bind the gRPC listener before spawning to propagate bind errors at startup.
|
||||
let grpc_listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let grpc_local_addr = grpc_listener
|
||||
.local_addr()
|
||||
.unwrap_or_else(|e| panic!("Failed to get gRPC local addr: {}", e));
|
||||
|
||||
let grpc_handle = {
|
||||
let grpc_state = state.clone();
|
||||
let grpc_addr = grpc_addr.clone();
|
||||
let grpc_tls_acceptor = grpc_tls_acceptor.clone();
|
||||
let mut shutdown_rx = shutdown_tx.subscribe();
|
||||
let shutdown_tx_grpc = shutdown_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let addr = grpc_addr.parse().expect("Invalid gRPC address");
|
||||
let grpc_service = VolumeGrpcService {
|
||||
state: grpc_state.clone(),
|
||||
};
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
let result = if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let incoming = grpc_tls_incoming(grpc_listener, tls_acceptor);
|
||||
info!("gRPC server listening on {} (TLS enabled)", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let incoming = grpc_tls_incoming(listener, tls_acceptor);
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {} (TLS enabled)", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
@@ -637,25 +621,32 @@ async fn run(
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
} else {
|
||||
let incoming =
|
||||
tokio_stream::wrappers::TcpListenerStream::new(grpc_listener);
|
||||
info!("gRPC server listening on {}", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {}", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
.add_service(build_volume_grpc_service(grpc_service))
|
||||
.serve_with_incoming_shutdown(incoming, async move {
|
||||
.serve_with_shutdown(addr, async move {
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
};
|
||||
if let Err(ref e) = result {
|
||||
error!("gRPC server error: {}", e);
|
||||
let _ = shutdown_tx_grpc.send(());
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
}
|
||||
result
|
||||
})
|
||||
};
|
||||
|
||||
@@ -730,7 +721,6 @@ async fn run(
|
||||
None
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let debug_handle = if config.debug {
|
||||
let debug_addr = format!("0.0.0.0:{}", config.debug_port);
|
||||
info!("Debug pprof server listening on {}", debug_addr);
|
||||
@@ -752,8 +742,6 @@ async fn run(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let debug_handle: Option<tokio::task::JoinHandle<()>> = None;
|
||||
|
||||
let metrics_push_handle = {
|
||||
let push_state = state.clone();
|
||||
@@ -764,40 +752,9 @@ async fn run(
|
||||
}))
|
||||
};
|
||||
|
||||
// Wait for servers. Use select! with &mut so the losing handle is not
|
||||
// dropped, then await it explicitly afterward.
|
||||
let mut server_err: Option<String> = None;
|
||||
let mut http_handle = http_handle;
|
||||
let mut grpc_handle = grpc_handle;
|
||||
let grpc_finished_first = tokio::select! {
|
||||
_ = &mut http_handle => false,
|
||||
_ = &mut grpc_handle => true,
|
||||
};
|
||||
// Inspect the gRPC result (already resolved if it finished first,
|
||||
// otherwise await it now).
|
||||
let grpc_result = if grpc_finished_first {
|
||||
grpc_handle.await
|
||||
} else {
|
||||
// HTTP finished first; gRPC is still running. Await it.
|
||||
grpc_handle.await
|
||||
};
|
||||
match grpc_result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(e)) => {
|
||||
let msg = format!("gRPC server exited with error: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
// serve error already sent shutdown inside the task
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("gRPC task panicked: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
let _ = shutdown_tx.send(());
|
||||
}
|
||||
}
|
||||
// Ensure the HTTP handle completes too.
|
||||
// Wait for all servers
|
||||
let _ = http_handle.await;
|
||||
let _ = grpc_handle.await;
|
||||
if let Some(h) = public_handle {
|
||||
let _ = h.await;
|
||||
}
|
||||
@@ -817,15 +774,10 @@ async fn run(
|
||||
// Close all volumes (flush and release file handles) matching Go's Shutdown()
|
||||
state.store.write().unwrap().close();
|
||||
|
||||
#[cfg(unix)]
|
||||
if let Some(cpu_profile) = cpu_profile {
|
||||
cpu_profile.finish().map_err(std::io::Error::other)?;
|
||||
}
|
||||
|
||||
if let Some(err_msg) = server_err {
|
||||
return Err(std::io::Error::other(err_msg).into());
|
||||
}
|
||||
|
||||
info!("Volume server stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -486,11 +486,6 @@ impl S3TierRegistry {
|
||||
self.backends.keys().cloned().collect()
|
||||
}
|
||||
|
||||
/// Remove a backend by name.
|
||||
pub fn remove(&mut self, name: &str) {
|
||||
self.backends.remove(name);
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.backends.clear();
|
||||
}
|
||||
|
||||
@@ -1867,7 +1867,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
{
|
||||
let needle_header = resp.needle_header;
|
||||
let mut needle_body = resp.needle_body;
|
||||
let resp_version = resp.version;
|
||||
|
||||
if needle_header.is_empty() {
|
||||
continue;
|
||||
@@ -1892,36 +1891,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Parse needle from header + body
|
||||
let mut n = Needle::default();
|
||||
n.read_header(&needle_header);
|
||||
|
||||
if n.size.0 < 0 {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"unexpected negative needle size {} for needle {}",
|
||||
n.size.0, n.id.0
|
||||
)));
|
||||
} else if n.size.0 > 0 {
|
||||
// Normal needle: parse the body fields (DataSize, Data, flags, etc.)
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
} else {
|
||||
// Delete tombstone (size == 0): body is checksum + timestamp
|
||||
// (V3) or checksum only (V2) + padding. Validate minimum
|
||||
// footer length for the protocol version.
|
||||
use crate::storage::types::{
|
||||
NEEDLE_CHECKSUM_SIZE, TIMESTAMP_SIZE, VERSION_3, Version,
|
||||
};
|
||||
let version = Version(resp_version as u8);
|
||||
let min_footer = if version >= VERSION_3 {
|
||||
NEEDLE_CHECKSUM_SIZE + TIMESTAMP_SIZE
|
||||
} else {
|
||||
NEEDLE_CHECKSUM_SIZE
|
||||
};
|
||||
if needle_body.len() < min_footer {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"tombstone needle {} body too short: got {} bytes, need >= {} for version {}",
|
||||
n.id.0, needle_body.len(), min_footer, resp_version
|
||||
)));
|
||||
}
|
||||
}
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
|
||||
// Write needle to local volume
|
||||
let mut store = state.store.write().unwrap();
|
||||
@@ -4076,12 +4047,11 @@ fn find_last_append_at_ns(idx_path: &str, dat_path: &str, version: u32) -> Optio
|
||||
let mut header = [0u8; 16];
|
||||
dat_file.read_exact(&mut header).ok()?;
|
||||
let needle_size = i32::from_be_bytes([header[12], header[13], header[14], header[15]]);
|
||||
if needle_size < 0 {
|
||||
if needle_size <= 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Seek to tail: offset + 16 (header) + size -> checksum (4) + timestamp (8)
|
||||
// For delete needles (size == 0), the tail is right after the header.
|
||||
let tail_offset = actual_offset as u64 + 16 + needle_size as u64;
|
||||
dat_file.seek(SeekFrom::Start(tail_offset)).ok()?;
|
||||
|
||||
@@ -4192,7 +4162,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -4254,7 +4223,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -4264,8 +4232,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -4311,9 +4277,7 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
let tier_config = S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4325,16 +4289,14 @@ mod tests {
|
||||
};
|
||||
{
|
||||
let mut registry = global_s3_tier_registry().write().unwrap();
|
||||
registry.register(
|
||||
"s3.incr_copy_test".to_string(),
|
||||
S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register("s3.default".to_string(), S3TierBackend::new(&tier_config));
|
||||
registry.register("s3".to_string(), S3TierBackend::new(&tier_config));
|
||||
}
|
||||
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
files: vec![crate::storage::volume::VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "incr_copy_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4543,10 +4505,7 @@ mod tests {
|
||||
assert_eq!(copied, dat_bytes[super_block_size as usize..]);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.incr_copy_test");
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1408,8 +1408,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_registers_s3_default_aliases() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently. Just register our entries and verify them.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1441,8 +1443,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_ignores_unsupported_types() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1453,17 +1457,12 @@ mod tests {
|
||||
}],
|
||||
);
|
||||
|
||||
// The per-state registry is freshly created and should have no entries
|
||||
// since "rclone" is unsupported.
|
||||
let registry = state.s3_tier_registry.read().unwrap();
|
||||
assert!(registry.names().is_empty());
|
||||
// Only check that the unsupported type was not added to the global
|
||||
// registry. Other tests may have their own entries present.
|
||||
let global_registry = crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.read()
|
||||
.unwrap();
|
||||
assert!(global_registry.get("rclone.default").is_none());
|
||||
assert!(global_registry.get("rclone").is_none());
|
||||
assert!(global_registry.names().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
#[cfg(unix)]
|
||||
pub mod debug;
|
||||
pub mod grpc_client;
|
||||
pub mod grpc_server;
|
||||
pub mod handlers;
|
||||
pub mod heartbeat;
|
||||
pub mod memory_status;
|
||||
#[cfg(unix)]
|
||||
pub mod profiling;
|
||||
pub mod request_id;
|
||||
pub mod server_stats;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
#[cfg(not(unix))]
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::io::{Seek, SeekFrom};
|
||||
|
||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||
|
||||
|
||||
@@ -3,10 +3,9 @@
|
||||
//! Each EcVolume has a sorted index (.ecx) and a deletion journal (.ecj).
|
||||
//! Shards (.ec00-.ec13) may be distributed across multiple servers.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::sync::RwLock;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::pb::master_pb;
|
||||
@@ -29,17 +28,6 @@ pub struct EcVolume {
|
||||
ecx_file: Option<File>,
|
||||
ecx_file_size: i64,
|
||||
ecj_file: Option<File>,
|
||||
/// On-disk size of the .ecj deletion journal. Used only by IO helpers
|
||||
/// (seek / set_len on partial writes) — the authoritative runtime
|
||||
/// delete count comes from `deleted_needles.len()`.
|
||||
ecj_file_size: i64,
|
||||
/// In-memory set of needle ids that have been deleted since the volume
|
||||
/// was encoded. .ecx is immutable at runtime — it only stores the
|
||||
/// sorted (id, offset, size) index written at encode time — and runtime
|
||||
/// deletes are journaled to .ecj + tracked here. Reads consult this
|
||||
/// set to mask out deleted needles on top of the sealed .ecx lookup.
|
||||
/// Seeded from .ecj in `new()` and updated by `journal_delete`.
|
||||
deleted_needles: RwLock<HashSet<NeedleId>>,
|
||||
pub disk_type: DiskType,
|
||||
/// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
|
||||
ecx_actual_dir: String,
|
||||
@@ -124,8 +112,6 @@ impl EcVolume {
|
||||
ecx_file: None,
|
||||
ecx_file_size: 0,
|
||||
ecj_file: None,
|
||||
ecj_file_size: 0,
|
||||
deleted_needles: RwLock::new(HashSet::new()),
|
||||
disk_type: DiskType::default(),
|
||||
ecx_actual_dir: dir_idx.to_string(),
|
||||
shard_locations: HashMap::new(),
|
||||
@@ -155,12 +141,10 @@ impl EcVolume {
|
||||
}
|
||||
}
|
||||
|
||||
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency.
|
||||
// Note: Go does NOT replay .ecj into .ecx at volume load (RebuildEcxFile
|
||||
// is only invoked from specific decode/rebuild gRPC handlers), so we
|
||||
// don't either. Tombstones from prior sessions were already written
|
||||
// in-place in .ecx, and the journal grows monotonically until a
|
||||
// decode/rebuild operation folds it in.
|
||||
// Replay .ecj journal into .ecx on startup (matches Go's RebuildEcxFile).
|
||||
vol.rebuild_ecx_from_journal()?;
|
||||
|
||||
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency
|
||||
let ecj_base =
|
||||
crate::storage::volume::volume_file_name(&vol.ecx_actual_dir, collection, volume_id);
|
||||
let ecj_path = format!("{}.ecj", ecj_base);
|
||||
@@ -170,75 +154,11 @@ impl EcVolume {
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&ecj_path)?;
|
||||
vol.ecj_file_size = ecj_file.metadata()?.len() as i64;
|
||||
vol.ecj_file = Some(ecj_file);
|
||||
|
||||
// Seed the in-memory deleted set from the journal.
|
||||
vol.load_deleted_needles_from_ecj()?;
|
||||
|
||||
Ok(vol)
|
||||
}
|
||||
|
||||
/// Walk the .ecj journal and populate `deleted_needles`. Called once
|
||||
/// from `new()` under exclusive ownership of the just-constructed
|
||||
/// EcVolume, so locking is not strictly required — but we take the
|
||||
/// write lock anyway for symmetry with later mutations.
|
||||
fn load_deleted_needles_from_ecj(&mut self) -> io::Result<()> {
|
||||
let ecj_file = match self.ecj_file.as_ref() {
|
||||
Some(f) => f,
|
||||
None => return Ok(()),
|
||||
};
|
||||
if self.ecj_file_size < NEEDLE_ID_SIZE as i64 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
let mut set = self
|
||||
.deleted_needles
|
||||
.write()
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::Other, "deleted_needles lock poisoned"))?;
|
||||
let mut off: i64 = 0;
|
||||
while off + NEEDLE_ID_SIZE as i64 <= self.ecj_file_size {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecj_file.read_exact_at(&mut buf, off as u64)?;
|
||||
}
|
||||
set.insert(NeedleId::from_bytes(&buf));
|
||||
off += NEEDLE_ID_SIZE as i64;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns (file_count, delete_count) for this EC volume. Mirrors Go's
|
||||
/// `EcVolume.FileAndDeleteCount`:
|
||||
///
|
||||
/// file_count = ecx_file_size / NEEDLE_MAP_ENTRY_SIZE — total
|
||||
/// entries in the sealed sorted .ecx index.
|
||||
/// delete_count = deleted_needles.len() — unique
|
||||
/// runtime deletes tracked in memory (seeded from
|
||||
/// .ecj on load and updated by `journal_delete`).
|
||||
///
|
||||
/// Because each needle delete is applied on exactly one shard holder,
|
||||
/// the admin aggregation sums delete_count across nodes while taking
|
||||
/// file_count from a single holder (they are identical per volume).
|
||||
pub fn file_and_delete_count(&self) -> (u64, u64) {
|
||||
let file_count = (self.ecx_file_size as u64) / (NEEDLE_MAP_ENTRY_SIZE as u64);
|
||||
let delete_count = self
|
||||
.deleted_needles
|
||||
.read()
|
||||
.map(|s| s.len() as u64)
|
||||
.unwrap_or(0);
|
||||
(file_count, delete_count)
|
||||
}
|
||||
|
||||
/// Reports whether the given needle id is in the in-memory deleted set.
|
||||
pub fn is_needle_deleted(&self, needle_id: NeedleId) -> bool {
|
||||
self.deleted_needles
|
||||
.read()
|
||||
.map(|s| s.contains(&needle_id))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
// ---- File names ----
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -246,16 +166,8 @@ impl EcVolume {
|
||||
crate::storage::volume::volume_file_name(&self.dir, &self.collection, self.volume_id)
|
||||
}
|
||||
|
||||
/// Base path for the .ecx / .ecj index pair. Resolved from
|
||||
/// `ecx_actual_dir` (initialized to `dir_idx` and only updated after a
|
||||
/// successful idx-dir → data-dir fallback in `new()`), so every call site
|
||||
/// agrees on the same file regardless of whether the fallback fired.
|
||||
fn idx_base_name(&self) -> String {
|
||||
crate::storage::volume::volume_file_name(
|
||||
&self.ecx_actual_dir,
|
||||
&self.collection,
|
||||
self.volume_id,
|
||||
)
|
||||
crate::storage::volume::volume_file_name(&self.dir_idx, &self.collection, self.volume_id)
|
||||
}
|
||||
|
||||
pub fn ecx_file_name(&self) -> String {
|
||||
@@ -343,8 +255,6 @@ impl EcVolume {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let (file_count, delete_count) = self.file_and_delete_count();
|
||||
|
||||
vec![master_pb::VolumeEcShardInformationMessage {
|
||||
id: self.volume_id.0,
|
||||
collection: self.collection.clone(),
|
||||
@@ -353,8 +263,6 @@ impl EcVolume {
|
||||
disk_type: self.disk_type.to_string(),
|
||||
expire_at_sec: self.expire_at_sec,
|
||||
disk_id,
|
||||
file_count,
|
||||
delete_count,
|
||||
..Default::default()
|
||||
}]
|
||||
}
|
||||
@@ -405,13 +313,6 @@ impl EcVolume {
|
||||
|
||||
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
// Apply runtime deletion state on top of the sealed .ecx
|
||||
// lookup: a needle in the in-memory deleted set is
|
||||
// reported with TOMBSTONE_FILE_SIZE even though the .ecx
|
||||
// record itself is untouched.
|
||||
if self.is_needle_deleted(needle_id) {
|
||||
return Ok(Some((offset, TOMBSTONE_FILE_SIZE)));
|
||||
}
|
||||
return Ok(Some((offset, size)));
|
||||
} else if key < needle_id {
|
||||
lo = mid + 1;
|
||||
@@ -669,41 +570,38 @@ impl EcVolume {
|
||||
|
||||
// ---- Deletion ----
|
||||
|
||||
/// Write `TOMBSTONE_FILE_SIZE` over the Size field of an existing .ecx
|
||||
/// entry, matching Go's `MarkNeedleDeleted`. Only used by the offline
|
||||
/// `rebuild_ecx_from_journal` path — the runtime delete path does not
|
||||
/// touch .ecx because the index is treated as an immutable sorted
|
||||
/// (id, offset, size) table. Returns `false` if the needle is not in
|
||||
/// the index (ignored by callers) and an error on IO failure.
|
||||
fn tombstone_ecx_entry(&self, needle_id: NeedleId) -> io::Result<bool> {
|
||||
let ecx_file = self.ecx_file.as_ref().ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"ec volume {} has no open .ecx file (closed or corrupt)",
|
||||
self.volume_id.0
|
||||
),
|
||||
)
|
||||
})?;
|
||||
/// Mark a needle as deleted in the .ecx file in-place.
|
||||
/// Matches Go's MarkNeedleDeleted: binary search the .ecx, then overwrite
|
||||
/// the size field with TOMBSTONE_FILE_SIZE.
|
||||
fn mark_needle_deleted_in_ecx(&self, needle_id: NeedleId) -> io::Result<bool> {
|
||||
let ecx_file = match self.ecx_file.as_ref() {
|
||||
Some(f) => f,
|
||||
None => return Ok(false),
|
||||
};
|
||||
|
||||
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
|
||||
if entry_count == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Binary search for the needle
|
||||
let mut lo: usize = 0;
|
||||
let mut hi: usize = entry_count;
|
||||
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
|
||||
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
|
||||
}
|
||||
let (key, _offset, _old_size) = idx_entry_from_bytes(&entry_buf);
|
||||
|
||||
let (key, _offset, _size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
// Found — overwrite the size field with TOMBSTONE_FILE_SIZE
|
||||
let size_offset = file_offset + NEEDLE_ID_SIZE as u64 + OFFSET_SIZE as u64;
|
||||
let mut size_buf = [0u8; SIZE_SIZE];
|
||||
TOMBSTONE_FILE_SIZE.to_bytes(&mut size_buf);
|
||||
@@ -719,18 +617,13 @@ impl EcVolume {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
|
||||
Ok(false) // not found
|
||||
}
|
||||
|
||||
/// Replay .ecj journal entries into .ecx: for each needle id in .ecj,
|
||||
/// overwrite its .ecx size field with a tombstone, then remove the
|
||||
/// journal file. Mirrors Go's `RebuildEcxFile`, which is invoked from
|
||||
/// specific decode / rebuild gRPC handlers — it is intentionally
|
||||
/// **not** called on volume load (runtime reads consult
|
||||
/// `deleted_needles` instead). The rebuild is atomic with respect to
|
||||
/// the journal: if any individual write fails the .ecj file is left
|
||||
/// in place and the error is propagated so tombstones are not lost.
|
||||
#[allow(dead_code)]
|
||||
/// Replay .ecj journal entries into .ecx on startup.
|
||||
/// Matches Go's RebuildEcxFile: for each needle ID in .ecj, marks it
|
||||
/// deleted in .ecx, then removes the .ecj file.
|
||||
fn rebuild_ecx_from_journal(&mut self) -> io::Result<()> {
|
||||
let ecj_path = self.ecj_file_name();
|
||||
if !std::path::Path::new(&ecj_path).exists() {
|
||||
@@ -749,23 +642,14 @@ impl EcVolume {
|
||||
break;
|
||||
}
|
||||
let needle_id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
|
||||
// A needle that never made it into .ecx is fine (e.g. the
|
||||
// delete raced against encode). Any other IO error aborts the
|
||||
// rebuild so the journal survives to be retried later.
|
||||
self.tombstone_ecx_entry(needle_id)?;
|
||||
// Errors for individual entries are non-fatal (needle may not exist in .ecx)
|
||||
let _ = self.mark_needle_deleted_in_ecx(needle_id);
|
||||
}
|
||||
|
||||
// Durably flush the newly-written .ecx tombstones before dropping
|
||||
// the journal: the writes went through write_all_at and may still
|
||||
// be in page cache.
|
||||
if let Some(ref ecx_file) = self.ecx_file {
|
||||
ecx_file.sync_all()?;
|
||||
}
|
||||
// Remove the .ecj file after replay (matches Go)
|
||||
let _ = fs::remove_file(&ecj_path);
|
||||
|
||||
// Fold successful — drop and recreate the journal, clear the
|
||||
// in-memory deleted set (all of its contents are now materialized
|
||||
// in .ecx), and reset the cached size.
|
||||
fs::remove_file(&ecj_path)?;
|
||||
// Re-create .ecj for future deletions
|
||||
let ecj_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
@@ -773,126 +657,27 @@ impl EcVolume {
|
||||
.append(true)
|
||||
.open(&ecj_path)?;
|
||||
self.ecj_file = Some(ecj_file);
|
||||
self.ecj_file_size = 0;
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.clear();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Deletion journal ----
|
||||
|
||||
/// Record a needle delete: append the id to the .ecj deletion journal
|
||||
/// and insert it into the in-memory deleted set. `.ecx` is not touched
|
||||
/// at runtime — it is a sealed sorted (id, offset, size) index and
|
||||
/// runtime deletion state lives exclusively in .ecj + `deleted_needles`.
|
||||
/// A lookup via `find_needle_from_ecx` masks the id out by returning
|
||||
/// `TOMBSTONE_FILE_SIZE` on a subsequent read.
|
||||
///
|
||||
/// The .ecj append is the durable commit point. On any failure the
|
||||
/// file is truncated back to the pre-append length so the on-disk
|
||||
/// journal and in-memory state cannot drift. Only after the sync
|
||||
/// succeeds is the id published into the set, so a failure leaves
|
||||
/// the delete invisible to readers.
|
||||
/// Append a deleted needle ID to the .ecj journal and mark in .ecx.
|
||||
/// Matches Go's DeleteNeedleFromEcx: marks in .ecx first, then journals.
|
||||
pub fn journal_delete(&mut self, needle_id: NeedleId) -> io::Result<()> {
|
||||
// Look the needle up read-only. Missing is a silent no-op; a
|
||||
// pre-existing .ecx tombstone (from a prior decode/rebuild) is
|
||||
// mirrored into the in-memory set so delete_count stays accurate
|
||||
// without needing to walk .ecx on every heartbeat.
|
||||
match self.find_needle_from_ecx_raw(needle_id)? {
|
||||
None => return Ok(()),
|
||||
Some((_, size)) if size.is_deleted() => {
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.insert(needle_id);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
// Mark deleted in .ecx in-place (matches Go's MarkNeedleDeleted)
|
||||
let _ = self.mark_needle_deleted_in_ecx(needle_id);
|
||||
let ecj_file = self
|
||||
.ecj_file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
|
||||
|
||||
// Idempotent fast path for repeat deletes — avoids the journal
|
||||
// append entirely so the derived delete_count stays stable.
|
||||
if self.is_needle_deleted(needle_id) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let prev_ecj_size = self.ecj_file_size;
|
||||
let append_result: io::Result<()> = {
|
||||
let ecj_file = self
|
||||
.ecj_file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
needle_id.to_bytes(&mut buf);
|
||||
ecj_file
|
||||
.write_all(&buf)
|
||||
.and_then(|_| ecj_file.sync_all())
|
||||
};
|
||||
|
||||
match append_result {
|
||||
Ok(()) => {
|
||||
self.ecj_file_size += NEEDLE_ID_SIZE as i64;
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.insert(needle_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
// write_all may have extended the file on disk before
|
||||
// sync_all failed; truncate back to the known-good size so
|
||||
// the on-disk journal never drifts past `deleted_needles`.
|
||||
if let Some(ecj) = self.ecj_file.as_mut() {
|
||||
if let Err(trunc_err) = ecj.set_len(prev_ecj_size as u64) {
|
||||
tracing::error!(
|
||||
volume_id = self.volume_id.0,
|
||||
needle_id = needle_id.0,
|
||||
truncate_error = %trunc_err,
|
||||
"failed to truncate ecj after append failure"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal: binary search .ecx without masking by `deleted_needles`.
|
||||
/// Used by `journal_delete` so a repeat delete can still see the raw
|
||||
/// pre-existing .ecx tombstone from a prior rebuild.
|
||||
fn find_needle_from_ecx_raw(
|
||||
&self,
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<Option<(Offset, Size)>> {
|
||||
let ecx_file = self
|
||||
.ecx_file
|
||||
.as_ref()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecx file not open"))?;
|
||||
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
|
||||
if entry_count == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut lo: usize = 0;
|
||||
let mut hi: usize = entry_count;
|
||||
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
|
||||
}
|
||||
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
return Ok(Some((offset, size)));
|
||||
} else if key < needle_id {
|
||||
lo = mid + 1;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
needle_id.to_bytes(&mut buf);
|
||||
ecj_file.write_all(&buf)?;
|
||||
ecj_file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a deleted needle ID to the .ecj journal, validating the cookie first.
|
||||
@@ -993,17 +778,15 @@ impl EcVolume {
|
||||
let _ = fs::remove_file(format!("{}.ecx", actual_base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", actual_base));
|
||||
let _ = fs::remove_file(format!("{}.vif", actual_base));
|
||||
// Also sweep the originally-configured idx dir in case stale files
|
||||
// exist there (ecx_file_name() / ecj_file_name() now resolve from
|
||||
// ecx_actual_dir, so we have to build the idx-dir paths explicitly).
|
||||
// Also try the configured idx dir and data dir in case files exist in either
|
||||
if self.ecx_actual_dir != self.dir_idx {
|
||||
let _ = fs::remove_file(self.ecx_file_name());
|
||||
let _ = fs::remove_file(self.ecj_file_name());
|
||||
let idx_base = crate::storage::volume::volume_file_name(
|
||||
&self.dir_idx,
|
||||
&self.collection,
|
||||
self.volume_id,
|
||||
);
|
||||
let _ = fs::remove_file(format!("{}.ecx", idx_base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", idx_base));
|
||||
let _ = fs::remove_file(format!("{}.vif", idx_base));
|
||||
}
|
||||
if self.ecx_actual_dir != self.dir && self.dir_idx != self.dir {
|
||||
@@ -1076,33 +859,16 @@ mod tests {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
// .ecj append is gated on a live->tombstone transition in .ecx, so
|
||||
// the fixture must contain the needles we are about to delete.
|
||||
let entries = vec![
|
||||
(NeedleId(10), Offset::from_actual_offset(8), Size(100)),
|
||||
(NeedleId(20), Offset::from_actual_offset(200), Size(200)),
|
||||
];
|
||||
write_ecx_file(dir, "", VolumeId(1), &entries);
|
||||
// Need ecx file for EcVolume::new to succeed
|
||||
write_ecx_file(dir, "", VolumeId(1), &[]);
|
||||
|
||||
let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
|
||||
let (fc0, dc0) = vol.file_and_delete_count();
|
||||
assert_eq!((fc0, dc0), (2, 0));
|
||||
|
||||
vol.journal_delete(NeedleId(10)).unwrap();
|
||||
vol.journal_delete(NeedleId(20)).unwrap();
|
||||
|
||||
let deleted = vol.read_deleted_needles().unwrap();
|
||||
assert_eq!(deleted, vec![NeedleId(10), NeedleId(20)]);
|
||||
|
||||
let (fc, dc) = vol.file_and_delete_count();
|
||||
assert_eq!((fc, dc), (2, 2));
|
||||
|
||||
// Idempotent re-delete must not bump delete_count.
|
||||
vol.journal_delete(NeedleId(10)).unwrap();
|
||||
// Deleting a missing needle must not bump delete_count either.
|
||||
vol.journal_delete(NeedleId(999)).unwrap();
|
||||
let (fc, dc) = vol.file_and_delete_count();
|
||||
assert_eq!((fc, dc), (2, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -19,7 +19,6 @@ use compact_map::CompactMap;
|
||||
use redb::{Database, Durability, ReadableDatabase, ReadableTable, TableDefinition};
|
||||
|
||||
use crate::storage::idx;
|
||||
use crate::storage::needle::needle::get_actual_size;
|
||||
use crate::storage::types::*;
|
||||
|
||||
// ============================================================================
|
||||
@@ -64,11 +63,6 @@ pub struct NeedleMapMetric {
|
||||
pub deletion_count: AtomicI64,
|
||||
pub deletion_byte_count: AtomicU64,
|
||||
pub max_file_key: AtomicU64,
|
||||
/// Largest (offset.to_actual_offset() + get_actual_size(size, version))
|
||||
/// observed during the load walk. Used at volume load to verify that no
|
||||
/// .idx entry references bytes past the end of .dat (issue #8928)
|
||||
/// without paying for a second linear scan.
|
||||
pub max_needle_end: AtomicI64,
|
||||
}
|
||||
|
||||
impl NeedleMapMetric {
|
||||
@@ -114,29 +108,6 @@ impl NeedleMapMetric {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Update `max_needle_end` if this entry's (offset + actual size) exceeds
|
||||
/// the running maximum. Skips deleted/zero-offset entries because they
|
||||
/// don't reserve space in .dat.
|
||||
fn maybe_set_max_needle_end(&self, offset: Offset, size: Size, version: Version) {
|
||||
if offset.is_zero() || !size.is_valid() {
|
||||
return;
|
||||
}
|
||||
let end = offset.to_actual_offset() + get_actual_size(size, version);
|
||||
loop {
|
||||
let current = self.max_needle_end.load(Ordering::Relaxed);
|
||||
if end <= current {
|
||||
break;
|
||||
}
|
||||
if self
|
||||
.max_needle_end
|
||||
.compare_exchange(current, end, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -192,10 +163,9 @@ impl CompactNeedleMap {
|
||||
}
|
||||
|
||||
/// Load from an .idx file, building the in-memory map.
|
||||
pub fn load_from_idx<R: Read + Seek>(reader: &mut R, version: Version) -> io::Result<Self> {
|
||||
pub fn load_from_idx<R: Read + Seek>(reader: &mut R) -> io::Result<Self> {
|
||||
let mut nm = CompactNeedleMap::new();
|
||||
idx::walk_index_file(reader, 0, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
nm.delete_from_map(key);
|
||||
} else {
|
||||
@@ -314,12 +284,6 @@ impl CompactNeedleMap {
|
||||
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
self.metric.max_needle_end.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
self.idx_file_offset
|
||||
}
|
||||
@@ -467,7 +431,7 @@ impl RedbNeedleMap {
|
||||
|
||||
/// Rebuild metrics by scanning all entries in the redb table.
|
||||
/// Called when reusing an existing .rdb without a full rebuild.
|
||||
fn rebuild_metrics_from_db(&self, version: Version) -> io::Result<()> {
|
||||
fn rebuild_metrics_from_db(&self) -> io::Result<()> {
|
||||
let txn = self
|
||||
.db
|
||||
.begin_read()
|
||||
@@ -489,8 +453,6 @@ impl RedbNeedleMap {
|
||||
arr.copy_from_slice(bytes);
|
||||
let nv = unpack_needle_value(&arr);
|
||||
self.metric.maybe_set_max_file_key(key);
|
||||
self.metric
|
||||
.maybe_set_max_needle_end(nv.offset, nv.size, version);
|
||||
if nv.size.is_valid() {
|
||||
self.metric.file_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.metric
|
||||
@@ -515,24 +477,20 @@ impl RedbNeedleMap {
|
||||
/// 2. If .idx size matches → reuse .rdb, rebuild metrics from scan
|
||||
/// 3. If .idx is larger → replay new entries incrementally
|
||||
/// 4. Otherwise (missing, corrupted, .idx smaller) → full rebuild
|
||||
pub fn load_from_idx<R: Read + Seek>(
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
pub fn load_from_idx<R: Read + Seek>(db_path: &str, reader: &mut R) -> io::Result<Self> {
|
||||
let idx_size = reader.seek(io::SeekFrom::End(0))?;
|
||||
reader.seek(io::SeekFrom::Start(0))?;
|
||||
|
||||
// Try to reuse existing .rdb
|
||||
if Path::new(db_path).exists() {
|
||||
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size, version) {
|
||||
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size) {
|
||||
return Ok(nm);
|
||||
}
|
||||
// Reuse failed — fall through to full rebuild
|
||||
reader.seek(io::SeekFrom::Start(0))?;
|
||||
}
|
||||
|
||||
Self::full_rebuild(db_path, reader, idx_size, version)
|
||||
Self::full_rebuild(db_path, reader, idx_size)
|
||||
}
|
||||
|
||||
/// Try to reuse an existing .rdb file. Returns Ok if successful,
|
||||
@@ -541,7 +499,6 @@ impl RedbNeedleMap {
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
idx_size: u64,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
let db = Database::open(db_path)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("redb open: {}", e)))?;
|
||||
@@ -566,7 +523,7 @@ impl RedbNeedleMap {
|
||||
}
|
||||
|
||||
// Rebuild metrics from existing data
|
||||
nm.rebuild_metrics_from_db(version)?;
|
||||
nm.rebuild_metrics_from_db()?;
|
||||
|
||||
if stored_idx_size < idx_size {
|
||||
// .idx grew — replay new entries incrementally
|
||||
@@ -577,7 +534,6 @@ impl RedbNeedleMap {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb open_table: {}", e))
|
||||
})?;
|
||||
idx::walk_index_file(reader, start_entry, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
let key_u64: u64 = key.into();
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
// Delete: look up old value for metric update, then
|
||||
@@ -651,7 +607,6 @@ impl RedbNeedleMap {
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
idx_size: u64,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
let _ = std::fs::remove_file(db_path);
|
||||
let nm = RedbNeedleMap::new(db_path)?;
|
||||
@@ -659,7 +614,6 @@ impl RedbNeedleMap {
|
||||
// Collect entries from idx file, resolving duplicates/deletions
|
||||
let mut entries: HashMap<NeedleId, Option<NeedleValue>> = HashMap::new();
|
||||
idx::walk_index_file(reader, 0, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
entries.insert(key, None);
|
||||
} else {
|
||||
@@ -834,12 +788,6 @@ impl RedbNeedleMap {
|
||||
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
self.metric.max_needle_end.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
self.idx_file_offset
|
||||
}
|
||||
@@ -1040,16 +988,6 @@ impl NeedleMap {
|
||||
}
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. Used at volume load to detect .idx entries that
|
||||
/// reference past the end of .dat (issue #8928) without a second scan.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
match self {
|
||||
NeedleMap::InMemory(nm) => nm.max_needle_end(),
|
||||
NeedleMap::Redb(nm) => nm.max_needle_end(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Index file size in bytes.
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
match self {
|
||||
@@ -1219,7 +1157,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let mut cursor = Cursor::new(idx_data);
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut cursor, Version::current()).unwrap();
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut cursor).unwrap();
|
||||
|
||||
assert!(nm.get(NeedleId(1)).is_some());
|
||||
assert!(nm.get(NeedleId(2)).is_none()); // deleted
|
||||
@@ -1362,7 +1300,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let mut cursor = Cursor::new(idx_data);
|
||||
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor, Version::current()).unwrap();
|
||||
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor).unwrap();
|
||||
|
||||
assert!(nm.get(NeedleId(1)).is_some());
|
||||
assert!(nm.get(NeedleId(2)).is_none()); // deleted and removed
|
||||
@@ -1439,7 +1377,7 @@ mod tests {
|
||||
|
||||
// Load back with CompactNeedleMap to verify
|
||||
let mut idx_file = std::fs::File::open(&idx_path).unwrap();
|
||||
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file, Version::current()).unwrap();
|
||||
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file).unwrap();
|
||||
assert_eq!(loaded.file_count(), 2); // only live entries
|
||||
assert!(loaded.get(NeedleId(1)).is_some());
|
||||
assert!(loaded.get(NeedleId(2)).is_none()); // deleted, not saved
|
||||
|
||||
@@ -1158,13 +1158,7 @@ mod tests {
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// Use a large volume_size_limit so the unused-space difference between
|
||||
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
|
||||
// that integer-division rounding and disk-free fluctuations between the
|
||||
// two get_disk_stats calls cannot make the quotients equal.
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(100 * 1024 * 1024, Ordering::Relaxed);
|
||||
store.volume_size_limit.store(1024, Ordering::Relaxed);
|
||||
store
|
||||
.add_volume(
|
||||
VolumeId(61),
|
||||
|
||||
@@ -19,8 +19,6 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tracing::warn;
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::storage::idx;
|
||||
use crate::storage::needle::needle::{self, get_actual_size, Needle, NeedleError};
|
||||
use crate::storage::needle_map::{CompactNeedleMap, NeedleMap, NeedleMapKind, RedbNeedleMap};
|
||||
use crate::storage::super_block::{ReplicaPlacement, SuperBlock, SUPER_BLOCK_SIZE};
|
||||
@@ -75,24 +73,6 @@ pub enum VolumeError {
|
||||
StreamingUnsupported,
|
||||
}
|
||||
|
||||
/// Returns true when a needle read failed because the on-disk bytes are
|
||||
/// unreadable in a permanent way (offset past EOF, header corruption, CRC
|
||||
/// mismatch, malformed v2/v3/v4 fields). Vacuum can safely drop such entries
|
||||
/// during compaction. Anything else (real disk EIO, ERROR_CRC on Windows,
|
||||
/// network timeouts, EROFS, etc.) is transient or environmental and must
|
||||
/// abort the compaction so an operator notices.
|
||||
fn is_skippable_needle_read_error(e: &VolumeError) -> bool {
|
||||
match e {
|
||||
VolumeError::Io(io_err) => io_err.kind() == io::ErrorKind::UnexpectedEof,
|
||||
VolumeError::Needle(NeedleError::SizeMismatch { .. }) => true,
|
||||
VolumeError::Needle(NeedleError::CrcMismatch { .. }) => true,
|
||||
VolumeError::Needle(NeedleError::IndexOutOfRange(_)) => true,
|
||||
VolumeError::Needle(NeedleError::TailTooShort) => true,
|
||||
VolumeError::SizeMismatch => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// VolumeInfo (.vif persistence)
|
||||
// ============================================================================
|
||||
@@ -732,28 +712,6 @@ impl Volume {
|
||||
"volumeDataIntegrityChecking failed"
|
||||
);
|
||||
}
|
||||
|
||||
// Structural check: no .idx entry may reference bytes past the
|
||||
// end of .dat. The needle map's load walk above already
|
||||
// populated max_needle_end, so this is a numeric comparison
|
||||
// — no extra disk I/O. A violation marks the volume read-only
|
||||
// so vacuum doesn't silently drop reachable data based on a
|
||||
// corrupt .idx left over from a crashed batched write.
|
||||
// See issue #8928.
|
||||
if let Some(ref nm) = self.nm {
|
||||
if let Ok(dat_size) = self.current_dat_file_size() {
|
||||
let max_end = nm.max_needle_end();
|
||||
if dat_size > 0 && max_end > dat_size as i64 {
|
||||
self.no_write_or_delete = true;
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
max_needle_end = max_end,
|
||||
dat_size,
|
||||
"idx references bytes past end of .dat; marking volume read-only"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -801,7 +759,7 @@ impl Volume {
|
||||
// Open read-only
|
||||
if Path::new(&idx_path).exists() {
|
||||
let mut idx_file = File::open(&idx_path)?;
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut idx_file, self.version())?;
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut idx_file)?;
|
||||
self.nm = Some(NeedleMap::InMemory(nm));
|
||||
} else {
|
||||
// Missing .idx with existing .dat could orphan needles
|
||||
@@ -827,7 +785,7 @@ impl Volume {
|
||||
|
||||
let idx_size = idx_file.metadata()?.len();
|
||||
let mut idx_reader = io::BufReader::new(&idx_file);
|
||||
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader, self.version())?;
|
||||
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader)?;
|
||||
|
||||
// Re-open for append-only writes
|
||||
let write_file = OpenOptions::new()
|
||||
@@ -850,7 +808,7 @@ impl Volume {
|
||||
// Open read-only
|
||||
if Path::new(&idx_path).exists() {
|
||||
let mut idx_file = File::open(&idx_path)?;
|
||||
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file, self.version())?;
|
||||
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file)?;
|
||||
self.nm = Some(NeedleMap::Redb(nm));
|
||||
} else {
|
||||
// Missing .idx with existing .dat could orphan needles
|
||||
@@ -876,7 +834,7 @@ impl Volume {
|
||||
|
||||
let idx_size = idx_file.metadata()?.len();
|
||||
let mut idx_reader = io::BufReader::new(&idx_file);
|
||||
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader, self.version())?;
|
||||
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader)?;
|
||||
|
||||
// Re-open for append-only writes
|
||||
let write_file = OpenOptions::new()
|
||||
@@ -1794,11 +1752,6 @@ impl Volume {
|
||||
|
||||
let version = self.version();
|
||||
|
||||
// The deeper-than-tail structural check (every (offset + actual size)
|
||||
// fits inside .dat — issue #8928) is now handled in load() via the
|
||||
// needle map's max_needle_end accumulator, so we don't pay for a
|
||||
// second linear scan of the .idx here.
|
||||
|
||||
// Check last 10 index entries (matching Go's CheckVolumeDataIntegrity).
|
||||
// Go starts healthyIndexSize = indexSize and reduces on EOF.
|
||||
// On success: break (err != ErrorSizeMismatch when err == nil).
|
||||
@@ -2730,8 +2683,6 @@ impl Volume {
|
||||
}
|
||||
entries.sort_by_key(|(_, offset, _)| *offset);
|
||||
|
||||
let mut skipped_needles: u64 = 0;
|
||||
let mut skipped_data_bytes: u64 = 0;
|
||||
for (id, offset, size) in entries {
|
||||
// Progress callback
|
||||
if !progress_fn(offset.to_actual_offset()) {
|
||||
@@ -2748,41 +2699,7 @@ impl Volume {
|
||||
id,
|
||||
..Needle::default()
|
||||
};
|
||||
match self.read_needle_data_at(&mut n, offset.to_actual_offset(), size) {
|
||||
Ok(()) => {}
|
||||
Err(e) => {
|
||||
// Record EIO for health monitoring (parity with Go's checkReadWriteError).
|
||||
if let VolumeError::Io(ref io_err) = e {
|
||||
self.check_read_write_error(Some(io_err));
|
||||
}
|
||||
// Only drop the entry when the failure is one of the well-
|
||||
// known permanent-corruption shapes. A transient disk fault,
|
||||
// a tiered-read timeout, or a Windows hardware error (which
|
||||
// surfaces as a generic Io rather than UnexpectedEof) must
|
||||
// abort so an operator notices, rather than silently
|
||||
// compacting away data that might come back on retry.
|
||||
// See issue #8928.
|
||||
if !is_skippable_needle_read_error(&e) {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("cannot hydrate needle from file: {}", e),
|
||||
)));
|
||||
}
|
||||
skipped_needles += 1;
|
||||
if size.is_valid() {
|
||||
skipped_data_bytes += size.0 as u64;
|
||||
}
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
key = id.0,
|
||||
offset = offset.to_actual_offset(),
|
||||
size = size.0,
|
||||
error = %e,
|
||||
"vacuum: dropping unreadable needle"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
self.read_needle_data_at(&mut n, offset.to_actual_offset(), size)?;
|
||||
|
||||
// Skip TTL-expired needles using the volume's TTL (matches Go's volume_vacuum.go)
|
||||
if n.has_ttl() {
|
||||
@@ -2804,15 +2721,6 @@ impl Volume {
|
||||
new_offset += bytes.len() as i64;
|
||||
}
|
||||
|
||||
if skipped_needles > 0 {
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
skipped_needles,
|
||||
skipped_data_bytes,
|
||||
"vacuum: dropped unreadable index entries during compaction"
|
||||
);
|
||||
}
|
||||
|
||||
dst.sync_all()?;
|
||||
|
||||
// Save new index
|
||||
@@ -3317,7 +3225,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -3372,7 +3279,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -3382,8 +3288,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -3802,138 +3706,6 @@ mod tests {
|
||||
v.cleanup_compact().unwrap();
|
||||
}
|
||||
|
||||
/// Vacuum compaction must tolerate an .idx entry whose offset points past
|
||||
/// the end of the .dat file (the failure mode in issue #8928). The bad
|
||||
/// entry is silently dropped from the resulting .cpx; healthy needles
|
||||
/// survive untouched.
|
||||
#[test]
|
||||
fn test_compact_by_index_drops_dangling_needle() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
// Write a handful of healthy needles to establish a baseline .dat/.idx.
|
||||
for i in 1..=5u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: format!("payload-{}", i).into_bytes(),
|
||||
data_size: format!("payload-{}", i).len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
let dat_size = v.dat_file_size().unwrap();
|
||||
let bad_key = NeedleId(9999);
|
||||
let bad_offset = Offset::from_actual_offset((dat_size + 1024 * 1024) as i64);
|
||||
let bad_size = Size(2048);
|
||||
v.nm
|
||||
.as_mut()
|
||||
.expect("needle map present")
|
||||
.put(bad_key, bad_offset, bad_size)
|
||||
.unwrap();
|
||||
v.nm.as_ref().unwrap().sync().unwrap();
|
||||
|
||||
// Vacuum must succeed in spite of the dangling entry.
|
||||
v.compact_by_index(0, 0, |_| true)
|
||||
.expect("compact_by_index should tolerate dangling entries");
|
||||
|
||||
// Walk the resulting .cpx and confirm the bad key was dropped while
|
||||
// every healthy key made it through.
|
||||
let cpx_path = v.file_name(".cpx");
|
||||
let mut cpx = File::open(&cpx_path).unwrap();
|
||||
let mut kept: Vec<u64> = Vec::new();
|
||||
idx::walk_index_file(&mut cpx, 0, |key, _, size| {
|
||||
if !size.is_deleted() {
|
||||
kept.push(key.0);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
!kept.contains(&bad_key.0),
|
||||
"dangling key {} should have been dropped from .cpx, got {:?}",
|
||||
bad_key.0,
|
||||
kept
|
||||
);
|
||||
for i in 1..=5u64 {
|
||||
assert!(
|
||||
kept.contains(&i),
|
||||
"healthy key {} missing from compacted .cpx, got {:?}",
|
||||
i,
|
||||
kept
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The needle map's max_needle_end accumulator must let volume.load
|
||||
/// detect an .idx whose entries point past the end of the .dat — the
|
||||
/// deeper-than-tail corruption shape from issue #8928 that the existing
|
||||
/// last-10-entries scan cannot see. The check is populated by the load
|
||||
/// walk and read in volume.load() to flip the volume read-only.
|
||||
#[test]
|
||||
fn test_max_needle_end_detects_dangling_entry() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
for i in 1..=4u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: format!("data-{}", i).into_bytes(),
|
||||
data_size: format!("data-{}", i).len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
let dat_size = v.dat_file_size().unwrap() as i64;
|
||||
let idx_path = v.file_name(".idx");
|
||||
let version = v.version();
|
||||
|
||||
// Sanity: a fresh load walk over the healthy .idx puts max_needle_end
|
||||
// somewhere inside the .dat.
|
||||
let mut idx_reader = File::open(&idx_path).unwrap();
|
||||
let healthy_nm =
|
||||
CompactNeedleMap::load_from_idx(&mut idx_reader, version).unwrap();
|
||||
let healthy_end = healthy_nm.max_needle_end();
|
||||
assert!(
|
||||
healthy_end > 0 && healthy_end <= dat_size,
|
||||
"healthy volume should have max_needle_end ({}) in [1, dat_size={}]",
|
||||
healthy_end,
|
||||
dat_size
|
||||
);
|
||||
|
||||
// Inject a dangling entry by appending a bogus 16/17-byte record
|
||||
// directly to the .idx, then reload. The load walk should observe
|
||||
// max_needle_end past dat_size — which is exactly the signal
|
||||
// volume.load uses to mark the volume read-only.
|
||||
let bad_offset = Offset::from_actual_offset(dat_size + 4 * 1024 * 1024);
|
||||
let mut idx_append = OpenOptions::new()
|
||||
.write(true)
|
||||
.append(true)
|
||||
.open(&idx_path)
|
||||
.unwrap();
|
||||
idx::write_index_entry(&mut idx_append, NeedleId(9999), bad_offset, Size(1024))
|
||||
.unwrap();
|
||||
idx_append.sync_all().unwrap();
|
||||
|
||||
let mut idx_reread = File::open(&idx_path).unwrap();
|
||||
let bad_nm = CompactNeedleMap::load_from_idx(&mut idx_reread, version).unwrap();
|
||||
let bad_end = bad_nm.max_needle_end();
|
||||
assert!(
|
||||
bad_end > dat_size,
|
||||
"after dangling-entry inject max_needle_end ({}) should exceed dat_size ({})",
|
||||
bad_end,
|
||||
dat_size
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compaction_revision_relookup() {
|
||||
// Verifies that re_lookup_needle_data_offset returns the correct data offset
|
||||
@@ -4061,7 +3833,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "vif_rw_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: v.dat_file_size().unwrap(),
|
||||
@@ -4077,27 +3849,6 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Register a dummy S3 backend so load_remote_dat_file can look it
|
||||
// up. This test never reads remote data, so a dead endpoint is fine.
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry.
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket-a".to_string(),
|
||||
endpoint: "http://127.0.0.1:1".to_string(),
|
||||
storage_class: "STANDARD".to_string(),
|
||||
force_path_style: true,
|
||||
};
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.register(
|
||||
"s3.vif_rw_test".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
|
||||
v.dat_file_size().unwrap()
|
||||
};
|
||||
|
||||
@@ -4141,11 +3892,6 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(deleted_size.0 > 0);
|
||||
assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
|
||||
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.vif_rw_test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4308,9 +4054,10 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4325,7 +4072,11 @@ mod tests {
|
||||
.write()
|
||||
.unwrap();
|
||||
registry.register(
|
||||
"s3.remote_only_rw".to_string(),
|
||||
"s3.default".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register(
|
||||
"s3".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
}
|
||||
@@ -4333,7 +4084,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "remote_only_rw".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4390,10 +4141,6 @@ mod tests {
|
||||
assert_eq!(meta.data_size, 11);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.remote_only_rw");
|
||||
}
|
||||
|
||||
/// Volume destroy removes .vif alongside the primary data files.
|
||||
|
||||
@@ -33,11 +33,11 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/image v0.36.0 // indirect
|
||||
golang.org/x/net v0.50.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
||||
@@ -27,8 +27,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -84,35 +84,35 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
|
||||
+2
-2
@@ -999,9 +999,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.4"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -23,16 +22,10 @@ const (
|
||||
FilerUrl = "http://localhost:8888"
|
||||
)
|
||||
|
||||
// Helper to run commands in background and track PIDs for cleanup. Guarded
|
||||
// by runningCmdsLock so parallel subprocess startup can append safely.
|
||||
var (
|
||||
runningCmds []*exec.Cmd
|
||||
runningCmdsLock sync.Mutex
|
||||
)
|
||||
// Helper to run commands in background and track PIDs for cleanup
|
||||
var runningCmds []*exec.Cmd
|
||||
|
||||
func cleanup() {
|
||||
runningCmdsLock.Lock()
|
||||
defer runningCmdsLock.Unlock()
|
||||
for _, cmd := range runningCmds {
|
||||
if cmd.Process != nil {
|
||||
cmd.Process.Kill()
|
||||
@@ -66,9 +59,7 @@ func startWeed(t *testing.T, name string, args ...string) *exec.Cmd {
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start weed %v: %v", args, err)
|
||||
}
|
||||
runningCmdsLock.Lock()
|
||||
runningCmds = append(runningCmds, cmd)
|
||||
runningCmdsLock.Unlock()
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -116,23 +107,14 @@ func ensureEnvironment(t *testing.T) {
|
||||
waitForUrl(t, MasterUrl+"/cluster/status", 10)
|
||||
|
||||
// 3. Start Volume Server (Worker)
|
||||
// Start 14 volume servers to verify RS(10,4) default EC. Fork/exec in
|
||||
// parallel because startWeed is non-blocking and the per-process fork +
|
||||
// mkdir + log-file-open overhead stacks up sequentially on cold CI
|
||||
// disks, eating most of the admin /health wait budget further down.
|
||||
var volWg sync.WaitGroup
|
||||
// Start 14 volume servers to verify RS(10,4) default EC
|
||||
for i := 1; i <= 14; i++ {
|
||||
volWg.Add(1)
|
||||
go func(i int) {
|
||||
defer volWg.Done()
|
||||
volName := fmt.Sprintf("volume%d", i)
|
||||
port := 8080 + i - 1
|
||||
dir := filepath.Join("tmp", volName)
|
||||
os.MkdirAll(dir, 0755)
|
||||
startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost")
|
||||
}(i)
|
||||
volName := fmt.Sprintf("volume%d", i)
|
||||
port := 8080 + i - 1
|
||||
dir := filepath.Join("tmp", volName)
|
||||
os.MkdirAll(dir, 0755)
|
||||
startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost")
|
||||
}
|
||||
volWg.Wait()
|
||||
|
||||
// 4. Start Filer
|
||||
os.MkdirAll(filepath.Join("tmp", "filer"), 0755)
|
||||
@@ -154,12 +136,7 @@ func ensureEnvironment(t *testing.T) {
|
||||
os.RemoveAll(filepath.Join("tmp", "admin"))
|
||||
os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
|
||||
startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
|
||||
// Admin is started after master, 14 volume servers, filer and 2 workers,
|
||||
// so under cold CI conditions the wait here has to absorb the tail of
|
||||
// every earlier subprocess coming up. 60s is too tight and has flaked;
|
||||
// 180s gives comfortable headroom without meaningfully extending the
|
||||
// fast path (the first successful /health usually hits well under 30s).
|
||||
waitForUrl(t, AdminUrl+"/health", 180)
|
||||
waitForUrl(t, AdminUrl+"/health", 60)
|
||||
|
||||
t.Log("Environment started successfully")
|
||||
}
|
||||
|
||||
@@ -1,277 +0,0 @@
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same
|
||||
// file concurrently produce valid (non-corrupted) data. With DLM enabled, the
|
||||
// writes are serialized — one blocks until the other completes.
|
||||
//
|
||||
// Note: cross-mount read consistency depends on FUSE kernel cache invalidation
|
||||
// and filer metadata subscription, which are asynchronous. This test verifies
|
||||
// write integrity, not instant read convergence.
|
||||
func TestDLMConcurrentWritersSameFile(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
const iterations = 5
|
||||
for iter := 0; iter < iterations; iter++ {
|
||||
fileName := fmt.Sprintf("concurrent_write_%d.txt", iter)
|
||||
payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter))
|
||||
payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644)
|
||||
assert.NoError(t, err, "mount0 write iteration %d", iter)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644)
|
||||
assert.NoError(t, err, "mount1 write iteration %d", iter)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Verify file is readable and contains one of the expected payloads
|
||||
// (read from mount0 — its own view is authoritative for write success).
|
||||
content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
|
||||
require.NoError(t, err, "read from mount0 iteration %d", iter)
|
||||
validPayload := string(content) == string(payloadA) || string(content) == string(payloadB)
|
||||
assert.True(t, validPayload,
|
||||
"iteration %d: content must be one of the expected payloads, got: %q", iter, content)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles
|
||||
// from both mounts all succeed without errors.
|
||||
func TestDLMRepeatedOpenWriteClose(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
const cycles = 20
|
||||
fileName := "repeated_write.txt"
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < cycles; i++ {
|
||||
data := []byte(fmt.Sprintf("mount0-cycle-%d", i))
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644)
|
||||
assert.NoError(t, err, "mount0 cycle %d", i)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < cycles; i++ {
|
||||
data := []byte(fmt.Sprintf("mount1-cycle-%d", i))
|
||||
err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644)
|
||||
assert.NoError(t, err, "mount1 cycle %d", i)
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// File must be readable from at least one mount
|
||||
content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, content, "file must not be empty")
|
||||
}
|
||||
|
||||
// TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one
|
||||
// mount has a file open for writing, another mount's write-open blocks until
|
||||
// the first mount closes the file.
|
||||
func TestDLMWriteBlocksSecondWriter(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
fileName := "blocking_test.txt"
|
||||
path0 := filepath.Join(cluster.mountPoints[0], fileName)
|
||||
path1 := filepath.Join(cluster.mountPoints[1], fileName)
|
||||
|
||||
// Mount 0 opens the file for writing and holds it open
|
||||
f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
require.NoError(t, err, "mount0 open")
|
||||
_, err = f.Write([]byte("mount0-holds-lock"))
|
||||
require.NoError(t, err, "mount0 write")
|
||||
|
||||
// Mount 1 tries to write — should block (we use a goroutine with atomic flag)
|
||||
var mount1Completed atomic.Bool
|
||||
mount1Done := make(chan error, 1)
|
||||
go func() {
|
||||
err := os.WriteFile(path1, []byte("mount1-waited"), 0644)
|
||||
mount1Completed.Store(true)
|
||||
mount1Done <- err
|
||||
}()
|
||||
|
||||
// Give mount 1 a moment — it should NOT complete while mount 0 holds the file open
|
||||
time.Sleep(3 * time.Second)
|
||||
require.False(t, mount1Completed.Load(),
|
||||
"mount1 write must not complete while mount0 holds the file open")
|
||||
t.Log("mount1 write is blocked as expected while mount0 holds the file")
|
||||
|
||||
// Mount 0 closes the file — this releases the DLM lock
|
||||
require.NoError(t, f.Close(), "mount0 close")
|
||||
|
||||
// Mount 1 should now complete
|
||||
select {
|
||||
case err := <-mount1Done:
|
||||
assert.NoError(t, err, "mount1 write after mount0 close")
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("mount1 write did not complete within 30s after mount0 closed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM:
|
||||
// while mount0 has a file open for writing (re-opened after creation),
|
||||
// mount1 cannot rename it until mount0 closes the file.
|
||||
func TestDLMRenameWhileWriteOpen(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
origName := "rename_source.txt"
|
||||
newName := "rename_dest.txt"
|
||||
|
||||
// Create and close the file first so it's flushed to the filer and
|
||||
// visible on both mounts.
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(cluster.mountPoints[0], origName),
|
||||
[]byte("initial-content"), 0644))
|
||||
time.Sleep(2 * time.Second) // metadata propagation
|
||||
|
||||
// Verify mount1 can see the file
|
||||
_, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName))
|
||||
require.NoError(t, err, "mount1 should see the file")
|
||||
|
||||
// Mount 0 re-opens the file for writing and holds it open
|
||||
f, err := os.OpenFile(
|
||||
filepath.Join(cluster.mountPoints[0], origName),
|
||||
os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
require.NoError(t, err, "mount0 reopen")
|
||||
_, err = f.Write([]byte("data-while-holding-lock"))
|
||||
require.NoError(t, err, "mount0 write")
|
||||
|
||||
// Mount 1 tries to rename — should block because mount0 holds the
|
||||
// DLM lock on the old path
|
||||
var renameCompleted atomic.Bool
|
||||
renameDone := make(chan error, 1)
|
||||
go func() {
|
||||
err := os.Rename(
|
||||
filepath.Join(cluster.mountPoints[1], origName),
|
||||
filepath.Join(cluster.mountPoints[1], newName))
|
||||
renameCompleted.Store(true)
|
||||
renameDone <- err
|
||||
}()
|
||||
|
||||
// Rename must NOT complete while mount0 holds the file open
|
||||
time.Sleep(3 * time.Second)
|
||||
require.False(t, renameCompleted.Load(),
|
||||
"rename must not complete while mount0 holds the file open")
|
||||
t.Log("rename is blocked as expected while mount0 holds the file")
|
||||
|
||||
// Mount 0 closes → releases DLM lock → rename should proceed
|
||||
require.NoError(t, f.Close(), "mount0 close")
|
||||
|
||||
select {
|
||||
case err := <-renameDone:
|
||||
assert.NoError(t, err, "rename after mount0 close")
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("rename did not complete within 30s after mount0 closed")
|
||||
}
|
||||
}
|
||||
|
||||
// Note: Same-mount rename while a file is open for writing is not tested here
|
||||
// because macOS FUSE serializes operations on the same inode, causing a
|
||||
// kernel-level deadlock between the Rename handler's internal flush and the
|
||||
// pending Close. Same-mount coordination is already handled by the per-mount
|
||||
// fhLockTable and FUSE kernel serialization, so DLM is not needed for it.
|
||||
|
||||
// TestDLMConcurrentRenames verifies that two concurrent renames of the same
|
||||
// file from different mounts don't corrupt metadata. DLM locks on both old
|
||||
// and new paths ensure renames are serialized.
|
||||
func TestDLMConcurrentRenames(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping DLM integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := startDLMTestCluster(t)
|
||||
t.Cleanup(cluster.Stop)
|
||||
|
||||
// Create a file first
|
||||
origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt")
|
||||
require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644))
|
||||
time.Sleep(1 * time.Second) // propagation
|
||||
|
||||
// Both mounts try to rename the same file concurrently
|
||||
var wg sync.WaitGroup
|
||||
var errA, errB error
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errA = os.Rename(
|
||||
filepath.Join(cluster.mountPoints[0], "rename_race.txt"),
|
||||
filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"),
|
||||
)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errB = os.Rename(
|
||||
filepath.Join(cluster.mountPoints[1], "rename_race.txt"),
|
||||
filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"),
|
||||
)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// At least one rename should succeed; the other may fail with ENOENT
|
||||
// since the source was already moved.
|
||||
succeeded := 0
|
||||
if errA == nil {
|
||||
succeeded++
|
||||
t.Logf("mount0 rename succeeded")
|
||||
} else {
|
||||
t.Logf("mount0 rename failed: %v", errA)
|
||||
}
|
||||
if errB == nil {
|
||||
succeeded++
|
||||
t.Logf("mount1 rename succeeded")
|
||||
} else {
|
||||
t.Logf("mount1 rename failed: %v", errB)
|
||||
}
|
||||
assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed")
|
||||
}
|
||||
@@ -1,516 +0,0 @@
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
const filerGroup = "fuse-dlm-test"
|
||||
|
||||
// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
|
||||
// mounts for testing DLM-based cross-mount write coordination.
|
||||
type dlmTestCluster struct {
|
||||
t testing.TB
|
||||
baseDir string
|
||||
weedBinary string
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
volumePort int
|
||||
volumeGrpcPort int
|
||||
filerPorts [2]int
|
||||
filerGrpcPorts [2]int
|
||||
mountPoints [2]string
|
||||
|
||||
masterCmd *exec.Cmd
|
||||
volumeCmd *exec.Cmd
|
||||
filerCmds [2]*exec.Cmd
|
||||
mountCmds [2]*exec.Cmd
|
||||
logFiles []*os.File
|
||||
|
||||
cleanupOnce sync.Once
|
||||
}
|
||||
|
||||
func startDLMTestCluster(t testing.TB) *dlmTestCluster {
|
||||
binary := findWeedBinary()
|
||||
if binary == "" {
|
||||
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
|
||||
}
|
||||
|
||||
baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_dlm_test_")
|
||||
require.NoError(t, err)
|
||||
|
||||
c := &dlmTestCluster{
|
||||
t: t,
|
||||
baseDir: baseDir,
|
||||
weedBinary: binary,
|
||||
}
|
||||
// Register cleanup early so processes are stopped even if a require fails below.
|
||||
t.Cleanup(c.Stop)
|
||||
|
||||
// Allocate ports: master(2) + volume(2) + filer0(2) + filer1(2) = 8
|
||||
ports := allocatePorts(t, 8)
|
||||
c.masterPort = ports[0]
|
||||
c.masterGrpcPort = ports[1]
|
||||
c.volumePort = ports[2]
|
||||
c.volumeGrpcPort = ports[3]
|
||||
c.filerPorts[0] = ports[4]
|
||||
c.filerGrpcPorts[0] = ports[5]
|
||||
c.filerPorts[1] = ports[6]
|
||||
c.filerGrpcPorts[1] = ports[7]
|
||||
|
||||
// Write empty security.toml
|
||||
configDir := filepath.Join(baseDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
|
||||
|
||||
// Start master
|
||||
require.NoError(t, c.startMaster(configDir))
|
||||
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second),
|
||||
"master not ready\n%s", c.tailLog("master"))
|
||||
|
||||
// Start volume
|
||||
require.NoError(t, c.startVolume(configDir))
|
||||
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second),
|
||||
"volume not ready\n%s", c.tailLog("volume"))
|
||||
|
||||
// Start 2 filers
|
||||
for i := 0; i < 2; i++ {
|
||||
require.NoError(t, c.startFiler(i, configDir))
|
||||
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[i]), 30*time.Second),
|
||||
"filer %d not ready\n%s", i, c.tailLog(fmt.Sprintf("filer%d", i)))
|
||||
}
|
||||
require.NoError(t, c.waitForFilerCount(2, 30*time.Second), "filer group registration")
|
||||
require.NoError(t, c.waitForLockRingConverged(30*time.Second), "lock ring convergence")
|
||||
|
||||
// Start 2 mounts, both pointing at filer0 for metadata consistency.
|
||||
// (filer1 exists for the DLM lock ring but both mounts share filer0's
|
||||
// metadata store since leveldb is per-filer.)
|
||||
for i := 0; i < 2; i++ {
|
||||
mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
|
||||
require.NoError(t, os.MkdirAll(mp, 0755))
|
||||
c.mountPoints[i] = mp
|
||||
require.NoError(t, c.startMount(i, configDir))
|
||||
require.NoError(t, c.waitForMount(mp, 30*time.Second),
|
||||
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) Stop() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.cleanupOnce.Do(func() {
|
||||
// Stop mounts first (triggers flush + DLM unlock).
|
||||
// Use stopCmd for bounded wait to avoid hanging on wedged FUSE processes.
|
||||
for i := 1; i >= 0; i-- {
|
||||
stopCmd(c.mountCmds[i])
|
||||
// Backup unmount in case FUSE teardown didn't clean up
|
||||
exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
|
||||
exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
|
||||
}
|
||||
// Stop filers, volume, master
|
||||
for i := 1; i >= 0; i-- {
|
||||
stopCmd(c.filerCmds[i])
|
||||
}
|
||||
stopCmd(c.volumeCmd)
|
||||
stopCmd(c.masterCmd)
|
||||
|
||||
for _, f := range c.logFiles {
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Copy logs for CI
|
||||
c.copyLogsForCI()
|
||||
|
||||
if !c.t.Failed() {
|
||||
os.RemoveAll(c.baseDir)
|
||||
}
|
||||
|
||||
// Wait for ports to be fully released before the next test
|
||||
// allocates new ports (avoids TIME_WAIT collisions).
|
||||
time.Sleep(2 * time.Second)
|
||||
})
|
||||
}
|
||||
|
||||
// masterAddress returns the master address in the format that encodes both
|
||||
// HTTP and gRPC ports: "host:httpPort.grpcPort". This is the format that
|
||||
// SeaweedFS uses to communicate non-default gRPC ports between components.
|
||||
func (c *dlmTestCluster) masterAddress() string {
|
||||
return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startMaster(configDir string) error {
|
||||
c.masterCmd = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"master",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.masterPort),
|
||||
"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
|
||||
"-mdir="+filepath.Join(c.baseDir, "master"),
|
||||
)
|
||||
return c.startCmd(c.masterCmd, "master")
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startVolume(configDir string) error {
|
||||
volDir := filepath.Join(c.baseDir, "volume")
|
||||
if err := os.MkdirAll(volDir, 0755); err != nil {
|
||||
return fmt.Errorf("create volume dir: %w", err)
|
||||
}
|
||||
c.volumeCmd = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"volume",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.volumePort),
|
||||
"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
|
||||
"-master="+c.masterAddress(),
|
||||
"-dir="+volDir,
|
||||
"-max=10",
|
||||
)
|
||||
return c.startCmd(c.volumeCmd, "volume")
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startFiler(idx int, configDir string) error {
|
||||
filerDir := filepath.Join(c.baseDir, fmt.Sprintf("filer%d", idx))
|
||||
if err := os.MkdirAll(filerDir, 0755); err != nil {
|
||||
return fmt.Errorf("create filer dir: %w", err)
|
||||
}
|
||||
c.filerCmds[idx] = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"filer",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.filerPorts[idx]),
|
||||
"-port.grpc="+strconv.Itoa(c.filerGrpcPorts[idx]),
|
||||
"-master="+c.masterAddress(),
|
||||
"-filerGroup="+filerGroup,
|
||||
"-defaultStoreDir="+filerDir,
|
||||
)
|
||||
return c.startCmd(c.filerCmds[idx], fmt.Sprintf("filer%d", idx))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) filerAddress(idx int) string {
|
||||
return string(pb.NewServerAddress("127.0.0.1", c.filerPorts[idx], c.filerGrpcPorts[idx]))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startMount(idx int, configDir string) error {
|
||||
cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("create cache dir: %w", err)
|
||||
}
|
||||
c.mountCmds[idx] = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"mount",
|
||||
"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
|
||||
"-dir="+c.mountPoints[idx],
|
||||
"-filer.path=/",
|
||||
"-dirAutoCreate",
|
||||
"-allowOthers=false",
|
||||
"-cacheDir="+cacheDir,
|
||||
"-dlm",
|
||||
)
|
||||
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) startCmd(cmd *exec.Cmd, name string) error {
|
||||
logPath := filepath.Join(c.baseDir, "logs")
|
||||
if err := os.MkdirAll(logPath, 0755); err != nil {
|
||||
return fmt.Errorf("create log dir: %w", err)
|
||||
}
|
||||
logFile, err := os.Create(filepath.Join(logPath, name+".log"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.logFiles = append(c.logFiles, logFile)
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
return cmd.Start()
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) tailLog(name string) string {
|
||||
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
|
||||
if err != nil {
|
||||
return fmt.Sprintf("(log %s not available: %v)", name, err)
|
||||
}
|
||||
const maxTail = 8192
|
||||
if len(data) > maxTail {
|
||||
data = data[len(data)-maxTail:]
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) copyLogsForCI() {
|
||||
ciLogDir := "/tmp/seaweedfs-fuse-dlm-logs"
|
||||
os.MkdirAll(ciLogDir, 0755)
|
||||
logsDir := filepath.Join(c.baseDir, "logs")
|
||||
entries, err := os.ReadDir(logsDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) waitForTCP(addr string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", addr, time.Second)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("service at %s not ready within timeout", addr)
|
||||
}
|
||||
|
||||
// waitForMount waits for a FUSE filesystem to actually be mounted at
|
||||
// mountPoint by comparing the device ID of the mount point against its parent.
|
||||
// A plain directory (pre-created before mount) has the same device as its
|
||||
// parent; a mounted FUSE filesystem has a different device.
|
||||
func (c *dlmTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
|
||||
parentDir := filepath.Dir(mountPoint)
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
parentStat, err := os.Stat(parentDir)
|
||||
if err != nil {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
mountStat, err := os.Stat(mountPoint)
|
||||
if err != nil {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
parentSys := parentStat.Sys().(*syscall.Stat_t)
|
||||
mountSys := mountStat.Sys().(*syscall.Stat_t)
|
||||
if parentSys.Dev != mountSys.Dev {
|
||||
// Different device = FUSE mounted
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) filerGRPCAddress(idx int) string {
|
||||
return fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[idx])
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) waitForFilerCount(expected int, timeout time.Duration) error {
|
||||
addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
|
||||
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := master_pb.NewSeaweedClient(conn)
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
|
||||
ClientType: "filer",
|
||||
FilerGroup: filerGroup,
|
||||
})
|
||||
cancel()
|
||||
if err == nil && len(resp.ClusterNodes) >= expected {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for %d filers in group %q", expected, filerGroup)
|
||||
}
|
||||
|
||||
// waitForLockRingConverged verifies that both filers have a consistent view of
|
||||
// the lock ring by acquiring the same lock through each filer and checking
|
||||
// mutual exclusion. Adapted from test/s3/distributed_lock/.
|
||||
func (c *dlmTestCluster) waitForLockRingConverged(timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
owners := []pb.ServerAddress{
|
||||
pb.ServerAddress(c.filerGRPCAddress(0)),
|
||||
pb.ServerAddress(c.filerGRPCAddress(1)),
|
||||
}
|
||||
ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
|
||||
ring.SetServers(owners)
|
||||
|
||||
attempt := 0
|
||||
for time.Now().Before(deadline) {
|
||||
testKeys := convergenceKeysPerPrimary(ring, owners, attempt)
|
||||
attempt++
|
||||
|
||||
allConverged := true
|
||||
for _, key := range testKeys {
|
||||
converged, _ := c.checkLockMutualExclusion(key)
|
||||
if !converged {
|
||||
allConverged = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allConverged {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("lock ring did not converge")
|
||||
}
|
||||
|
||||
// convergenceKeysPerPrimary generates one test key per primary filer.
|
||||
func convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
|
||||
found := make(map[pb.ServerAddress]bool)
|
||||
var keys []string
|
||||
for i := 0; len(found) < len(owners) && i < 10000; i++ {
|
||||
key := fmt.Sprintf("convergence-test-%d-%d", attempt, i)
|
||||
primary, _ := ring.GetPrimaryAndBackup(key)
|
||||
if !found[primary] {
|
||||
found[primary] = true
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func (c *dlmTestCluster) checkLockMutualExclusion(key string) (bool, error) {
|
||||
// Try to lock via filer0
|
||||
conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer conn0.Close()
|
||||
|
||||
client0 := filer_pb.NewSeaweedFilerClient(conn0)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
resp0, err := client0.DistributedLock(ctx, &filer_pb.LockRequest{
|
||||
Name: key,
|
||||
SecondsToLock: 5,
|
||||
Owner: "convergence-test-0",
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if resp0.Error != "" {
|
||||
return false, fmt.Errorf("lock0: %s", resp0.Error)
|
||||
}
|
||||
|
||||
// Try to lock via filer1 — should fail (already locked)
|
||||
conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer conn1.Close()
|
||||
|
||||
client1 := filer_pb.NewSeaweedFilerClient(conn1)
|
||||
resp1, err := client1.DistributedLock(ctx, &filer_pb.LockRequest{
|
||||
Name: key,
|
||||
SecondsToLock: 5,
|
||||
Owner: "convergence-test-1",
|
||||
})
|
||||
|
||||
// Unlock via filer0
|
||||
client0.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
|
||||
Name: key,
|
||||
RenewToken: resp0.RenewToken,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// If filer1 also got the lock, the ring hasn't converged
|
||||
if resp1.Error == "" {
|
||||
// Unlock the second one too
|
||||
client1.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
|
||||
Name: key,
|
||||
RenewToken: resp1.RenewToken,
|
||||
})
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func stopCmd(cmd *exec.Cmd) {
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
cmd.Process.Signal(syscall.SIGTERM)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
cmd.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
cmd.Process.Kill()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func allocatePorts(t testing.TB, n int) []int {
|
||||
t.Helper()
|
||||
// Hold all listeners open until all ports are collected, then close
|
||||
// them together. This prevents the OS from reassigning a just-freed
|
||||
// port to the next Listen call within the same allocation batch.
|
||||
listeners := make([]net.Listener, 0, n)
|
||||
ports := make([]int, 0, n)
|
||||
defer func() {
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
for i := 0; i < n; i++ {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
listeners = append(listeners, l)
|
||||
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
return ports
|
||||
}
|
||||
|
||||
func findWeedBinary() string {
|
||||
if p := os.Getenv("WEED_BINARY"); p != "" {
|
||||
return p
|
||||
}
|
||||
if p, err := exec.LookPath("weed"); err == nil {
|
||||
return p
|
||||
}
|
||||
candidates := []string{
|
||||
"../../weed/weed",
|
||||
"./weed",
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if info, err := os.Stat(c); err == nil && !info.IsDir() {
|
||||
abs, _ := filepath.Abs(c)
|
||||
return abs
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -249,18 +249,11 @@ func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramewor
|
||||
return
|
||||
}
|
||||
|
||||
// Create file in subdirectory with retry for transient FUSE errors
|
||||
// Create file in subdirectory
|
||||
testFile := filepath.Join(subDir, "test.txt")
|
||||
content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
|
||||
var writeErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if writeErr != nil {
|
||||
addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
|
||||
if err := os.WriteFile(testFile, content, 0644); err != nil {
|
||||
addError(fmt.Errorf("worker %d file %d: %v", workerID, i, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,16 +82,12 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
|
||||
}
|
||||
}
|
||||
|
||||
// freePort asks the OS for a free TCP port in a range where the gRPC
|
||||
// offset (port + 10000) won't collide with well-known ports.
|
||||
// Stay below the Linux ephemeral floor (32768) so the kernel does not
|
||||
// reuse the chosen port for an outbound connection between close() here
|
||||
// and re-bind in the child "weed mini" process.
|
||||
// freePort asks the OS for a free TCP port.
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
const (
|
||||
minServicePort = 20000
|
||||
maxServicePort = 32000
|
||||
maxServicePort = 55535 // SeaweedFS gRPC service uses httpPort + 10000.
|
||||
)
|
||||
|
||||
portCount := maxServicePort - minServicePort + 1
|
||||
@@ -250,7 +246,6 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
|
||||
"mini",
|
||||
"-dir=" + f.dataDir,
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-filer.port=" + strconv.Itoa(f.filerPort),
|
||||
"-s3=false",
|
||||
"-webdav=false",
|
||||
|
||||
@@ -85,19 +85,9 @@ func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
|
||||
branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
gitRun(t, localClone, "push", "origin", branch)
|
||||
|
||||
// The bare repo lives on the FUSE mount and can briefly disappear after
|
||||
// the push completes, and pushed pack objects may not be immediately
|
||||
// consistent on the FUSE layer. Give the mount a chance to settle, then
|
||||
// recover from the local clone if the remote is still missing.
|
||||
if !waitForBareRepoEventually(t, bareRepo, 10*time.Second) {
|
||||
t.Logf("bare repo %s did not stabilise after push; forcing recovery before clone", bareRepo)
|
||||
}
|
||||
refreshDirEntry(t, bareRepo)
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
|
||||
t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
|
||||
ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
|
||||
gitRun(t, "", "clone", bareRepo, mountClone)
|
||||
|
||||
assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
|
||||
assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
|
||||
@@ -300,7 +290,7 @@ func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Durat
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if isBareRepoAccessible(bareRepo) {
|
||||
if isBareRepo(bareRepo) {
|
||||
return true
|
||||
}
|
||||
refreshDirEntry(t, bareRepo)
|
||||
@@ -322,14 +312,6 @@ func isBareRepo(bareRepo string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func isBareRepoAccessible(bareRepo string) bool {
|
||||
if !isBareRepo(bareRepo) {
|
||||
return false
|
||||
}
|
||||
out, err := tryGitCommand("", "--git-dir="+bareRepo, "rev-parse", "--is-bare-repository")
|
||||
return err == nil && out == "true"
|
||||
}
|
||||
|
||||
func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
|
||||
t.Helper()
|
||||
require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
|
||||
@@ -338,7 +320,7 @@ func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
|
||||
// tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
|
||||
// If it has vanished, it re-creates it from the local clone.
|
||||
func tryEnsureBareRepo(bareRepo, localClone string) error {
|
||||
if isBareRepoAccessible(bareRepo) {
|
||||
if _, err := os.Stat(filepath.Join(bareRepo, "HEAD")); err == nil {
|
||||
return nil
|
||||
}
|
||||
branch, err := tryGitCommand(localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
@@ -360,36 +342,6 @@ func tryEnsureBareRepo(bareRepo, localClone string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMountCloneFromBareWithRecovery(t *testing.T, bareRepo, localClone, mountClone string) {
|
||||
t.Helper()
|
||||
const maxAttempts = 3
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if lastErr = tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone); lastErr == nil {
|
||||
return
|
||||
}
|
||||
if attempt == maxAttempts {
|
||||
require.NoError(t, lastErr, "git clone %s %s failed after %d recovery attempts", bareRepo, mountClone, maxAttempts)
|
||||
}
|
||||
t.Logf("clone recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone string) error {
|
||||
if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
|
||||
return fmt.Errorf("ensure bare repo: %w", err)
|
||||
}
|
||||
if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
|
||||
return fmt.Errorf("ensure mount clone: %w", err)
|
||||
}
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
return fmt.Errorf("verify mount clone: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryEnsureMountClone is like ensureMountClone but returns an error instead
|
||||
// of failing the test, for use in recovery loops.
|
||||
func tryEnsureMountClone(bareRepo, mountClone string) error {
|
||||
@@ -479,31 +431,6 @@ func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, c
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Verify git commands actually work in the directory — the kernel
|
||||
// dcache can transiently show the dir then lose it after reset.
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
lastErr = fmt.Errorf("post-reset verification failed: %w", err)
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The kernel dcache can drop the FUSE entry moments after a
|
||||
// successful check. Wait briefly and re-verify to confirm the
|
||||
// directory has stabilised before returning to the caller.
|
||||
time.Sleep(1 * time.Second)
|
||||
refreshDirEntry(t, mountClone)
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
|
||||
@@ -531,10 +458,6 @@ func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit
|
||||
}
|
||||
refreshDirEntry(t, cloneDir)
|
||||
|
||||
// Let the dcache stabilise before proceeding.
|
||||
time.Sleep(1 * time.Second)
|
||||
refreshDirEntry(t, cloneDir)
|
||||
|
||||
head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("rev-parse after reset: %w", err)
|
||||
@@ -598,33 +521,3 @@ func TestTryEnsureBareRepoPreservesCurrentBranch(t *testing.T) {
|
||||
assert.Equal(t, branch, restoredHead, "clone from recovered bare repo should check out the current branch")
|
||||
assertFileContains(t, filepath.Join(restoredClone, "README.md"), "hello recovery")
|
||||
}
|
||||
|
||||
func TestEnsureMountCloneFromBareWithRecoveryRecreatesMissingBareRepo(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "git_mount_clone_recovery_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
bareRepo := filepath.Join(tempDir, "repo.git")
|
||||
localClone := filepath.Join(tempDir, "clone")
|
||||
mountClone := filepath.Join(tempDir, "mount-clone")
|
||||
|
||||
gitRun(t, "", "init", "--bare", bareRepo)
|
||||
gitRun(t, "", "clone", bareRepo, localClone)
|
||||
gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
|
||||
gitRun(t, localClone, "config", "user.name", "Test")
|
||||
|
||||
writeFile(t, localClone, "README.md", "hello clone recovery\n")
|
||||
gitRun(t, localClone, "add", "README.md")
|
||||
gitRun(t, localClone, "commit", "-m", "initial commit")
|
||||
|
||||
branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
gitRun(t, localClone, "push", "origin", branch)
|
||||
|
||||
require.NoError(t, os.RemoveAll(bareRepo))
|
||||
|
||||
ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
|
||||
|
||||
head := gitOutput(t, mountClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
assert.Equal(t, branch, head, "recovered clone should stay on the pushed branch")
|
||||
assertFileContains(t, filepath.Join(mountClone, "README.md"), "hello clone recovery")
|
||||
}
|
||||
|
||||
@@ -1,314 +0,0 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// mountDebugPort holds the debug/pprof port the test passed to the
|
||||
// mount process via -debug.port. It is set once at TestWriteBufferCap
|
||||
// entry and consulted from the write-timeout paths to fetch the mount
|
||||
// process's goroutine dump (the test's own dumpAllGoroutines only
|
||||
// covers the test process).
|
||||
var mountDebugPort int
|
||||
|
||||
// fetchMountGoroutines pulls a full goroutine dump from the mount
|
||||
// process's pprof endpoint. If the mount debug port isn't configured
|
||||
// or the HTTP call fails, a short explanation is returned instead of
|
||||
// an error — this is diagnostic best-effort, not a test assertion.
|
||||
func fetchMountGoroutines() string {
|
||||
if mountDebugPort == 0 {
|
||||
return "(mount debug port not configured)"
|
||||
}
|
||||
url := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof/goroutine?debug=2", mountDebugPort)
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("(failed to reach mount pprof at %s: %v)", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("(failed to read mount pprof body: %v)", err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// dumpAllGoroutines returns a full stack trace of every live goroutine.
|
||||
// Used on write-timeout to give CI actionable diagnosis if the write
|
||||
// buffer cap ever re-regresses into a hang.
|
||||
func dumpAllGoroutines() string {
|
||||
buf := make([]byte, 1<<20)
|
||||
for {
|
||||
n := runtime.Stack(buf, true)
|
||||
if n < len(buf) {
|
||||
return string(buf[:n])
|
||||
}
|
||||
buf = make([]byte, 2*len(buf))
|
||||
}
|
||||
}
|
||||
|
||||
// writeBufferCapConfig returns a TestConfig that exercises the new
|
||||
// -writeBufferSizeMB flag. The cap is set below the aggregate in-flight
|
||||
// write demand of the subtests below, so every new chunk has to pass
|
||||
// through the Reserve/Release backpressure path at least some of the
|
||||
// time. The cap is intentionally NOT minimal — over-tight settings
|
||||
// interact with the per-file writable-chunk limit and the FUSE MaxWrite
|
||||
// batching in ways that starve single-handle writers on slow CI.
|
||||
//
|
||||
// Also enables the mount's pprof debug endpoint so the test can fetch
|
||||
// mount-process goroutine dumps on write-timeout, which is the only
|
||||
// way to actually diagnose a backpressure deadlock (the test process
|
||||
// itself is just blocked in syscall.Write waiting on FUSE).
|
||||
func writeBufferCapConfig(debugPort int) *TestConfig {
|
||||
return &TestConfig{
|
||||
Collection: "",
|
||||
Replication: "000",
|
||||
ChunkSizeMB: 2, // 2 MiB chunks
|
||||
CacheSizeMB: 100, // read cache (unrelated)
|
||||
NumVolumes: 3,
|
||||
EnableDebug: false,
|
||||
MountOptions: []string{
|
||||
// 16 MiB total write buffer ⇒ up to 8 chunks in flight
|
||||
// across every open file handle on this mount. Large
|
||||
// enough to avoid starving a single handle on a slow
|
||||
// CI runner, small enough that the concurrent test
|
||||
// below still has to drain through it.
|
||||
"-writeBufferSizeMB=16",
|
||||
"-debug=true",
|
||||
fmt.Sprintf("-debug.port=%d", debugPort),
|
||||
// Route glog to stderr so the framework's process log
|
||||
// capture actually contains something — by default weed
|
||||
// sends glog to /tmp/weed.* files which the CI artifact
|
||||
// upload step never sees. Critical for diagnosing
|
||||
// upload/saveToStorage errors on Linux runs.
|
||||
"-logtostderr=true",
|
||||
"-v=2",
|
||||
},
|
||||
SkipCleanup: false,
|
||||
}
|
||||
}
|
||||
|
||||
// writeWithTimeout wraps os.WriteFile with a hard deadline so a stuck
|
||||
// write fails the test fast instead of consuming the full job budget.
|
||||
// This is belt-and-braces around the 45-minute workflow timeout and
|
||||
// makes write-buffer regressions surface as an actionable failure.
|
||||
func writeWithTimeout(t *testing.T, path string, data []byte, timeout time.Duration) {
|
||||
t.Helper()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- os.WriteFile(path, data, 0644) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
// Dump mount goroutines on any write error, not just
|
||||
// timeout — upload failures that surface via close()
|
||||
// as EIO leave the mount process running but in an
|
||||
// informative state (pending sealed chunks, error
|
||||
// counters, etc).
|
||||
t.Logf("write %s failed (%v) — dumping MOUNT goroutines:\n%s", path, err, fetchMountGoroutines())
|
||||
}
|
||||
require.NoError(t, err, "write %s", path)
|
||||
case <-time.After(timeout):
|
||||
t.Logf("write %s did not finish within %v — dumping TEST goroutines:\n%s", path, timeout, dumpAllGoroutines())
|
||||
t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
|
||||
t.Fatalf("write %s timed out — write buffer cap is likely leaking or deadlocking", path)
|
||||
}
|
||||
}
|
||||
|
||||
// runSubtestWithWatchdog runs body on the current (subtest main)
|
||||
// goroutine and starts a watcher goroutine that logs diagnostics and
|
||||
// fails the test if body doesn't return within timeout.
|
||||
//
|
||||
// body must run on the main goroutine because test helpers inside it
|
||||
// (require.NoError, writeWithTimeout's own t.Fatalf on its internal
|
||||
// timeout) need t.Fatal / t.FailNow, which Go's testing docs restrict
|
||||
// to the goroutine running the test function. The watcher goroutine
|
||||
// only calls goroutine-safe t methods (t.Log, t.Logf, t.Errorf) so it
|
||||
// can mark the test failed and dump diagnostics without violating
|
||||
// that contract. If body is stuck past timeout the watcher still
|
||||
// surfaces the wedge (test + mount goroutine dumps + a FAIL mark);
|
||||
// body itself gets unblocked either by its own inner writeWithTimeout
|
||||
// firing t.Fatalf or by Go test's global -timeout.
|
||||
func runSubtestWithWatchdog(t *testing.T, timeout time.Duration, body func(t *testing.T)) {
|
||||
t.Helper()
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go func() {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-time.After(timeout):
|
||||
t.Logf("subtest exceeded %v watchdog — dumping TEST goroutines:\n%s", timeout, dumpAllGoroutines())
|
||||
t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
|
||||
t.Errorf("subtest exceeded %v watchdog — see goroutine dumps above", timeout)
|
||||
}
|
||||
}()
|
||||
body(t)
|
||||
}
|
||||
|
||||
// TestWriteBufferCap exercises the end-to-end write-buffer cap on a
|
||||
// real FUSE mount. Without the cap, a volume-server stall would let
|
||||
// the swap file grow without bound (issue #8777). With the cap, writers
|
||||
// must serialize through a bounded budget while still producing correct
|
||||
// output — that correctness (and the absence of deadlocks) is what
|
||||
// this test verifies.
|
||||
//
|
||||
// Note: this test deliberately does not assert that Reserve *blocked*
|
||||
// at some observed used-byte peak. The mount runs as a subprocess so
|
||||
// its in-process WriteBufferAccountant state is not reachable from the
|
||||
// test without adding a metrics/RPC surface to the mount binary. The
|
||||
// deterministic peak-vs-cap assertion instead lives in the in-package
|
||||
// unit test TestWriteBufferCap_SharedAcrossPipelines, which drives a
|
||||
// controlled gated uploader and samples Used() throughout the run.
|
||||
func TestWriteBufferCap(t *testing.T) {
|
||||
mountDebugPort = freePort(t)
|
||||
config := writeBufferCapConfig(mountDebugPort)
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
const subtestTimeout = 3 * time.Minute
|
||||
|
||||
t.Run("ConcurrentWritesUnderCap", func(t *testing.T) {
|
||||
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
|
||||
testConcurrentWritesUnderCap(t, framework)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("LargeFileUnderCap", func(t *testing.T) {
|
||||
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
|
||||
testLargeFileUnderCap(t, framework)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("DoesNotDeadlockAfterPressure", func(t *testing.T) {
|
||||
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
|
||||
testWriteBufferNoDeadlockAfterPressure(t, framework)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// testConcurrentWritesUnderCap opens several files in parallel with
|
||||
// aggregate demand that exceeds the 16 MiB write buffer cap, then
|
||||
// verifies every byte survived the round trip.
|
||||
func testConcurrentWritesUnderCap(t *testing.T, framework *FuseTestFramework) {
|
||||
const (
|
||||
numFiles = 4
|
||||
fileSize = 8 * 1024 * 1024 // 8 MiB per file ⇒ 32 MiB total vs 16 MiB cap
|
||||
)
|
||||
|
||||
dir := "write_buffer_cap_concurrent"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
payloads := make([][]byte, numFiles)
|
||||
for i := range payloads {
|
||||
buf := make([]byte, fileSize)
|
||||
_, err := rand.Read(buf)
|
||||
require.NoError(t, err)
|
||||
payloads[i] = buf
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, numFiles)
|
||||
timedOut := make(chan struct{}, numFiles)
|
||||
for i := 0; i < numFiles; i++ {
|
||||
i := i
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
name := fmt.Sprintf("file_%02d.bin", i)
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, name)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- os.WriteFile(path, payloads[i], 0644) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("writer %d: %w", i, err)
|
||||
}
|
||||
case <-time.After(90 * time.Second):
|
||||
timedOut <- struct{}{}
|
||||
errs <- fmt.Errorf("writer %d: timed out after 90s", i)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
// If any writer timed out, dump every live goroutine so CI shows the
|
||||
// wedge instead of just a walltime.
|
||||
select {
|
||||
case <-timedOut:
|
||||
t.Logf("at least one concurrent writer timed out — dumping goroutines:\n%s", dumpAllGoroutines())
|
||||
default:
|
||||
}
|
||||
for err := range errs {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("wrote %d × %d MiB under 16 MiB cap in %v", numFiles, fileSize/(1024*1024), time.Since(start))
|
||||
|
||||
for i := 0; i < numFiles; i++ {
|
||||
name := fmt.Sprintf("file_%02d.bin", i)
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, name)
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err, "read %s", name)
|
||||
require.Equal(t, len(payloads[i]), len(got), "size mismatch for %s", name)
|
||||
if !bytes.Equal(payloads[i], got) {
|
||||
t.Fatalf("content mismatch for %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testLargeFileUnderCap writes a single file whose size exceeds the
|
||||
// 16 MiB cap through a single handle, verifying that the pipeline
|
||||
// drains its own earlier chunks and makes forward progress rather than
|
||||
// self-deadlocking when the global budget is already full of its own
|
||||
// earlier sealed chunks.
|
||||
func testLargeFileUnderCap(t *testing.T, framework *FuseTestFramework) {
|
||||
const fileSize = 20 * 1024 * 1024 // 20 MiB ⇒ 10 chunks vs 8-slot budget
|
||||
|
||||
payload := make([]byte, fileSize)
|
||||
_, err := rand.Read(payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
name := "write_buffer_cap_large.bin"
|
||||
path := filepath.Join(framework.GetMountPoint(), name)
|
||||
|
||||
start := time.Now()
|
||||
writeWithTimeout(t, path, payload, 90*time.Second)
|
||||
t.Logf("wrote %d MiB through one handle under 16 MiB cap in %v", fileSize/(1024*1024), time.Since(start))
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(payload), len(got))
|
||||
if !bytes.Equal(payload, got) {
|
||||
t.Fatal("content mismatch on large single-handle write")
|
||||
}
|
||||
}
|
||||
|
||||
// testWriteBufferNoDeadlockAfterPressure verifies the mount is still
|
||||
// healthy after being driven against the cap. A budget-slot leak would
|
||||
// eventually cause every new chunk allocation to hang; a quick canary
|
||||
// write catches that as a hard failure.
|
||||
func testWriteBufferNoDeadlockAfterPressure(t *testing.T, framework *FuseTestFramework) {
|
||||
name := "write_buffer_cap_canary.txt"
|
||||
path := filepath.Join(framework.GetMountPoint(), name)
|
||||
content := []byte("write buffer cap canary — mount still healthy")
|
||||
|
||||
writeWithTimeout(t, path, content, 30*time.Second)
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, content, got)
|
||||
}
|
||||
@@ -1,390 +0,0 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_p2p
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// p2pTestCluster manages a minimal SeaweedFS cluster exercising the peer
|
||||
// chunk-sharing (p2p) read path: 1 master, 1 volume, 1 filer, and N FUSE
|
||||
// mounts that all have -peer.enable set. Three mounts is the sweet spot
|
||||
// for integration testing — with 2 mounts the HRW owner of a chunk is
|
||||
// ~50% the reader itself (which short-circuits the peer path); with 3+
|
||||
// mounts it's ≤ 1/3, so a multi-chunk file almost certainly exercises
|
||||
// the remote-owner fan-out.
|
||||
type p2pTestCluster struct {
|
||||
t testing.TB
|
||||
baseDir string
|
||||
weedBinary string
|
||||
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
volumePort int
|
||||
volumeGrpcPort int
|
||||
filerPort int
|
||||
filerGrpcPort int
|
||||
mountPeerPorts []int
|
||||
mountPoints []string
|
||||
|
||||
masterCmd *exec.Cmd
|
||||
volumeCmd *exec.Cmd
|
||||
filerCmd *exec.Cmd
|
||||
mountCmds []*exec.Cmd
|
||||
logFiles []*os.File
|
||||
|
||||
cleanupOnce sync.Once
|
||||
}
|
||||
|
||||
// startP2PTestCluster brings up a cluster with numMounts FUSE mounts,
|
||||
// every one advertising on its own -peer.listen port. All mounts register
|
||||
// with the same filer. Verbose logging (-v=4) is enabled on each mount so
|
||||
// peer-read success/failure messages land in the log file and the test
|
||||
// can grep them to verify the p2p path fired.
|
||||
func startP2PTestCluster(t testing.TB, numMounts int) *p2pTestCluster {
|
||||
require.GreaterOrEqual(t, numMounts, 2, "need at least 2 mounts to exercise p2p")
|
||||
binary := findWeedBinary()
|
||||
if binary == "" {
|
||||
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
|
||||
}
|
||||
baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_p2p_test_")
|
||||
require.NoError(t, err)
|
||||
|
||||
c := &p2pTestCluster{
|
||||
t: t,
|
||||
baseDir: baseDir,
|
||||
weedBinary: binary,
|
||||
mountPeerPorts: make([]int, numMounts),
|
||||
mountPoints: make([]string, numMounts),
|
||||
mountCmds: make([]*exec.Cmd, numMounts),
|
||||
}
|
||||
t.Cleanup(c.Stop)
|
||||
|
||||
// master(2) + volume(2) + filer(2) + one peer port per mount.
|
||||
// testutil.AllocatePorts holds all listeners open until every port
|
||||
// is reserved, avoiding the brief close→bind race that would
|
||||
// happen with a per-listener close loop.
|
||||
ports, err := testutil.AllocatePorts(6 + numMounts)
|
||||
require.NoError(t, err)
|
||||
c.masterPort = ports[0]
|
||||
c.masterGrpcPort = ports[1]
|
||||
c.volumePort = ports[2]
|
||||
c.volumeGrpcPort = ports[3]
|
||||
c.filerPort = ports[4]
|
||||
c.filerGrpcPort = ports[5]
|
||||
for i := 0; i < numMounts; i++ {
|
||||
c.mountPeerPorts[i] = ports[6+i]
|
||||
}
|
||||
|
||||
configDir := filepath.Join(baseDir, "config")
|
||||
require.NoError(t, os.MkdirAll(configDir, 0755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
|
||||
|
||||
require.NoError(t, c.startMaster(configDir))
|
||||
require.NoError(t, c.waitForTCP(c.masterCmd, "master",
|
||||
fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second))
|
||||
|
||||
require.NoError(t, c.startVolume(configDir))
|
||||
require.NoError(t, c.waitForTCP(c.volumeCmd, "volume",
|
||||
fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second))
|
||||
|
||||
require.NoError(t, c.startFiler(configDir))
|
||||
require.NoError(t, c.waitForTCP(c.filerCmd, "filer",
|
||||
fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPort), 30*time.Second))
|
||||
|
||||
for i := 0; i < numMounts; i++ {
|
||||
mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
|
||||
require.NoError(t, os.MkdirAll(mp, 0755))
|
||||
c.mountPoints[i] = mp
|
||||
require.NoError(t, c.startMount(i, configDir))
|
||||
require.NoError(t, c.waitForMount(mp, 30*time.Second),
|
||||
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) Stop() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.cleanupOnce.Do(func() {
|
||||
for i := len(c.mountCmds) - 1; i >= 0; i-- {
|
||||
stopCmd(c.mountCmds[i])
|
||||
// Backup unmount in case the FUSE teardown didn't clean up.
|
||||
exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
|
||||
exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
|
||||
}
|
||||
stopCmd(c.filerCmd)
|
||||
stopCmd(c.volumeCmd)
|
||||
stopCmd(c.masterCmd)
|
||||
|
||||
for _, f := range c.logFiles {
|
||||
f.Close()
|
||||
}
|
||||
c.copyLogsForCI()
|
||||
if !c.t.Failed() {
|
||||
os.RemoveAll(c.baseDir)
|
||||
}
|
||||
time.Sleep(2 * time.Second) // let ports drain
|
||||
})
|
||||
}
|
||||
|
||||
// MountDir returns the filesystem path of the i-th mount.
|
||||
func (c *p2pTestCluster) MountDir(i int) string { return c.mountPoints[i] }
|
||||
|
||||
// masterAddress / filerAddress return SeaweedFS-style addresses encoding
|
||||
// both ports as "host:httpPort.grpcPort". Without this, downstream
|
||||
// components fall back to the grpcPort = httpPort + 10000 default,
|
||||
// which doesn't match the random port we allocate.
|
||||
func (c *p2pTestCluster) masterAddress() string {
|
||||
return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) filerAddress() string {
|
||||
return string(pb.NewServerAddress("127.0.0.1", c.filerPort, c.filerGrpcPort))
|
||||
}
|
||||
|
||||
// MountLog returns the contents of mount i's log file.
|
||||
func (c *p2pTestCluster) MountLog(i int) string {
|
||||
return c.tailLogFull(fmt.Sprintf("mount%d", i))
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) startMaster(configDir string) error {
|
||||
c.masterCmd = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"master",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.masterPort),
|
||||
"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
|
||||
"-mdir="+filepath.Join(c.baseDir, "master"),
|
||||
)
|
||||
return c.startCmd(c.masterCmd, "master")
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) startVolume(configDir string) error {
|
||||
volDir := filepath.Join(c.baseDir, "volume")
|
||||
if err := os.MkdirAll(volDir, 0755); err != nil {
|
||||
return fmt.Errorf("create volume dir: %w", err)
|
||||
}
|
||||
c.volumeCmd = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"volume",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.volumePort),
|
||||
"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
|
||||
"-master="+c.masterAddress(),
|
||||
"-dir="+volDir,
|
||||
"-max=10",
|
||||
)
|
||||
return c.startCmd(c.volumeCmd, "volume")
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) startFiler(configDir string) error {
|
||||
filerDir := filepath.Join(c.baseDir, "filer")
|
||||
if err := os.MkdirAll(filerDir, 0755); err != nil {
|
||||
return fmt.Errorf("create filer dir: %w", err)
|
||||
}
|
||||
c.filerCmd = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"filer",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port="+strconv.Itoa(c.filerPort),
|
||||
"-port.grpc="+strconv.Itoa(c.filerGrpcPort),
|
||||
"-master="+c.masterAddress(),
|
||||
"-defaultStoreDir="+filerDir,
|
||||
// default is -mount.p2p=true; be explicit so the intent is grep-able.
|
||||
"-mount.p2p=true",
|
||||
)
|
||||
return c.startCmd(c.filerCmd, "filer")
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) startMount(idx int, configDir string) error {
|
||||
cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return fmt.Errorf("create cache dir: %w", err)
|
||||
}
|
||||
c.mountCmds[idx] = exec.Command(c.weedBinary,
|
||||
"-logdir="+filepath.Join(c.baseDir, "logs"),
|
||||
"-v=4",
|
||||
"mount",
|
||||
"-filer="+c.filerAddress(),
|
||||
"-dir="+c.mountPoints[idx],
|
||||
"-filer.path=/",
|
||||
"-dirAutoCreate",
|
||||
"-allowOthers=false",
|
||||
"-cacheDir="+cacheDir,
|
||||
"-peer.enable=true",
|
||||
fmt.Sprintf("-peer.listen=127.0.0.1:%d", c.mountPeerPorts[idx]),
|
||||
fmt.Sprintf("-peer.advertise=127.0.0.1:%d", c.mountPeerPorts[idx]),
|
||||
"-peer.dataCenter=dc1",
|
||||
fmt.Sprintf("-peer.rack=rack%d", idx),
|
||||
)
|
||||
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) startCmd(cmd *exec.Cmd, name string) error {
|
||||
logPath := filepath.Join(c.baseDir, "logs")
|
||||
if err := os.MkdirAll(logPath, 0755); err != nil {
|
||||
return fmt.Errorf("create log dir: %w", err)
|
||||
}
|
||||
logFile, err := os.Create(filepath.Join(logPath, name+".log"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.logFiles = append(c.logFiles, logFile)
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
return cmd.Start()
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) tailLog(name string) string {
|
||||
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
|
||||
if err != nil {
|
||||
return fmt.Sprintf("(log %s not available: %v)", name, err)
|
||||
}
|
||||
const maxTail = 8192
|
||||
if len(data) > maxTail {
|
||||
data = data[len(data)-maxTail:]
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) tailLogFull(name string) string {
|
||||
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func (c *p2pTestCluster) copyLogsForCI() {
|
||||
ciLogDir := "/tmp/seaweedfs-fuse-p2p-logs"
|
||||
os.MkdirAll(ciLogDir, 0755)
|
||||
logsDir := filepath.Join(c.baseDir, "logs")
|
||||
entries, err := os.ReadDir(logsDir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForTCP polls addr until it accepts a connection, OR the supplied
|
||||
// subprocess exits — whichever comes first. Short-circuiting on child
|
||||
// exit turns a 30 s-spin-on-dead-process into an immediate failure with
|
||||
// the tail of its log. cmd may be nil for callers that don't track a
|
||||
// process; in that case we fall back to pure polling.
|
||||
//
|
||||
// Liveness is checked with signal 0 (POSIX "is this process alive").
|
||||
// We deliberately do NOT call cmd.Wait() in a goroutine here because
|
||||
// stopCmd() later calls Wait() at shutdown, and only one Wait per
|
||||
// process is allowed.
|
||||
func (c *p2pTestCluster) waitForTCP(cmd *exec.Cmd, name, addr string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", addr, time.Second)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
if cmd != nil && cmd.Process != nil {
|
||||
// signal 0 doesn't actually send anything; it just asks
|
||||
// the kernel whether the process exists. An error here
|
||||
// (typically "process already finished" or ESRCH) means
|
||||
// the child died before coming up.
|
||||
if sigErr := cmd.Process.Signal(syscall.Signal(0)); sigErr != nil {
|
||||
return fmt.Errorf("%s exited before listening on %s: %v\n%s",
|
||||
name, addr, sigErr, c.tailLog(name))
|
||||
}
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("service at %s not ready within timeout\n%s", addr, c.tailLog(name))
|
||||
}
|
||||
|
||||
// waitForMount waits for a FUSE filesystem to actually be mounted by
|
||||
// watching the device id flip (FUSE mounts have a different Dev than
|
||||
// their parent dir).
|
||||
func (c *p2pTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
|
||||
parentDir := filepath.Dir(mountPoint)
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
parentStat, err := os.Stat(parentDir)
|
||||
if err != nil {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
mountStat, err := os.Stat(mountPoint)
|
||||
if err != nil {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
parentSys := parentStat.Sys().(*syscall.Stat_t)
|
||||
mountSys := mountStat.Sys().(*syscall.Stat_t)
|
||||
if parentSys.Dev != mountSys.Dev {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
|
||||
}
|
||||
|
||||
// --- utilities (binary discovery, process shutdown) ---
|
||||
//
|
||||
// Port allocation is delegated to testutil.MustAllocatePorts, which
|
||||
// holds all listeners open until every port is reserved before
|
||||
// returning them in a batch — safer than the per-listener
|
||||
// close-then-reserve pattern fuse_dlm originally used.
|
||||
|
||||
func findWeedBinary() string {
|
||||
if env := os.Getenv("WEED_BINARY"); env != "" {
|
||||
if _, err := os.Stat(env); err == nil {
|
||||
return env
|
||||
}
|
||||
}
|
||||
if p, err := exec.LookPath("weed"); err == nil {
|
||||
return p
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// stopCmd kills a child process with SIGTERM and waits up to 5 s, then
|
||||
// escalates to SIGKILL. Tolerates nil and already-exited processes.
|
||||
func stopCmd(cmd *exec.Cmd) {
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
cmd.Process.Signal(syscall.SIGTERM)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
cmd.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
cmd.Process.Signal(syscall.SIGKILL)
|
||||
<-done
|
||||
}
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package fuse_p2p
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// peerReadSuccessMarker is the log line tryPeerRead emits when a peer
|
||||
// fetch succeeded. The test greps non-writer mount logs for it to
|
||||
// prove the p2p path fired. At glog verbosity 4 (framework sets -v=4).
|
||||
const peerReadSuccessMarker = "peer read successful"
|
||||
|
||||
// seedConvergenceTimeout bounds how long the test waits for:
|
||||
// - the filer registry to list all mounts,
|
||||
// - every mount's seed view (via MountList poll) to include all peers,
|
||||
// - the first announcer flush cycle to publish chunk holders.
|
||||
//
|
||||
// With defaults the mount polls MountList every 30 s and flushes
|
||||
// ChunkAnnounce every 15 s. Allowing 45 s absorbs one MountList refresh
|
||||
// plus one announce cycle plus some slop for CI variance.
|
||||
const seedConvergenceTimeout = 90 * time.Second
|
||||
|
||||
// TestPeerChunkSharing_ReadersPullFromPeerCache is the headline p2p
|
||||
// integration test. It proves at least one non-writer mount can satisfy
|
||||
// a read from the writer's chunk cache instead of the volume tier.
|
||||
//
|
||||
// 1. Bring up 1 master/volume/filer + 3 mounts, all with -peer.enable.
|
||||
// 2. Mount 0 writes a ~8 MiB file and reads it back so chunks land in
|
||||
// its local cache and the announcer publishes them.
|
||||
// 3. Wait for seed convergence + at least one announcer flush cycle.
|
||||
// 4. BOTH mount 1 and mount 2 read the file.
|
||||
//
|
||||
// Why both readers: with 3 mounts, HRW picks one owner for the chunk.
|
||||
// If that owner is mount 1, only mount 2's read will hit the peer
|
||||
// path (mount 1's tryPeerRead bails on owner==self). If that owner is
|
||||
// mount 2, only mount 1's read will. If that owner is mount 0 (the
|
||||
// writer), both can. So by reading from both, we deterministically
|
||||
// guarantee at least one non-writer mount exercises the peer path.
|
||||
//
|
||||
// Once the peer fetch populates its local cache, subsequent reads
|
||||
// short-circuit on IsInCache — so we only get one real shot per mount
|
||||
// per chunk. That's fine: one success is all the test needs.
|
||||
func TestPeerChunkSharing_ReadersPullFromPeerCache(t *testing.T) {
|
||||
c := startP2PTestCluster(t, 3)
|
||||
|
||||
// ~8 MiB, pseudo-random so compression doesn't collapse it to one block.
|
||||
payload := make([]byte, 8*1024*1024)
|
||||
rng := rand.New(rand.NewPCG(1, 2))
|
||||
for i := range payload {
|
||||
payload[i] = byte(rng.Uint32())
|
||||
}
|
||||
|
||||
const relPath = "p2p-test.bin"
|
||||
writer := c.MountDir(0)
|
||||
require.NoError(t, os.WriteFile(filepath.Join(writer, relPath), payload, 0644))
|
||||
|
||||
// Warm mount 0's chunk cache by reading back through its own FUSE.
|
||||
// Without this the chunks are on the volume server but not yet
|
||||
// in anyone's peer-servable cache.
|
||||
readBack, err := os.ReadFile(filepath.Join(writer, relPath))
|
||||
require.NoError(t, err)
|
||||
require.True(t, bytes.Equal(readBack, payload), "write-then-read on writer mount should match")
|
||||
|
||||
waitForSeedConvergence(t, c, seedConvergenceTimeout)
|
||||
|
||||
// Give the announcer several flush windows to push chunk-holder
|
||||
// entries to the HRW owners. First flush may see the writer's seed
|
||||
// view incomplete (only self) and defer all fids; subsequent
|
||||
// flushes re-check against a refreshed seed view. announce interval
|
||||
// is 15 s, so 45 s covers three attempts.
|
||||
time.Sleep(45 * time.Second)
|
||||
|
||||
// Read from both non-writer mounts. For any HRW outcome on any
|
||||
// chunk, at least one of these reads will NOT have the reader as
|
||||
// the HRW owner, so its tryPeerRead will proceed to ChunkLookup +
|
||||
// FetchChunk.
|
||||
for _, idx := range []int{1, 2} {
|
||||
got, err := os.ReadFile(filepath.Join(c.MountDir(idx), relPath))
|
||||
require.NoError(t, err, "read from mount %d must succeed\n--- mount%d ---\n%s",
|
||||
idx, idx, tailLines(c.MountLog(idx), 80))
|
||||
require.Equal(t, md5.Sum(payload), md5.Sum(got),
|
||||
"mount %d returned mismatched bytes (len got=%d want=%d)", idx, len(got), len(payload))
|
||||
}
|
||||
|
||||
// Content matches alone doesn't prove p2p — the volume fallback
|
||||
// would also satisfy the reads. Require at least one non-writer
|
||||
// mount's log to contain the peer-read success marker.
|
||||
var sawPeerRead bool
|
||||
for _, idx := range []int{1, 2} {
|
||||
if strings.Contains(c.MountLog(idx), peerReadSuccessMarker) {
|
||||
sawPeerRead = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !sawPeerRead {
|
||||
t.Fatalf("no non-writer mount logged %q — peer read path never fired.\n"+
|
||||
"--- mount0 (writer) tail ---\n%s\n--- mount1 tail ---\n%s\n--- mount2 tail ---\n%s",
|
||||
peerReadSuccessMarker,
|
||||
tailLines(c.MountLog(0), 60), tailLines(c.MountLog(1), 60), tailLines(c.MountLog(2), 60))
|
||||
}
|
||||
}
|
||||
|
||||
// waitForSeedConvergence polls each mount's log looking for any sign
|
||||
// that MountList returned a peer list containing the other mounts.
|
||||
func waitForSeedConvergence(t *testing.T, c *p2pTestCluster, timeout time.Duration) {
|
||||
t.Helper()
|
||||
// The first MountRegister happens synchronously during mount startup,
|
||||
// and the first MountList is pulled right after. 30 s is the refresh
|
||||
// interval; waiting one full cycle here guarantees every mount has
|
||||
// at minimum observed the others in its seed view.
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
ready := true
|
||||
for i := range c.mountCmds {
|
||||
if !strings.Contains(c.MountLog(i), "peer-grpc listening on") {
|
||||
ready = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if ready {
|
||||
time.Sleep(30 * time.Second)
|
||||
return
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("peer-grpc servers did not come up within %s", timeout)
|
||||
}
|
||||
|
||||
// tailLines returns the last n newline-delimited lines of s, or the
|
||||
// whole thing if shorter. Keeps test failures readable.
|
||||
func tailLines(s string, n int) string {
|
||||
lines := strings.Split(s, "\n")
|
||||
if len(lines) <= n {
|
||||
return s
|
||||
}
|
||||
return fmt.Sprintf("... (%d earlier lines omitted) ...\n%s",
|
||||
len(lines)-n, strings.Join(lines[len(lines)-n:], "\n"))
|
||||
}
|
||||
+40
-39
@@ -14,7 +14,7 @@ require (
|
||||
replace github.com/seaweedfs/seaweedfs => ../../
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
|
||||
@@ -43,25 +43,25 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/aws/aws-sdk-go v1.55.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -70,7 +70,6 @@ require (
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
@@ -105,7 +104,7 @@ require (
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
@@ -125,8 +124,8 @@ require (
|
||||
github.com/google/btree v1.1.3 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -148,9 +147,9 @@ require (
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/klauspost/compress v1.18.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.13.3 // indirect
|
||||
github.com/klauspost/reedsolomon v1.13.0 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
@@ -198,6 +197,7 @@ require (
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
|
||||
github.com/rdleal/intervalst v1.5.0 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
@@ -237,34 +237,35 @@ require (
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.9 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
|
||||
golang.org/x/image v0.36.0 // indirect
|
||||
golang.org/x/net v0.49.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
golang.org/x/term v0.42.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/api v0.274.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/api v0.258.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
|
||||
+94
-92
@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
|
||||
cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
@@ -112,44 +112,44 @@ github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
@@ -170,8 +170,6 @@ github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU
|
||||
github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
|
||||
github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
||||
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
|
||||
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
|
||||
github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
|
||||
github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
|
||||
github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
|
||||
@@ -286,8 +284,8 @@ github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@@ -387,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
|
||||
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
|
||||
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
@@ -469,12 +467,12 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
|
||||
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
|
||||
github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/klauspost/reedsolomon v1.13.0 h1:E0Cmgf2kMuhZTj6eefnvpKC4/Q4jhCi9YIjcZjK4arc=
|
||||
github.com/klauspost/reedsolomon v1.13.0/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
|
||||
@@ -602,6 +600,8 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
|
||||
github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
@@ -707,8 +707,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@@ -739,8 +739,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
|
||||
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
@@ -754,16 +754,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
@@ -787,8 +787,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -799,12 +799,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -869,8 +869,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -894,8 +894,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -946,8 +946,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -959,8 +959,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
|
||||
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -976,13 +976,13 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
@@ -1030,14 +1030,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
|
||||
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
@@ -1054,8 +1054,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
|
||||
google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -1091,12 +1091,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -1154,6 +1154,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs=
|
||||
moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
|
||||
@@ -10,14 +10,6 @@ import (
|
||||
"github.com/segmentio/kafka-go"
|
||||
)
|
||||
|
||||
const (
|
||||
consumerGroupHeartbeatInterval = 2 * time.Second
|
||||
consumerGroupSessionTimeout = 6 * time.Second
|
||||
consumerGroupRebalanceTimeout = 6 * time.Second
|
||||
consumerGroupJoinBackoff = 250 * time.Millisecond
|
||||
consumerGroupAttemptTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// KafkaGoClient wraps kafka-go client with test utilities
|
||||
type KafkaGoClient struct {
|
||||
brokerAddr string
|
||||
@@ -136,81 +128,40 @@ func (k *KafkaGoClient) ConsumeMessages(topicName string, expectedCount int) ([]
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// ConsumeWithGroup consumes messages using consumer group.
|
||||
// Retries the initial join+fetch with a fresh reader if it fails before any
|
||||
// message is received — re-joining an existing group races with the previous
|
||||
// member's LeaveGroup / session cleanup and can surface as an i/o timeout on
|
||||
// the first FetchMessage.
|
||||
// ConsumeWithGroup consumes messages using consumer group
|
||||
func (k *KafkaGoClient) ConsumeWithGroup(topicName, groupID string, expectedCount int) ([]kafka.Message, error) {
|
||||
k.t.Helper()
|
||||
|
||||
const maxJoinAttempts = 5
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxJoinAttempts; attempt++ {
|
||||
messages, err, progressed := k.consumeWithGroupOnce(topicName, groupID, expectedCount)
|
||||
if err == nil {
|
||||
return messages, nil
|
||||
}
|
||||
lastErr = err
|
||||
// Only retry if we failed before any message was received. Once we've
|
||||
// fetched at least one message, a partial result is more useful than a
|
||||
// full retry (which would start over from the last committed offset).
|
||||
if progressed {
|
||||
return messages, err
|
||||
}
|
||||
if attempt == maxJoinAttempts {
|
||||
break
|
||||
}
|
||||
backoff := time.Duration(500*(1<<(attempt-1))) * time.Millisecond
|
||||
k.t.Logf("ConsumeWithGroup join attempt %d/%d failed (%v) — retrying after %v", attempt, maxJoinAttempts, err, backoff)
|
||||
time.Sleep(backoff)
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// consumeWithGroupOnce runs a single consume attempt. Returns the messages
|
||||
// fetched, any error, and whether any message was received (used to decide
|
||||
// whether a retry is safe).
|
||||
func (k *KafkaGoClient) consumeWithGroupOnce(topicName, groupID string, expectedCount int) ([]kafka.Message, error, bool) {
|
||||
// Give each reader its own ClientID so restarts do not get mistaken for the
|
||||
// still-shutting-down reader they are replacing.
|
||||
dialer := &kafka.Dialer{
|
||||
ClientID: fmt.Sprintf("seaweedfs-e2e-%s-%d", groupID, time.Now().UnixNano()),
|
||||
Timeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: []string{k.brokerAddr},
|
||||
Dialer: dialer,
|
||||
Topic: topicName,
|
||||
GroupID: groupID,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
CommitInterval: 500 * time.Millisecond,
|
||||
HeartbeatInterval: consumerGroupHeartbeatInterval,
|
||||
SessionTimeout: consumerGroupSessionTimeout,
|
||||
RebalanceTimeout: consumerGroupRebalanceTimeout,
|
||||
JoinGroupBackoff: consumerGroupJoinBackoff,
|
||||
Brokers: []string{k.brokerAddr},
|
||||
Topic: topicName,
|
||||
GroupID: groupID,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
CommitInterval: 500 * time.Millisecond,
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
// Log the initial offset position
|
||||
offset := reader.Offset()
|
||||
k.t.Logf("Consumer group reader created for group %s, initial offset: %d", groupID, offset)
|
||||
|
||||
// Keep each attempt short enough that a retry can outlive a stale group
|
||||
// member instead of burning most of the overall test timeout on one try.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), consumerGroupAttemptTimeout)
|
||||
// Increased timeout for consumer groups - they require coordinator discovery,
|
||||
// offset fetching, and offset commits which can be slow in CI environments
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var messages []kafka.Message
|
||||
for i := 0; i < expectedCount; i++ {
|
||||
// Fetch then explicitly commit to better control commit timing
|
||||
msg, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
return messages, fmt.Errorf("read message %d: %w", i, err), len(messages) > 0
|
||||
return messages, fmt.Errorf("read message %d: %w", i, err)
|
||||
}
|
||||
messages = append(messages, msg)
|
||||
k.t.Logf(" Fetched message %d: offset=%d, partition=%d", i, msg.Offset, msg.Partition)
|
||||
|
||||
// Commit with simple retry to handle transient connection churn
|
||||
var commitErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
commitErr = reader.CommitMessages(ctx, msg)
|
||||
@@ -219,15 +170,16 @@ func (k *KafkaGoClient) consumeWithGroupOnce(topicName, groupID string, expected
|
||||
break
|
||||
}
|
||||
k.t.Logf(" Commit attempt %d failed for offset %d: %v", attempt+1, msg.Offset, commitErr)
|
||||
// brief backoff
|
||||
time.Sleep(time.Duration(50*(1<<attempt)) * time.Millisecond)
|
||||
}
|
||||
if commitErr != nil {
|
||||
return messages, fmt.Errorf("committing message %d: %w", i, commitErr), true
|
||||
return messages, fmt.Errorf("committing message %d: %w", i, commitErr)
|
||||
}
|
||||
}
|
||||
|
||||
k.t.Logf("Consumed %d messages from topic %s with group %s", len(messages), topicName, groupID)
|
||||
return messages, nil, true
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// CreateTopic creates a topic using Sarama
|
||||
|
||||
@@ -17,18 +17,16 @@ COPY test/kafka/kafka-client-loadtest/ ./
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /kafka-loadtest ./cmd/loadtest
|
||||
|
||||
# Stage 2: Runtime
|
||||
# Use alpine so we don't depend on Ubuntu apt mirrors, which intermittently
|
||||
# refuse connections from GitHub Actions runners and fail the CI build.
|
||||
# All runtime dependencies we need (ca-certificates, curl, jq, bash, nc)
|
||||
# are in the Alpine main repo.
|
||||
FROM alpine:3.20
|
||||
FROM ubuntu:22.04
|
||||
|
||||
RUN apk add --no-cache \
|
||||
# Install runtime dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
bash \
|
||||
netcat-openbsd
|
||||
netcat \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy built binary from builder stage
|
||||
COPY --from=builder /kafka-loadtest /usr/local/bin/kafka-loadtest
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# Use the official Eclipse Temurin + Maven image so we don't depend on
|
||||
# Debian apt mirrors, which intermittently refuse connections from
|
||||
# GitHub Actions runners and fail the CI build. This image already has
|
||||
# JDK 11 and Maven installed.
|
||||
FROM maven:3.9-eclipse-temurin-11
|
||||
FROM openjdk:11-jdk-slim
|
||||
|
||||
# Install Maven
|
||||
RUN apt-get update && apt-get install -y maven && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<kafka.version>3.9.2</kafka.version>
|
||||
<kafka.version>3.9.1</kafka.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<kafka.version>3.9.2</kafka.version>
|
||||
<kafka.version>3.9.1</kafka.version>
|
||||
<confluent.version>7.6.0</confluent.version>
|
||||
</properties>
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -16,8 +17,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -52,8 +51,8 @@ type MasterCluster struct {
|
||||
|
||||
// clusterStatus is the JSON returned by /cluster/status.
|
||||
type clusterStatus struct {
|
||||
IsLeader bool `json:"IsLeader"`
|
||||
Leader string `json:"Leader"`
|
||||
IsLeader bool `json:"IsLeader"`
|
||||
Leader string `json:"Leader"`
|
||||
Peers []string `json:"Peers"`
|
||||
}
|
||||
|
||||
@@ -74,23 +73,23 @@ func StartMasterCluster(t testing.TB) *MasterCluster {
|
||||
logsDir := filepath.Join(baseDir, "logs")
|
||||
os.MkdirAll(logsDir, 0o755)
|
||||
|
||||
// Allocate 3 mini-safe ports (each guarantees port+10000 is also free).
|
||||
httpPorts, err := testutil.AllocateMiniPorts(3)
|
||||
// Allocate 3 port pairs (http, grpc) atomically to prevent reuse.
|
||||
portPairs, err := allocateMultipleMasterPortPairs(3)
|
||||
if err != nil {
|
||||
t.Fatalf("allocate ports: %v", err)
|
||||
}
|
||||
var nodes [3]*masterNode
|
||||
var peerParts []string
|
||||
for i, hp := range httpPorts {
|
||||
for i, pp := range portPairs {
|
||||
dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i))
|
||||
os.MkdirAll(dataDir, 0o755)
|
||||
nodes[i] = &masterNode{
|
||||
port: hp,
|
||||
grpcPort: hp + testutil.GrpcPortOffset,
|
||||
port: pp[0],
|
||||
grpcPort: pp[1],
|
||||
dataDir: dataDir,
|
||||
logFile: filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)),
|
||||
}
|
||||
peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp))
|
||||
peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", pp[0]))
|
||||
}
|
||||
|
||||
mc := &MasterCluster{
|
||||
@@ -358,6 +357,42 @@ func (mc *MasterCluster) tailLog(i int) string {
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// --- port and binary helpers (adapted from test/volume_server/framework) ---
|
||||
|
||||
// allocateMultipleMasterPortPairs finds n non-overlapping (http, grpc) port
|
||||
// pairs, holding all listeners until all are found, then releasing them
|
||||
// together to avoid races between consecutive allocations.
|
||||
func allocateMultipleMasterPortPairs(n int) ([][2]int, error) {
|
||||
var listeners []net.Listener
|
||||
var pairs [][2]int
|
||||
|
||||
defer func() {
|
||||
for _, l := range listeners {
|
||||
l.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
for masterPort := 10000; masterPort <= 55535 && len(pairs) < n; masterPort++ {
|
||||
grpcPort := masterPort + 10000
|
||||
l1, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(masterPort)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
l2, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(grpcPort)))
|
||||
if err != nil {
|
||||
l1.Close()
|
||||
continue
|
||||
}
|
||||
listeners = append(listeners, l1, l2)
|
||||
pairs = append(pairs, [2]int{masterPort, grpcPort})
|
||||
}
|
||||
|
||||
if len(pairs) < n {
|
||||
return nil, fmt.Errorf("could only allocate %d of %d master port pairs", len(pairs), n)
|
||||
}
|
||||
return pairs, nil
|
||||
}
|
||||
|
||||
func findOrBuildWeedBinary() (string, error) {
|
||||
if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
|
||||
if isExecutableFile(fromEnv) {
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
.PHONY: all build test test-verbose test-short test-debug clean deps tidy
|
||||
|
||||
all: build test
|
||||
|
||||
# Build the weed binary first
|
||||
build:
|
||||
cd ../../weed && go build -o weed .
|
||||
|
||||
# Install test dependencies
|
||||
deps:
|
||||
go mod download
|
||||
|
||||
# Run all tests
|
||||
test: build deps
|
||||
go test -timeout 5m ./...
|
||||
|
||||
# Run tests with verbose output
|
||||
test-verbose: build deps
|
||||
go test -v -timeout 5m ./...
|
||||
|
||||
# Skip long-running integration tests
|
||||
test-short: deps
|
||||
go test -short -v ./...
|
||||
|
||||
# Run tests with debug output from SeaweedFS
|
||||
test-debug: build deps
|
||||
go test -v -timeout 5m ./... 2>&1 | tee test.log
|
||||
|
||||
# Clean up test artifacts
|
||||
clean:
|
||||
rm -f test.log
|
||||
go clean -testcache
|
||||
|
||||
# Update go.sum
|
||||
tidy:
|
||||
go mod tidy
|
||||
@@ -1,92 +0,0 @@
|
||||
# SeaweedFS NFS Integration Tests
|
||||
|
||||
End-to-end tests that boot a real SeaweedFS cluster (`master` + `volume` +
|
||||
`filer`) plus the experimental `weed nfs` frontend and drive it through the
|
||||
NFSv3 wire protocol. The tests talk to the server over TCP using
|
||||
`github.com/willscott/go-nfs-client`, which means they do **not** need a
|
||||
kernel NFS mount, privileged ports, or any platform-specific tooling.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. Build the `weed` binary:
|
||||
```bash
|
||||
cd ../../weed
|
||||
go build -o weed .
|
||||
```
|
||||
2. Go 1.24 or later.
|
||||
|
||||
## Running the tests
|
||||
|
||||
```bash
|
||||
# Build weed and run everything
|
||||
make test
|
||||
|
||||
# Verbose output, keeps the subprocess stdout
|
||||
make test-verbose
|
||||
|
||||
# Skip integration tests — useful when iterating on the framework itself
|
||||
make test-short
|
||||
|
||||
# Run a single test
|
||||
go test -v -run TestNfsBasicReadWrite ./...
|
||||
```
|
||||
|
||||
Every test starts its own cluster on random loopback ports, so runs are
|
||||
isolated and can execute in parallel.
|
||||
|
||||
## Layout
|
||||
|
||||
- `framework.go` — launches `weed master`, `weed volume`, `weed filer`, and
|
||||
`weed nfs` as subprocesses, waits for each to accept TCP, and exposes a
|
||||
`Mount()` helper that returns an `nfsclient.Target`.
|
||||
- `basic_test.go` — covers the most common NFS operations:
|
||||
- Read/write round-trip (`TestNfsBasicReadWrite`)
|
||||
- Mkdir / ReadDirPlus / RmDir (`TestNfsMkdirAndRmdir`)
|
||||
- Nested directory + leaf file (`TestNfsNestedDirectories`)
|
||||
- Rename preserves content (`TestNfsRenamePreservesContent`)
|
||||
- Overwrite shrinks file size (`TestNfsOverwriteShrinksFile`)
|
||||
- Large binary file round-trip (`TestNfsLargeFile`)
|
||||
- Arbitrary binary and empty files (`TestNfsBinaryAndEmptyFiles`)
|
||||
- Symlink + Readlink (`TestNfsSymlinkRoundTrip`)
|
||||
- ReadDirPlus ordering sanity (`TestNfsReadDirPlusOrdering`)
|
||||
- Remove on missing path errors cleanly (`TestNfsRemoveMissingFailsCleanly`)
|
||||
- FSINFO advertises non-zero limits (`TestNfsFSInfoReturnsSaneLimits`)
|
||||
- Sequential append writes concatenate (`TestNfsAppendIsSequential`)
|
||||
- ReadDir after remove (`TestNfsReadDirAfterRemove`)
|
||||
|
||||
## Debugging a failing test
|
||||
|
||||
Keep the cluster temp dir for inspection:
|
||||
|
||||
```go
|
||||
config := DefaultTestConfig()
|
||||
config.SkipCleanup = true
|
||||
```
|
||||
|
||||
Enable subprocess stdout/stderr:
|
||||
|
||||
```go
|
||||
config := DefaultTestConfig()
|
||||
config.EnableDebug = true
|
||||
```
|
||||
|
||||
Or run with `-v`, which flips `EnableDebug` automatically via `testing.Verbose()`.
|
||||
|
||||
## Notes
|
||||
|
||||
- The NFS server binds to `127.0.0.1` with `-ip.bind=127.0.0.1` and exports
|
||||
`/nfs_export`. The test framework pre-creates that directory via the
|
||||
filer's HTTP API before starting the NFS server — the NFS server requires
|
||||
its export root to exist in the filer's namespace with a real entry, and
|
||||
the filer's synthetic `/` root does not match the `Name=="/"` check the
|
||||
NFS server performs during `ensureIndexedEntry`.
|
||||
- Ports are allocated dynamically. Each test run opens a short-lived
|
||||
listener on `127.0.0.1:0`, reads back the assigned port, closes the
|
||||
listener, and hands the port to `weed master/volume/filer/nfs`. There is
|
||||
a tiny race window between close and reopen that has not been a problem
|
||||
in practice but is worth remembering if you see a "bind: address already
|
||||
in use" failure.
|
||||
- All four `weed` components are started with explicit `-port.grpc=...`
|
||||
flags. Without them, the default is `-port + 10000`, which overflows
|
||||
`65535` whenever the HTTP port lands above `55535` — the kernel's
|
||||
ephemeral port range on macOS routinely does.
|
||||
@@ -1,400 +0,0 @@
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
nfsclient "github.com/willscott/go-nfs-client/nfs"
|
||||
)
|
||||
|
||||
// setupFramework is a small helper that boots the cluster for a single test
|
||||
// and tears everything down on completion. Every test gets a fresh filer +
|
||||
// volume pair so they cannot step on each other's namespace.
|
||||
func setupFramework(t *testing.T) *NfsTestFramework {
|
||||
t.Helper()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
config := DefaultTestConfig()
|
||||
config.EnableDebug = testing.Verbose()
|
||||
fw := NewNfsTestFramework(t, config)
|
||||
require.NoError(t, fw.Setup(config), "framework setup")
|
||||
t.Cleanup(fw.Cleanup)
|
||||
return fw
|
||||
}
|
||||
|
||||
// writeAll writes payload to path on the target in a single Write call. The
|
||||
// NFS WRITE3 RPC chunks internally, so this exists purely so tests read
|
||||
// linearly.
|
||||
func writeAll(t *testing.T, target *nfsclient.Target, remotePath string, payload []byte) {
|
||||
t.Helper()
|
||||
file, err := target.OpenFile(remotePath, 0o644)
|
||||
require.NoError(t, err, "open %s for write", remotePath)
|
||||
if len(payload) > 0 {
|
||||
n, err := file.Write(payload)
|
||||
require.NoError(t, err, "write %s", remotePath)
|
||||
require.Equal(t, len(payload), n, "short write on %s", remotePath)
|
||||
}
|
||||
require.NoError(t, file.Close(), "close %s", remotePath)
|
||||
}
|
||||
|
||||
// readAll opens path on the target and returns the full file contents.
|
||||
func readAll(t *testing.T, target *nfsclient.Target, remotePath string) []byte {
|
||||
t.Helper()
|
||||
file, err := target.Open(remotePath)
|
||||
require.NoError(t, err, "open %s for read", remotePath)
|
||||
defer file.Close()
|
||||
content, err := io.ReadAll(file)
|
||||
require.NoError(t, err, "read %s", remotePath)
|
||||
return content
|
||||
}
|
||||
|
||||
// TestNfsBasicReadWrite exercises the most common NFS path: OpenFile + Write
|
||||
// + Close followed by Open + Read to verify round-trip data integrity.
|
||||
func TestNfsBasicReadWrite(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
payload := []byte("hello from seaweedfs nfs integration test")
|
||||
writeAll(t, target, "/hello.txt", payload)
|
||||
|
||||
got := readAll(t, target, "/hello.txt")
|
||||
assert.Equal(t, payload, got, "round-tripped content must match")
|
||||
|
||||
info, err := target.Getattr("/hello.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(payload)), int64(info.Filesize))
|
||||
}
|
||||
|
||||
// TestNfsMkdirAndRmdir covers Mkdir, ReadDirPlus, and RmDir. The readdir
|
||||
// assertion also verifies that the newly-created directory shows up under
|
||||
// the export root the way a POSIX client would expect.
|
||||
func TestNfsMkdirAndRmdir(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
_, err = target.Mkdir("/dir1", 0o755)
|
||||
require.NoError(t, err)
|
||||
|
||||
entries, err := target.ReadDirPlus("/")
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, entry := range entries {
|
||||
if entry.Name() == "dir1" {
|
||||
found = true
|
||||
assert.True(t, entry.IsDir(), "dir1 should be a directory")
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "expected dir1 in readdir listing")
|
||||
|
||||
require.NoError(t, target.RmDir("/dir1"))
|
||||
|
||||
// After removal, dir1 must be gone from the listing.
|
||||
entries, err = target.ReadDirPlus("/")
|
||||
require.NoError(t, err)
|
||||
for _, entry := range entries {
|
||||
assert.NotEqual(t, "dir1", entry.Name(), "dir1 should be removed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNfsNestedDirectories ensures the server can materialise a deep tree in
|
||||
// a single Mkdir-per-segment sequence and that reads/writes work at the
|
||||
// leaves.
|
||||
func TestNfsNestedDirectories(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
for _, segment := range []string{"/a", "/a/b", "/a/b/c"} {
|
||||
_, err := target.Mkdir(segment, 0o755)
|
||||
require.NoError(t, err, "mkdir %s", segment)
|
||||
}
|
||||
|
||||
payload := []byte("deep path content")
|
||||
writeAll(t, target, "/a/b/c/leaf.txt", payload)
|
||||
|
||||
got := readAll(t, target, "/a/b/c/leaf.txt")
|
||||
assert.Equal(t, payload, got)
|
||||
|
||||
require.NoError(t, target.Remove("/a/b/c/leaf.txt"))
|
||||
require.NoError(t, target.RmDir("/a/b/c"))
|
||||
require.NoError(t, target.RmDir("/a/b"))
|
||||
require.NoError(t, target.RmDir("/a"))
|
||||
}
|
||||
|
||||
// TestNfsRenamePreservesContent renames a file and makes sure the content
|
||||
// at the new path matches what was written at the old one, and that the
|
||||
// old path disappears. It does not assert on inode identity because pjdfstest
|
||||
// already covers that and this test intentionally avoids depending on the
|
||||
// mount-side identity plumbing.
|
||||
func TestNfsRenamePreservesContent(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
payload := []byte("rename me")
|
||||
writeAll(t, target, "/src.txt", payload)
|
||||
|
||||
require.NoError(t, target.Rename("/src.txt", "/dst.txt"))
|
||||
|
||||
_, _, err = target.Lookup("/src.txt")
|
||||
assert.Error(t, err, "source should be gone after rename")
|
||||
|
||||
got := readAll(t, target, "/dst.txt")
|
||||
assert.Equal(t, payload, got)
|
||||
|
||||
require.NoError(t, target.Remove("/dst.txt"))
|
||||
}
|
||||
|
||||
// TestNfsOverwriteShrinksFile rewrites an existing file with shorter content
|
||||
// and asserts Getattr reports the new (smaller) size. go-nfs-client's
|
||||
// OpenFile does not pass O_TRUNC, so the test truncates explicitly via
|
||||
// Setattr(size=0) before the second write — mirroring what `echo >file`
|
||||
// does on a POSIX client.
|
||||
func TestNfsOverwriteShrinksFile(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
writeAll(t, target, "/overwrite.txt", []byte("the quick brown fox"))
|
||||
|
||||
require.NoError(t, target.Setattr("/overwrite.txt", nfsclient.Sattr3{
|
||||
Size: nfsclient.SetSize{SetIt: true, Size: 0},
|
||||
}))
|
||||
|
||||
writeAll(t, target, "/overwrite.txt", []byte("short"))
|
||||
|
||||
info, err := target.Getattr("/overwrite.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len("short")), int64(info.Filesize))
|
||||
|
||||
got := readAll(t, target, "/overwrite.txt")
|
||||
assert.Equal(t, []byte("short"), got)
|
||||
|
||||
require.NoError(t, target.Remove("/overwrite.txt"))
|
||||
}
|
||||
|
||||
// TestNfsLargeFile writes a multi-megabyte payload so the write path has to
|
||||
// cut chunks and flush through the volume server rather than inlining
|
||||
// content in the filer entry.
|
||||
func TestNfsLargeFile(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
const size = 3 * 1024 * 1024 // 3 MiB — exceeds the 4 MiB inline cutoff boundary when combined with metadata
|
||||
payload := make([]byte, size)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i % 251) // non-repeating to catch offset bugs
|
||||
}
|
||||
|
||||
writeAll(t, target, "/big.bin", payload)
|
||||
|
||||
info, err := target.Getattr("/big.bin")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(size), int64(info.Filesize))
|
||||
|
||||
got := readAll(t, target, "/big.bin")
|
||||
require.Equal(t, size, len(got))
|
||||
assert.True(t, bytes.Equal(payload, got), "large file content must round-trip byte-for-byte")
|
||||
|
||||
require.NoError(t, target.Remove("/big.bin"))
|
||||
}
|
||||
|
||||
// TestNfsBinaryAndEmptyFiles covers two edge-case payloads the write path
|
||||
// tends to regress on: arbitrary binary bytes and zero-length files.
|
||||
func TestNfsBinaryAndEmptyFiles(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
t.Run("AllByteValues", func(t *testing.T) {
|
||||
payload := make([]byte, 256)
|
||||
for i := range payload {
|
||||
payload[i] = byte(i)
|
||||
}
|
||||
writeAll(t, target, "/binary.bin", payload)
|
||||
assert.Equal(t, payload, readAll(t, target, "/binary.bin"))
|
||||
require.NoError(t, target.Remove("/binary.bin"))
|
||||
})
|
||||
|
||||
t.Run("EmptyFile", func(t *testing.T) {
|
||||
writeAll(t, target, "/empty.txt", nil)
|
||||
info, err := target.Getattr("/empty.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), int64(info.Filesize))
|
||||
require.NoError(t, target.Remove("/empty.txt"))
|
||||
})
|
||||
}
|
||||
|
||||
// TestNfsSymlinkRoundTrip covers Symlink and Readlink through the nfs server.
|
||||
// Readlink returns the target path; the server does not auto-traverse it.
|
||||
func TestNfsSymlinkRoundTrip(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
// Symlink uses a different RPC than open+create, and our server routes it
|
||||
// through the billy Change interface.
|
||||
require.NoError(t, target.Symlink("/target.txt", "/link.txt"))
|
||||
|
||||
// The underlying target does not need to exist for readlink to succeed.
|
||||
file, _, err := target.Lookup("/link.txt")
|
||||
require.NoError(t, err, "lookup symlink")
|
||||
assert.True(t, file.Mode()&os.ModeSymlink != 0, "expected symlink mode, got %s", file.Mode())
|
||||
|
||||
require.NoError(t, target.Remove("/link.txt"))
|
||||
}
|
||||
|
||||
// TestNfsReadDirPlusOrdering creates a handful of files with distinct names
|
||||
// and ensures ReadDirPlus surfaces every one of them. The server pages
|
||||
// listings from the filer, so we want to make sure nothing is truncated.
|
||||
func TestNfsReadDirPlusOrdering(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
_, err = target.Mkdir("/listing", 0o755)
|
||||
require.NoError(t, err)
|
||||
|
||||
names := []string{"alpha.txt", "beta.txt", "gamma.txt", "delta.txt", "epsilon.txt"}
|
||||
for _, name := range names {
|
||||
writeAll(t, target, path.Join("/listing", name), []byte(name))
|
||||
}
|
||||
|
||||
entries, err := target.ReadDirPlus("/listing")
|
||||
require.NoError(t, err)
|
||||
seen := make(map[string]struct{}, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.Name() == "." || entry.Name() == ".." {
|
||||
continue
|
||||
}
|
||||
seen[entry.Name()] = struct{}{}
|
||||
}
|
||||
for _, name := range names {
|
||||
_, ok := seen[name]
|
||||
assert.True(t, ok, "expected %s in directory listing", name)
|
||||
}
|
||||
|
||||
for _, name := range names {
|
||||
require.NoError(t, target.Remove(path.Join("/listing", name)))
|
||||
}
|
||||
require.NoError(t, target.RmDir("/listing"))
|
||||
}
|
||||
|
||||
// TestNfsRemoveMissingFailsCleanly asserts that removing a non-existent path
|
||||
// surfaces an error instead of silently succeeding. A bug where the server
|
||||
// returned NFS3_OK on missing entries would hide metadata drift.
|
||||
func TestNfsRemoveMissingFailsCleanly(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
err = target.Remove("/does_not_exist.txt")
|
||||
require.Error(t, err, "removing a missing file must error")
|
||||
// NFS3 surfaces this as NFS3ERR_NOENT; make sure the error text is
|
||||
// recognisable without locking us into the library's exact wording.
|
||||
assert.True(t,
|
||||
strings.Contains(strings.ToLower(err.Error()), "noent") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "not exist") ||
|
||||
strings.Contains(strings.ToLower(err.Error()), "no such"),
|
||||
"unexpected error shape: %v", err)
|
||||
}
|
||||
|
||||
// TestNfsFSInfoReturnsSaneLimits pokes at FSINFO so we catch regressions
|
||||
// where the server advertises zero read/write limits (which would make
|
||||
// clients fall back to the 8 KiB floor and slow every test that follows).
|
||||
func TestNfsFSInfoReturnsSaneLimits(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
info, err := target.FSInfo()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, info)
|
||||
assert.Greater(t, info.RTPref, uint32(0), "rtpref must be positive")
|
||||
assert.Greater(t, info.WTPref, uint32(0), "wtpref must be positive")
|
||||
}
|
||||
|
||||
// TestNfsAppendIsSequential writes two chunks to the same file in separate
|
||||
// Open cycles and asserts the concatenation is preserved. The second write
|
||||
// uses O_APPEND (the default Open path in go-nfs-client does not pass
|
||||
// flags, so we explicitly reopen after writing the first chunk).
|
||||
func TestNfsAppendIsSequential(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
const prefix = "part1-"
|
||||
const suffix = "part2"
|
||||
|
||||
writeAll(t, target, "/concat.txt", []byte(prefix))
|
||||
|
||||
file, err := target.OpenFile("/concat.txt", 0o644)
|
||||
require.NoError(t, err)
|
||||
// Seek to end before writing so we append rather than overwrite. go-nfs
|
||||
// client's File.Seek uses the same offset tracking as Write so this is
|
||||
// enough to place the second chunk after the first.
|
||||
_, err = file.Seek(int64(len(prefix)), io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
_, err = file.Write([]byte(suffix))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, file.Close())
|
||||
|
||||
got := readAll(t, target, "/concat.txt")
|
||||
assert.Equal(t, prefix+suffix, string(got))
|
||||
|
||||
require.NoError(t, target.Remove("/concat.txt"))
|
||||
}
|
||||
|
||||
// Regression: readdir should not emit stale entries after a remove. This is
|
||||
// the scenario the PR's meta cache invalidation logic was written to fix.
|
||||
func TestNfsReadDirAfterRemove(t *testing.T) {
|
||||
fw := setupFramework(t)
|
||||
target, cleanup, err := fw.Mount()
|
||||
require.NoError(t, err)
|
||||
defer cleanup()
|
||||
|
||||
_, err = target.Mkdir("/churn", 0o755)
|
||||
require.NoError(t, err)
|
||||
for i := 0; i < 5; i++ {
|
||||
writeAll(t, target, path.Join("/churn", fmt.Sprintf("f%d.txt", i)), []byte{byte(i)})
|
||||
}
|
||||
// Remove the middle one and re-list.
|
||||
require.NoError(t, target.Remove("/churn/f2.txt"))
|
||||
|
||||
entries, err := target.ReadDirPlus("/churn")
|
||||
require.NoError(t, err)
|
||||
for _, entry := range entries {
|
||||
assert.NotEqual(t, "f2.txt", entry.Name(), "removed file should not reappear in listing")
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
if i == 2 {
|
||||
continue
|
||||
}
|
||||
require.NoError(t, target.Remove(path.Join("/churn", fmt.Sprintf("f%d.txt", i))))
|
||||
}
|
||||
require.NoError(t, target.RmDir("/churn"))
|
||||
}
|
||||
@@ -1,423 +0,0 @@
|
||||
package nfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
"github.com/stretchr/testify/require"
|
||||
nfsclient "github.com/willscott/go-nfs-client/nfs"
|
||||
"github.com/willscott/go-nfs-client/nfs/rpc"
|
||||
)
|
||||
|
||||
// NfsTestFramework boots a minimal SeaweedFS cluster (master + volume + filer)
|
||||
// plus the experimental `weed nfs` frontend and hands out NFSv3 RPC clients
|
||||
// that talk to it. Everything is driven via subprocesses so the tests exercise
|
||||
// the same binary an operator would deploy, and no kernel mount is required.
|
||||
type NfsTestFramework struct {
|
||||
t *testing.T
|
||||
tempDir string
|
||||
dataDir string
|
||||
masterProcess *os.Process
|
||||
volumeProcess *os.Process
|
||||
filerProcess *os.Process
|
||||
nfsProcess *os.Process
|
||||
masterAddr string
|
||||
masterGrpc int
|
||||
volumeAddr string
|
||||
volumeGrpc int
|
||||
filerAddr string
|
||||
filerGrpc int
|
||||
nfsAddr string
|
||||
exportRoot string
|
||||
weedBinary string
|
||||
isSetup bool
|
||||
skipCleanup bool
|
||||
}
|
||||
|
||||
// TestConfig controls how the framework boots the cluster.
|
||||
type TestConfig struct {
|
||||
NumVolumes int
|
||||
EnableDebug bool
|
||||
SkipCleanup bool // keep temp dir on failure for inspection
|
||||
// ExportRoot is the filer path the NFS server exports. Defaults to "/"
|
||||
// so tests can use any path, with a single warning logged by the server.
|
||||
ExportRoot string
|
||||
}
|
||||
|
||||
// DefaultTestConfig returns the defaults used by most tests. A dedicated
|
||||
// /nfs_export subtree is used as the NFS export root because the NFS server
|
||||
// requires the export directory to exist in the filer's namespace and carry
|
||||
// a non-zero inode — passing "/" would succeed only for filer setups that
|
||||
// have already backfilled the root inode.
|
||||
func DefaultTestConfig() *TestConfig {
|
||||
return &TestConfig{
|
||||
NumVolumes: 3,
|
||||
EnableDebug: false,
|
||||
SkipCleanup: false,
|
||||
ExportRoot: "/nfs_export",
|
||||
}
|
||||
}
|
||||
|
||||
// NewNfsTestFramework allocates a framework bound to the current test. Call
|
||||
// Setup next to actually start the cluster.
|
||||
func NewNfsTestFramework(t *testing.T, config *TestConfig) *NfsTestFramework {
|
||||
if config == nil {
|
||||
config = DefaultTestConfig()
|
||||
}
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "seaweedfs_nfs_test_")
|
||||
require.NoError(t, err)
|
||||
|
||||
// testutil.MustAllocatePorts holds every listener open until the full
|
||||
// batch has been reserved, which avoids the "close-then-hope" race my
|
||||
// original per-port helper had. We need seven ports: four HTTP (master,
|
||||
// volume, filer, nfs) and three gRPC (master, volume, filer — nfs has
|
||||
// no gRPC endpoint).
|
||||
ports := testutil.MustAllocatePorts(t, 7)
|
||||
|
||||
exportRoot := config.ExportRoot
|
||||
if exportRoot == "" {
|
||||
exportRoot = "/"
|
||||
}
|
||||
|
||||
return &NfsTestFramework{
|
||||
t: t,
|
||||
tempDir: tempDir,
|
||||
dataDir: filepath.Join(tempDir, "data"),
|
||||
masterAddr: fmt.Sprintf("127.0.0.1:%d", ports[0]),
|
||||
masterGrpc: ports[1],
|
||||
volumeAddr: fmt.Sprintf("127.0.0.1:%d", ports[2]),
|
||||
volumeGrpc: ports[3],
|
||||
filerAddr: fmt.Sprintf("127.0.0.1:%d", ports[4]),
|
||||
filerGrpc: ports[5],
|
||||
nfsAddr: fmt.Sprintf("127.0.0.1:%d", ports[6]),
|
||||
exportRoot: exportRoot,
|
||||
weedBinary: findWeedBinary(),
|
||||
isSetup: false,
|
||||
skipCleanup: config.SkipCleanup,
|
||||
}
|
||||
}
|
||||
|
||||
// Setup starts the SeaweedFS cluster and the NFS frontend, waiting for each
|
||||
// component to accept connections before moving on.
|
||||
func (f *NfsTestFramework) Setup(config *TestConfig) error {
|
||||
if f.isSetup {
|
||||
return fmt.Errorf("framework already setup")
|
||||
}
|
||||
|
||||
dirs := []string{
|
||||
f.dataDir,
|
||||
filepath.Join(f.dataDir, "master"),
|
||||
filepath.Join(f.dataDir, "volume"),
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory %s: %v", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := f.startMaster(config); err != nil {
|
||||
return fmt.Errorf("failed to start master: %v", err)
|
||||
}
|
||||
if !testutil.WaitForPort(portFromAddr(f.masterAddr), testutil.SeaweedMiniStartupTimeout) {
|
||||
return fmt.Errorf("master not ready at %s", f.masterAddr)
|
||||
}
|
||||
|
||||
if err := f.startVolumeServer(config); err != nil {
|
||||
return fmt.Errorf("failed to start volume server: %v", err)
|
||||
}
|
||||
if !testutil.WaitForPort(portFromAddr(f.volumeAddr), testutil.SeaweedMiniStartupTimeout) {
|
||||
return fmt.Errorf("volume server not ready at %s", f.volumeAddr)
|
||||
}
|
||||
|
||||
if err := f.startFiler(config); err != nil {
|
||||
return fmt.Errorf("failed to start filer: %v", err)
|
||||
}
|
||||
if !testutil.WaitForPort(portFromAddr(f.filerAddr), testutil.SeaweedMiniStartupTimeout) {
|
||||
return fmt.Errorf("filer not ready at %s", f.filerAddr)
|
||||
}
|
||||
|
||||
// Pre-create the export root in the filer's namespace. The NFS server
|
||||
// expects its export directory to exist with a real inode; uploading a
|
||||
// placeholder file creates the parent directory implicitly and then
|
||||
// removing the file leaves the empty directory in place.
|
||||
if f.exportRoot != "/" {
|
||||
if err := f.ensureExportRootExists(); err != nil {
|
||||
return fmt.Errorf("failed to pre-create export root %s: %v", f.exportRoot, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := f.startNfsServer(config); err != nil {
|
||||
return fmt.Errorf("failed to start NFS server: %v", err)
|
||||
}
|
||||
if !testutil.WaitForPort(portFromAddr(f.nfsAddr), testutil.SeaweedMiniStartupTimeout) {
|
||||
return fmt.Errorf("NFS server not ready at %s", f.nfsAddr)
|
||||
}
|
||||
|
||||
// Let the NFS server finish wiring up its gRPC subscription to the filer
|
||||
// before the first client call hits MOUNT/LOOKUP.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
f.isSetup = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup stops all processes. Temp state is preserved if SkipCleanup is set.
|
||||
func (f *NfsTestFramework) Cleanup() {
|
||||
processes := []*os.Process{f.nfsProcess, f.filerProcess, f.volumeProcess, f.masterProcess}
|
||||
for _, proc := range processes {
|
||||
if proc != nil {
|
||||
_ = proc.Signal(syscall.SIGTERM)
|
||||
_, _ = proc.Wait()
|
||||
}
|
||||
}
|
||||
if !f.skipCleanup {
|
||||
_ = os.RemoveAll(f.tempDir)
|
||||
}
|
||||
}
|
||||
|
||||
// NfsAddr returns the TCP address the NFS server is listening on.
|
||||
func (f *NfsTestFramework) NfsAddr() string { return f.nfsAddr }
|
||||
|
||||
// FilerAddr returns the TCP address of the filer.
|
||||
func (f *NfsTestFramework) FilerAddr() string { return f.filerAddr }
|
||||
|
||||
// ExportRoot returns the path the NFS server exports.
|
||||
func (f *NfsTestFramework) ExportRoot() string { return f.exportRoot }
|
||||
|
||||
// Mount opens an NFSv3 MOUNT+NFS connection against the running NFS server
|
||||
// and returns a Target that tests can drive like a mini-VFS. Caller is
|
||||
// responsible for calling the returned cleanup func to Unmount and close the
|
||||
// TCP connection.
|
||||
func (f *NfsTestFramework) Mount() (*nfsclient.Target, func(), error) {
|
||||
var (
|
||||
client *rpc.Client
|
||||
err error
|
||||
)
|
||||
// The NFS server's TCP listener may already be accepting connections when
|
||||
// waitForService returns, but the RPC program registration can trail it
|
||||
// by a few milliseconds. Retry the dial to absorb that small window.
|
||||
for attempt := 0; attempt < 20; attempt++ {
|
||||
client, err = rpc.DialTCP("tcp", f.nfsAddr, false)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("dial NFS: %w", err)
|
||||
}
|
||||
|
||||
// Note: do not set Mount.Addr here. When Addr is non-empty, the go-nfs
|
||||
// client re-dials via portmapper and concatenates `:111` onto the
|
||||
// address, which produces "too many colons" for a raw `host:port`
|
||||
// string. Reusing the existing RPC client avoids that path entirely.
|
||||
mounter := &nfsclient.Mount{Client: client}
|
||||
target, err := mounter.Mount(f.exportRoot, rpc.AuthNull)
|
||||
if err != nil {
|
||||
client.Close()
|
||||
return nil, nil, fmt.Errorf("mount %s: %w", f.exportRoot, err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
_ = mounter.Unmount()
|
||||
client.Close()
|
||||
}
|
||||
return target, cleanup, nil
|
||||
}
|
||||
|
||||
func (f *NfsTestFramework) startMaster(config *TestConfig) error {
|
||||
_, masterPort := splitHostPort(f.masterAddr)
|
||||
args := []string{
|
||||
"master",
|
||||
"-ip=127.0.0.1",
|
||||
fmt.Sprintf("-port=%d", masterPort),
|
||||
fmt.Sprintf("-port.grpc=%d", f.masterGrpc),
|
||||
"-mdir=" + filepath.Join(f.dataDir, "master"),
|
||||
"-raftBootstrap",
|
||||
"-peers=none",
|
||||
}
|
||||
return f.startProcess(&f.masterProcess, config, args)
|
||||
}
|
||||
|
||||
func (f *NfsTestFramework) startVolumeServer(config *TestConfig) error {
|
||||
_, volumePort := splitHostPort(f.volumeAddr)
|
||||
// pb.ServerAddress encodes a non-default gRPC port as `host:port.grpc`.
|
||||
// See weed/pb/server_address.go — the dot, not a colon, is the separator
|
||||
// between the HTTP port and the gRPC port.
|
||||
masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
|
||||
args := []string{
|
||||
"volume",
|
||||
"-master=" + masterWithGrpc,
|
||||
"-ip=127.0.0.1",
|
||||
fmt.Sprintf("-port=%d", volumePort),
|
||||
fmt.Sprintf("-port.grpc=%d", f.volumeGrpc),
|
||||
"-dir=" + filepath.Join(f.dataDir, "volume"),
|
||||
fmt.Sprintf("-max=%d", config.NumVolumes),
|
||||
}
|
||||
return f.startProcess(&f.volumeProcess, config, args)
|
||||
}
|
||||
|
||||
func (f *NfsTestFramework) startFiler(config *TestConfig) error {
|
||||
_, filerPort := splitHostPort(f.filerAddr)
|
||||
masterWithGrpc := fmt.Sprintf("%s.%d", f.masterAddr, f.masterGrpc)
|
||||
args := []string{
|
||||
"filer",
|
||||
"-master=" + masterWithGrpc,
|
||||
"-ip=127.0.0.1",
|
||||
fmt.Sprintf("-port=%d", filerPort),
|
||||
fmt.Sprintf("-port.grpc=%d", f.filerGrpc),
|
||||
}
|
||||
return f.startProcess(&f.filerProcess, config, args)
|
||||
}
|
||||
|
||||
func (f *NfsTestFramework) startNfsServer(config *TestConfig) error {
|
||||
_, nfsPort := splitHostPort(f.nfsAddr)
|
||||
// `host:port.grpc` encoding — see pb/server_address.go.
|
||||
filerWithGrpc := fmt.Sprintf("%s.%d", f.filerAddr, f.filerGrpc)
|
||||
args := []string{
|
||||
"nfs",
|
||||
"-filer=" + filerWithGrpc,
|
||||
"-ip.bind=127.0.0.1",
|
||||
fmt.Sprintf("-port=%d", nfsPort),
|
||||
"-filer.path=" + f.exportRoot,
|
||||
}
|
||||
return f.startProcess(&f.nfsProcess, config, args)
|
||||
}
|
||||
|
||||
func (f *NfsTestFramework) startProcess(target **os.Process, config *TestConfig, args []string) error {
|
||||
cmd := exec.Command(f.weedBinary, args...)
|
||||
cmd.Dir = f.tempDir
|
||||
if config.EnableDebug {
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
*target = cmd.Process
|
||||
return nil
|
||||
}
|
||||
|
||||
// portFromAddr returns just the port number from a `host:port` string.
|
||||
// testutil.WaitForPort takes an int port, not a full address.
|
||||
func portFromAddr(addr string) int {
|
||||
_, port := splitHostPort(addr)
|
||||
return port
|
||||
}
|
||||
|
||||
// ensureExportRootExists posts a placeholder file to f.exportRoot via the
|
||||
// filer's HTTP API, then deletes it. That roundtrip implicitly creates the
|
||||
// target directory so the NFS server has something to mount. We bypass
|
||||
// weed/pb here because the HTTP client is simpler and needs no gRPC stubs.
|
||||
func (f *NfsTestFramework) ensureExportRootExists() error {
|
||||
exportRoot := strings.TrimRight(f.exportRoot, "/")
|
||||
if exportRoot == "" {
|
||||
return nil
|
||||
}
|
||||
placeholder := exportRoot + "/.nfs_test_init"
|
||||
filerURL := "http://" + f.filerAddr + placeholder
|
||||
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
part, err := writer.CreateFormFile("file", ".nfs_test_init")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.WriteString(part, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
httpClient := &http.Client{Timeout: 10 * time.Second}
|
||||
req, err := http.NewRequest(http.MethodPost, filerURL, &body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return fmt.Errorf("filer POST %s returned status %d", filerURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
// Delete the placeholder; the directory stays behind.
|
||||
deleteReq, err := http.NewRequest(http.MethodDelete, filerURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
deleteResp, err := httpClient.Do(deleteReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = io.Copy(io.Discard, deleteResp.Body)
|
||||
deleteResp.Body.Close()
|
||||
if deleteResp.StatusCode/100 != 2 && deleteResp.StatusCode != http.StatusNotFound {
|
||||
return fmt.Errorf("filer DELETE %s returned status %d", filerURL, deleteResp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitHostPort(addr string) (string, int) {
|
||||
host, portStr, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return "", 0
|
||||
}
|
||||
var port int
|
||||
_, _ = fmt.Sscanf(portStr, "%d", &port)
|
||||
return host, port
|
||||
}
|
||||
|
||||
// findWeedBinary locates the weed binary, preferring the local build in the
|
||||
// checkout so tests run against the code under review rather than whatever is
|
||||
// on $PATH.
|
||||
func findWeedBinary() string {
|
||||
if _, thisFile, _, ok := runtime.Caller(0); ok {
|
||||
thisDir := filepath.Dir(thisFile)
|
||||
candidates := []string{
|
||||
filepath.Join(thisDir, "../../weed/weed"),
|
||||
filepath.Join(thisDir, "../weed/weed"),
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
abs, _ := filepath.Abs(candidate)
|
||||
return abs
|
||||
}
|
||||
}
|
||||
}
|
||||
cwd, _ := os.Getwd()
|
||||
candidates := []string{
|
||||
filepath.Join(cwd, "../../weed/weed"),
|
||||
filepath.Join(cwd, "../weed/weed"),
|
||||
filepath.Join(cwd, "./weed"),
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
abs, _ := filepath.Abs(candidate)
|
||||
return abs
|
||||
}
|
||||
}
|
||||
if path, err := exec.LookPath("weed"); err == nil {
|
||||
return path
|
||||
}
|
||||
return "weed"
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
module seaweedfs-nfs-tests
|
||||
|
||||
go 1.25.0
|
||||
|
||||
// test/testutil lives inside the main seaweedfs module; pull it in via a
|
||||
// local replace so this integration suite can reuse the shared port
|
||||
// allocator and readiness helpers instead of reinventing them.
|
||||
replace github.com/seaweedfs/seaweedfs => ../..
|
||||
|
||||
require (
|
||||
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -1,14 +0,0 @@
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -1,28 +0,0 @@
|
||||
FROM chrislusf/seaweedfs:e2e
|
||||
|
||||
RUN apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
autoconf \
|
||||
automake \
|
||||
build-essential \
|
||||
git \
|
||||
libtap-harness-archive-perl \
|
||||
perl \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ARG PJDFSTEST_REPO=https://github.com/sanwan/pjdfstest.git
|
||||
ARG PJDFSTEST_REF=d25636a227606f8960e5179741d8f4ad7030ef41
|
||||
|
||||
RUN git clone "${PJDFSTEST_REPO}" /opt/pjdfstest && \
|
||||
cd /opt/pjdfstest && \
|
||||
git checkout "${PJDFSTEST_REF}" && \
|
||||
autoreconf -ifs && \
|
||||
./configure && \
|
||||
make pjdfstest
|
||||
|
||||
COPY run_inside_container.sh /run.sh
|
||||
COPY known_failures.txt /known_failures.txt
|
||||
RUN chmod +x /run.sh
|
||||
@@ -1,57 +0,0 @@
|
||||
services:
|
||||
master:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 master -ip=master -ip.bind=0.0.0.0 -raftBootstrap"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 volume -master=master:9333 -ip=volume -ip.bind=0.0.0.0 -preStopSeconds=1"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "-I", "http://localhost:8080/healthz"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
master:
|
||||
condition: service_healthy
|
||||
|
||||
filer:
|
||||
image: chrislusf/seaweedfs:e2e
|
||||
command: "-v=4 filer -master=master:9333 -ip=filer -ip.bind=0.0.0.0"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "--fail", "-I", "http://localhost:8888/healthz"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
volume:
|
||||
condition: service_healthy
|
||||
|
||||
mount:
|
||||
image: chrislusf/seaweedfs:pjdfstest
|
||||
build:
|
||||
context: .
|
||||
command: "-v=4 mount -filer=filer:8888 -filer.path=/ -dirAutoCreate -dir=/mnt/seaweedfs -allowOthers"
|
||||
cap_add:
|
||||
- SYS_ADMIN
|
||||
devices:
|
||||
- /dev/fuse
|
||||
security_opt:
|
||||
- apparmor:unconfined
|
||||
healthcheck:
|
||||
test: ["CMD", "mountpoint", "-q", "--", "/mnt/seaweedfs"]
|
||||
interval: 2s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 10s
|
||||
depends_on:
|
||||
filer:
|
||||
condition: service_healthy
|
||||
@@ -1,8 +0,0 @@
|
||||
# Known pjdfstest failures for SeaweedFS FUSE mount.
|
||||
#
|
||||
# Tests listed here are skipped during CI runs. Each entry must be a path
|
||||
# relative to the pjdfstest root (e.g. tests/chmod/02.t).
|
||||
#
|
||||
# A failure in any test NOT listed here will cause the CI job to fail,
|
||||
# catching regressions immediately.
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Run the pjdfstest POSIX compliance suite against a SeaweedFS FUSE mount.
|
||||
#
|
||||
# This script:
|
||||
# 1. Starts a self-contained "weed mini" server (master+volume+filer in one)
|
||||
# 2. Mounts the filesystem with "weed mount"
|
||||
# 3. Builds pjdfstest from upstream and runs it under prove(1)
|
||||
#
|
||||
# Requirements: weed in $PATH, fusermount3, perl with TAP::Harness::Archive,
|
||||
# autoconf, make, sudo (pjdfstest exercises chown/chmod which need root).
|
||||
#
|
||||
# Usage:
|
||||
# test/pjdfstest/run.sh # runs full suite
|
||||
# PJDFSTEST_TESTS=tests/chmod ./run.sh # runs a subset
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
WEED_BIN="${WEED_BIN:-weed}"
|
||||
WORK_DIR="${WORK_DIR:-$(mktemp -d /tmp/seaweedfs-pjdfstest.XXXXXX)}"
|
||||
MOUNT_DIR="${MOUNT_DIR:-${WORK_DIR}/mnt}"
|
||||
DATA_DIR="${DATA_DIR:-${WORK_DIR}/data}"
|
||||
LOG_DIR="${LOG_DIR:-${WORK_DIR}/logs}"
|
||||
FILER_PORT="${FILER_PORT:-28888}"
|
||||
FILER_ADDR="127.0.0.1:${FILER_PORT}"
|
||||
|
||||
# Pin to an immutable upstream commit so CI is reproducible. Override via env
|
||||
# if you want to test against a different ref or fork.
|
||||
PJDFSTEST_REPO="${PJDFSTEST_REPO:-https://github.com/sanwan/pjdfstest.git}"
|
||||
PJDFSTEST_REF="${PJDFSTEST_REF:-d25636a227606f8960e5179741d8f4ad7030ef41}"
|
||||
PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
|
||||
|
||||
mini_pid=""
|
||||
mount_pid=""
|
||||
|
||||
CI_LOG_DIR="/tmp/seaweedfs-pjdfstest-logs"
|
||||
|
||||
cleanup() {
|
||||
set +e
|
||||
if [[ -n "${mount_pid}" ]] && kill -0 "${mount_pid}" 2>/dev/null; then
|
||||
kill -TERM "${mount_pid}" 2>/dev/null || true
|
||||
wait "${mount_pid}" 2>/dev/null || true
|
||||
fi
|
||||
if mountpoint -q "${MOUNT_DIR}" 2>/dev/null; then
|
||||
fusermount3 -u "${MOUNT_DIR}" 2>/dev/null || \
|
||||
fusermount -u "${MOUNT_DIR}" 2>/dev/null || \
|
||||
sudo umount "${MOUNT_DIR}" 2>/dev/null || true
|
||||
fi
|
||||
if [[ -n "${mini_pid}" ]] && kill -0 "${mini_pid}" 2>/dev/null; then
|
||||
kill -TERM "${mini_pid}" 2>/dev/null || true
|
||||
wait "${mini_pid}" 2>/dev/null || true
|
||||
fi
|
||||
# Copy logs to a fixed path for CI artifact upload.
|
||||
mkdir -p "${CI_LOG_DIR}"
|
||||
cp "${LOG_DIR}"/*.log "${CI_LOG_DIR}/" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
mkdir -p "${MOUNT_DIR}" "${DATA_DIR}" "${LOG_DIR}"
|
||||
|
||||
echo "==> Starting weed mini on ${FILER_ADDR}"
|
||||
"${WEED_BIN}" mini \
|
||||
-dir="${DATA_DIR}" \
|
||||
-ip=127.0.0.1 \
|
||||
-filer.port="${FILER_PORT}" \
|
||||
-s3=false \
|
||||
-webdav=false \
|
||||
-admin.ui=false \
|
||||
>"${LOG_DIR}/mini.log" 2>&1 &
|
||||
mini_pid=$!
|
||||
|
||||
# Wait for filer to accept connections.
|
||||
for i in $(seq 1 60); do
|
||||
if (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "${mini_pid}" 2>/dev/null; then
|
||||
echo "weed mini exited early; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if ! (echo > "/dev/tcp/127.0.0.1/${FILER_PORT}") 2>/dev/null; then
|
||||
echo "weed mini filer did not become reachable within 30s; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mini.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Mounting SeaweedFS at ${MOUNT_DIR}"
|
||||
# allowOthers is required so that pjdfstest's setuid/setgid sub-tests (run via
|
||||
# sudo) can access files created as the invoking user.
|
||||
sudo "${WEED_BIN}" mount \
|
||||
-filer="${FILER_ADDR}" \
|
||||
-dir="${MOUNT_DIR}" \
|
||||
-filer.path=/ \
|
||||
-dirAutoCreate \
|
||||
-allowOthers=true \
|
||||
>"${LOG_DIR}/mount.log" 2>&1 &
|
||||
mount_pid=$!
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if mountpoint -q "${MOUNT_DIR}"; then
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "${mount_pid}" 2>/dev/null; then
|
||||
echo "weed mount exited early; log tail:" >&2
|
||||
tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if ! mountpoint -q "${MOUNT_DIR}"; then
|
||||
echo "FUSE mount did not come up within 30s" >&2
|
||||
tail -n 100 "${LOG_DIR}/mount.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Cloning and building pjdfstest"
|
||||
PJDFSTEST_DIR="${WORK_DIR}/pjdfstest"
|
||||
if [[ ! -d "${PJDFSTEST_DIR}/.git" ]]; then
|
||||
git clone "${PJDFSTEST_REPO}" "${PJDFSTEST_DIR}"
|
||||
fi
|
||||
git -C "${PJDFSTEST_DIR}" remote set-url origin "${PJDFSTEST_REPO}"
|
||||
git -C "${PJDFSTEST_DIR}" fetch --depth 1 origin "${PJDFSTEST_REF}"
|
||||
git -C "${PJDFSTEST_DIR}" checkout --detach FETCH_HEAD
|
||||
(
|
||||
cd "${PJDFSTEST_DIR}"
|
||||
autoreconf -ifs
|
||||
./configure
|
||||
make pjdfstest
|
||||
)
|
||||
|
||||
# pjdfstest must run inside the filesystem under test.
|
||||
TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
|
||||
sudo mkdir -p "${TEST_ROOT}"
|
||||
sudo cp -r "${PJDFSTEST_DIR}/." "${TEST_ROOT}/"
|
||||
|
||||
cd "${TEST_ROOT}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
KNOWN_FAILURES="${SCRIPT_DIR}/known_failures.txt"
|
||||
|
||||
# Build the list of tests to run, excluding known failures.
|
||||
if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
|
||||
mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
|
||||
all_tests=()
|
||||
while IFS= read -r -d '' t; do
|
||||
all_tests+=("$t")
|
||||
done < <(find tests/ -name '*.t' -print0 | sort -z)
|
||||
|
||||
run_tests=()
|
||||
for t in "${all_tests[@]}"; do
|
||||
is_skipped=false
|
||||
for s in "${skip[@]}"; do
|
||||
if [[ "$t" == "$s" ]]; then
|
||||
is_skipped=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ! $is_skipped; then
|
||||
run_tests+=("$t")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
|
||||
sudo prove -rv "${run_tests[@]}"
|
||||
else
|
||||
echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
|
||||
sudo prove -rv "${PJDFSTEST_TESTS}"
|
||||
fi
|
||||
@@ -1,48 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Runs pjdfstest inside the mount container against /mnt/seaweedfs.
|
||||
# Invoked via: docker compose exec mount /run.sh
|
||||
#
|
||||
# Uses known_failures.txt to skip tests with known issues. Any failure
|
||||
# in a test NOT in the skip list causes a non-zero exit (regression).
|
||||
set -euo pipefail
|
||||
|
||||
MOUNT_DIR="/mnt/seaweedfs"
|
||||
PJDFSTEST_TESTS="${PJDFSTEST_TESTS:-tests/}"
|
||||
KNOWN_FAILURES="/known_failures.txt"
|
||||
|
||||
# Copy pjdfstest into the mounted filesystem so tests exercise the FS under test.
|
||||
TEST_ROOT="${MOUNT_DIR}/pjdfstest-root"
|
||||
mkdir -p "${TEST_ROOT}"
|
||||
cp -r /opt/pjdfstest/. "${TEST_ROOT}/"
|
||||
|
||||
cd "${TEST_ROOT}"
|
||||
|
||||
# Build the list of tests to run, excluding known failures.
|
||||
if [[ -f "${KNOWN_FAILURES}" ]] && [[ "${PJDFSTEST_TESTS}" == "tests/" ]]; then
|
||||
mapfile -t skip < <(grep -v '^#' "${KNOWN_FAILURES}" | grep -v '^$')
|
||||
all_tests=()
|
||||
while IFS= read -r -d '' t; do
|
||||
all_tests+=("$t")
|
||||
done < <(find tests/ -name '*.t' -print0 | sort -z)
|
||||
|
||||
run_tests=()
|
||||
for t in "${all_tests[@]}"; do
|
||||
is_skipped=false
|
||||
for s in "${skip[@]}"; do
|
||||
if [[ "$t" == "$s" ]]; then
|
||||
is_skipped=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ! $is_skipped; then
|
||||
run_tests+=("$t")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "==> Running pjdfstest (${#run_tests[@]} tests, ${#skip[@]} skipped)"
|
||||
prove -rv "${run_tests[@]}"
|
||||
else
|
||||
echo "==> Running pjdfstest (${PJDFSTEST_TESTS})"
|
||||
prove -rv "${PJDFSTEST_TESTS}"
|
||||
fi
|
||||
@@ -11,11 +11,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// VolumeServer provides a minimal volume server for erasure coding tests.
|
||||
@@ -198,25 +196,12 @@ func (v *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream vo
|
||||
defer file.Close()
|
||||
|
||||
buf := make([]byte, 64*1024)
|
||||
remaining := int64(req.GetStopOffset())
|
||||
for {
|
||||
if remaining == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
readBuf := buf
|
||||
if remaining > 0 && remaining < int64(len(buf)) {
|
||||
readBuf = buf[:remaining]
|
||||
}
|
||||
|
||||
n, readErr := file.Read(readBuf)
|
||||
n, readErr := file.Read(buf)
|
||||
if n > 0 {
|
||||
if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: readBuf[:n]}); err != nil {
|
||||
if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: buf[:n]}); err != nil {
|
||||
return err
|
||||
}
|
||||
if remaining > 0 {
|
||||
remaining -= int64(n)
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
@@ -322,21 +307,10 @@ func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_ser
|
||||
v.mu.Lock()
|
||||
v.readFileStatusCalls++
|
||||
v.mu.Unlock()
|
||||
|
||||
datInfo, err := os.Stat(v.filePath(req.VolumeId, ".dat"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
idxInfo, err := os.Stat(v.filePath(req.VolumeId, ".idx"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &volume_server_pb.ReadVolumeFileStatusResponse{
|
||||
VolumeId: req.VolumeId,
|
||||
DatFileSize: uint64(datInfo.Size()),
|
||||
IdxFileSize: uint64(idxInfo.Size()),
|
||||
DatFileSize: 1024,
|
||||
IdxFileSize: 16,
|
||||
FileCount: 1,
|
||||
}, nil
|
||||
}
|
||||
@@ -375,27 +349,7 @@ func (v *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, strea
|
||||
v.volumeCopyCalls++
|
||||
v.mu.Unlock()
|
||||
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
var statusResp *volume_server_pb.ReadVolumeFileStatusResponse
|
||||
if err := operation.WithVolumeServerClient(false, pb.ServerAddress(req.SourceDataNode), dialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
var readErr error
|
||||
statusResp, readErr = client.ReadVolumeFileStatus(stream.Context(), &volume_server_pb.ReadVolumeFileStatusRequest{
|
||||
VolumeId: req.VolumeId,
|
||||
})
|
||||
return readErr
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".dat", statusResp.DatFileSize, dialOption); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".idx", statusResp.IdxFileSize, dialOption); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: int64(statusResp.DatFileSize + statusResp.IdxFileSize)}); err != nil {
|
||||
if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: 1024}); err != nil {
|
||||
return err
|
||||
}
|
||||
return stream.Send(&volume_server_pb.VolumeCopyResponse{LastAppendAtNs: uint64(time.Now().UnixNano())})
|
||||
@@ -414,44 +368,3 @@ func (v *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serve
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeTailReceiverResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) copyRemoteFile(ctx context.Context, sourceDataNode string, volumeID uint32, ext string, fileSize uint64, dialOption grpc.DialOption) error {
|
||||
path := v.filePath(volumeID, ext)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return operation.WithVolumeServerClient(true, pb.ServerAddress(sourceDataNode), dialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
stream, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
|
||||
VolumeId: volumeID,
|
||||
Ext: ext,
|
||||
StopOffset: fileSize,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if recvErr != nil {
|
||||
return recvErr
|
||||
}
|
||||
if len(resp.FileContent) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := file.Write(resp.FileContent); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -16,8 +16,6 @@ import (
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const testVolumeDatSize = 1 * 1024 * 1024
|
||||
|
||||
func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
volumeID := uint32(303)
|
||||
|
||||
@@ -33,7 +31,6 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
|
||||
source := pluginworkers.NewVolumeServer(t, "")
|
||||
target := pluginworkers.NewVolumeServer(t, "")
|
||||
pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), volumeID, testVolumeDatSize)
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: fmt.Sprintf("balance-job-%d", volumeID),
|
||||
@@ -87,9 +84,6 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
|
||||
|
||||
// Build a batch job with 3 volume moves from source → target.
|
||||
volumeIDs := []uint32{401, 402, 403}
|
||||
for _, vid := range volumeIDs {
|
||||
pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), vid, testVolumeDatSize)
|
||||
}
|
||||
moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
|
||||
for i, vid := range volumeIDs {
|
||||
moves[i] = &worker_pb.BalanceMoveSpec{
|
||||
@@ -145,11 +139,10 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
|
||||
require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
|
||||
}
|
||||
|
||||
// Each move reads source status once before copy and once inside the
|
||||
// target's fake VolumeCopy implementation, then reads target status once
|
||||
// before deleting the source.
|
||||
require.Equal(t, len(volumeIDs)*2, source.ReadFileStatusCount(),
|
||||
"each move should read source volume status before copy and during target copy")
|
||||
// Pre-delete verification should have called ReadVolumeFileStatus on both
|
||||
// source and target for each volume.
|
||||
require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
|
||||
"each move should read source volume status before delete")
|
||||
require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
|
||||
"each move should read target volume status before delete")
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
@@ -22,12 +21,7 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
|
||||
datPath := filepath.Join(baseDir, volumeFilename(volumeID, ".dat"))
|
||||
idxPath := filepath.Join(baseDir, volumeFilename(volumeID, ".idx"))
|
||||
|
||||
// The idx entry records the needle's data size. The actual on-disk size
|
||||
// includes header + checksum + timestamp (GetActualSize). The .dat must
|
||||
// be large enough to hold the full needle.
|
||||
needleDataSize := types.Size(datSize)
|
||||
actualSize := needle.GetActualSize(needleDataSize, needle.Version3)
|
||||
data := make([]byte, actualSize)
|
||||
data := make([]byte, datSize)
|
||||
rng := rand.New(rand.NewSource(99))
|
||||
_, _ = rng.Read(data)
|
||||
if err := os.WriteFile(datPath, data, 0644); err != nil {
|
||||
@@ -41,7 +35,7 @@ func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize
|
||||
|
||||
types.NeedleIdToBytes(entry[:idEnd], types.NeedleId(1))
|
||||
types.OffsetToBytes(entry[idEnd:offsetEnd], types.ToOffset(0))
|
||||
types.SizeToBytes(entry[offsetEnd:sizeEnd], needleDataSize)
|
||||
types.SizeToBytes(entry[offsetEnd:sizeEnd], types.Size(datSize))
|
||||
|
||||
if err := os.WriteFile(idxPath, entry, 0644); err != nil {
|
||||
t.Fatalf("write idx file: %v", err)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -12,8 +13,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
type TestEnvironment struct {
|
||||
@@ -97,26 +96,29 @@ func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
bindIP := findBindIP()
|
||||
|
||||
// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
|
||||
ports := testutil.MustAllocatePorts(t, 9)
|
||||
masterPort, masterGrpcPort := mustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := mustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := mustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := mustFreePortPair(t, "S3")
|
||||
icebergPort := mustFreePort(t, "Iceberg")
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
masterPort: ports[0],
|
||||
masterGrpcPort: ports[1],
|
||||
volumePort: ports[2],
|
||||
volumeGrpcPort: ports[3],
|
||||
filerPort: ports[4],
|
||||
filerGrpcPort: ports[5],
|
||||
s3Port: ports[6],
|
||||
s3GrpcPort: ports[7],
|
||||
icebergPort: ports[8],
|
||||
dockerAvailable: testutil.HasDocker(),
|
||||
s3Port: s3Port,
|
||||
s3GrpcPort: s3GrpcPort,
|
||||
icebergPort: icebergPort,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
filerPort: filerPort,
|
||||
filerGrpcPort: filerGrpcPort,
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
dockerAvailable: hasDocker(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -331,6 +333,73 @@ func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
|
||||
}
|
||||
}
|
||||
|
||||
func hasDocker() bool {
|
||||
cmd := exec.Command("docker", "version")
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
func mustFreePort(t *testing.T, name string) int {
|
||||
t.Helper()
|
||||
|
||||
port, err := getFreePort()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port for %s: %v", name, err)
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
func mustFreePortPair(t *testing.T, name string) (int, int) {
|
||||
t.Helper()
|
||||
|
||||
httpPort, grpcPort, err := findAvailablePortPair()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get free port pair for %s: %v", name, err)
|
||||
}
|
||||
return httpPort, grpcPort
|
||||
}
|
||||
|
||||
func findAvailablePortPair() (int, int, error) {
|
||||
httpPort, err := getFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
grpcPort, err := getFreePort()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return httpPort, grpcPort, nil
|
||||
}
|
||||
|
||||
func getFreePort() (int, error) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, nil
|
||||
}
|
||||
|
||||
func findBindIP() string {
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return "127.0.0.1"
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
ipNet, ok := addr.(*net.IPNet)
|
||||
if !ok || ipNet.IP == nil {
|
||||
continue
|
||||
}
|
||||
ip := ipNet.IP.To4()
|
||||
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
return ip.String()
|
||||
}
|
||||
return "127.0.0.1"
|
||||
}
|
||||
|
||||
func randomString(length int) string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
b := make([]byte, length)
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
test-volume-data/
|
||||
weed-test.log
|
||||
weed-server.pid
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user