mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-09 16:15:51 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
91b834bd9d | ||
|
|
2f84fdfc8d | ||
|
|
c820280996 | ||
|
|
2033ec28d7 | ||
|
|
e3ef3a5b56 | ||
|
|
0a4ec5ba15 |
@@ -4,10 +4,6 @@ on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -22,7 +18,6 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
|
||||
@@ -9,96 +9,17 @@ permissions:
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -109,40 +30,40 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v7
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
name: "docker: build latest container"
|
||||
|
||||
# Manual fallback only. On tag push, container_release_unified.yml already
|
||||
# re-tags the released versioned image as `latest` / `latest_large_disk`,
|
||||
# so a full rebuild here is unnecessary. Run this manually if you need to
|
||||
# rebuild `latest` from an arbitrary ref.
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
@@ -24,22 +23,15 @@ on:
|
||||
- all
|
||||
- standard
|
||||
- large_disk
|
||||
publish:
|
||||
description: 'Publish images and manifests'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
variants: ${{ steps.set-variants.outputs.variants }}
|
||||
publish: ${{ steps.set-publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Select variants for this run
|
||||
id: set-variants
|
||||
@@ -50,89 +42,9 @@ jobs:
|
||||
variants='["standard","large_disk"]'
|
||||
fi
|
||||
echo "variants=$variants" >> "$GITHUB_OUTPUT"
|
||||
- name: Select publish mode
|
||||
id: set-publish
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
needs: [setup]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -167,32 +79,11 @@ jobs:
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
@@ -220,14 +111,14 @@ jobs:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -238,7 +129,7 @@ jobs:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ needs.setup.outputs.publish == 'true' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
@@ -258,151 +149,10 @@ jobs:
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Build local scan image tarball
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64
|
||||
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
# since they all use the same alpine base, so a single-arch scan
|
||||
# provides sufficient coverage without multiplying CI time.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
- name: Trivy gate (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, trivy-scan]
|
||||
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
|
||||
needs: [setup, build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
@@ -430,12 +180,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -4,19 +4,11 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_foundationdb:
|
||||
@@ -34,7 +26,7 @@ jobs:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
|
||||
type=ref,event=tag,suffix=_large_disk_foundationdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
@@ -50,7 +42,7 @@ jobs:
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -29,7 +29,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
@@ -40,82 +39,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
|
||||
# emulated cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
# ── Build Docker containers ─────────────────────────────────────────
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# Build sequentially to avoid rate limits
|
||||
@@ -128,23 +52,20 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk - multi-arch
|
||||
|
||||
# Large disk - multi-arch
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# Full tags - multi-arch
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- variant: large_disk_full
|
||||
@@ -152,42 +73,19 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
rust_variant: large-disk
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -237,14 +135,14 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -300,14 +198,14 @@ jobs:
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -358,128 +256,7 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
|
||||
echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
|
||||
# we sometimes need to publish through known findings (e.g. unfixed upstream
|
||||
# CVE, base-image lag).
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
variant: normal
|
||||
- source_suffix: _large_disk
|
||||
variant: large_disk
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Trivy report (${{ matrix.variant }})
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan the multi-arch tag on GHCR (already pushed by the build job).
|
||||
# Trivy scans the runner's native platform; OS packages are identical
|
||||
# across architectures since they all share the same alpine base.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
# Point `latest` (and `latest_large_disk`) at the just-released versioned
|
||||
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
|
||||
# no QEMU, no separate workflow. Replaces the old container_latest.yml
|
||||
# rebuild that often failed or lagged behind the release. Independent of
|
||||
# trivy-scan: vuln findings are reported but do not block `latest`.
|
||||
tag-latest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
latest_tag: latest
|
||||
- source_suffix: _large_disk
|
||||
latest_tag: latest_large_disk
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
|
||||
run: |
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
|
||||
DST_TAG="${{ matrix.latest_tag }}"
|
||||
|
||||
echo "Tagging ghcr.io/chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "ghcr.io/chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -88,13 +88,13 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
|
||||
|
||||
@@ -57,12 +57,12 @@ jobs:
|
||||
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
# Enable Docker buildkit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
|
||||
# Build with retry logic
|
||||
for i in {1..3}; do
|
||||
echo "Build attempt $i/3"
|
||||
@@ -77,18 +77,10 @@ jobs:
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Start services with wait
|
||||
docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Rotate buildx cache
|
||||
if: always()
|
||||
run: |
|
||||
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
|
||||
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
|
||||
rm -rf /tmp/.buildx-cache
|
||||
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
|
||||
|
||||
- name: Run FIO 4k
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
@@ -1,69 +0,0 @@
|
||||
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run P2P integration tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-p2p-test-logs
|
||||
path: /tmp/seaweedfs-fuse-p2p-logs/
|
||||
retention-days: 3
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v5
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
@@ -68,135 +68,6 @@ jobs:
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "✓ Security configuration renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing IAM gRPC opt-in path ==="
|
||||
# Regression test: the filer registers the IAM gRPC service the
|
||||
# Admin UI Users tab calls only when jwt.filer_signing.key is in
|
||||
# security.toml. Operators must be able to enable that without
|
||||
# the cert-manager mTLS bundle.
|
||||
# Install PyYAML explicitly: this block runs before the later
|
||||
# security+S3 block that does the same install, and we don't
|
||||
# want to rely on the runner image shipping it.
|
||||
pip install pyyaml -q
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import subprocess, sys, yaml
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def docs(manifest):
|
||||
return [d for d in yaml.safe_load_all(manifest) if d]
|
||||
|
||||
def configmap(manifest, name):
|
||||
for d in docs(manifest):
|
||||
if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
|
||||
return d
|
||||
return None
|
||||
|
||||
def workload_mounts(manifest, name):
|
||||
for d in docs(manifest):
|
||||
if d.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
if d["metadata"]["name"] != name:
|
||||
continue
|
||||
pod = d["spec"]["template"]["spec"]
|
||||
vols = {v["name"] for v in pod.get("volumes", [])}
|
||||
mounts = set()
|
||||
for c in pod.get("containers", []):
|
||||
for vm in c.get("volumeMounts", []):
|
||||
mounts.add(vm["name"])
|
||||
return vols, mounts
|
||||
return None, None
|
||||
|
||||
failed = []
|
||||
|
||||
# Case 1: defaults. The chart historically rendered nothing
|
||||
# security-related; preserve that so this PR is non-breaking on
|
||||
# existing installs.
|
||||
out = render({})
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append("defaults: security ConfigMap should not render")
|
||||
else:
|
||||
print("✓ defaults: no security-config ConfigMap (unchanged)")
|
||||
|
||||
# Case 2: filerWrite=true alone is the documented opt-in for
|
||||
# the Admin UI Users tab. Configmap must render with
|
||||
# [jwt.filer_signing] and NO [grpc.*] sections (cert paths
|
||||
# only exist with mTLS).
|
||||
out = render({
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true",
|
||||
"admin.enabled": "true",
|
||||
})
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("filerWrite=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["data"]["security.toml"]
|
||||
if "[jwt.filer_signing]" not in toml:
|
||||
failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
|
||||
if "[grpc" in toml:
|
||||
failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
|
||||
if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
|
||||
print("✓ filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
|
||||
|
||||
# Case 3: filer + admin pods must MOUNT the security ConfigMap
|
||||
# under filerWrite=true so the JWT key reaches both processes.
|
||||
# Cert volumes must NOT be present (no mTLS).
|
||||
for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
|
||||
vols, mounts = workload_mounts(out, wl)
|
||||
if vols is None:
|
||||
failed.append(f"filerWrite=true: workload {wl} not found")
|
||||
continue
|
||||
if "security-config" not in vols or "security-config" not in mounts:
|
||||
failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
|
||||
else:
|
||||
print(f"✓ filerWrite=true: {wl} mounts security-config")
|
||||
cert_vols = {v for v in vols if v.endswith("-cert")}
|
||||
if cert_vols:
|
||||
failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")
|
||||
|
||||
# Case 4: enableSecurity=true must still render the full toml
|
||||
# with both [jwt.signing] and [grpc.*]. Guards against the
|
||||
# decoupling change accidentally regressing the mTLS path.
|
||||
out = render({"global.seaweedfs.enableSecurity": "true"})
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("enableSecurity=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["data"]["security.toml"]
|
||||
missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
|
||||
if missing:
|
||||
failed.append(f"enableSecurity=true: security.toml missing {missing}")
|
||||
else:
|
||||
print("✓ enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
|
||||
|
||||
# Case 5: helper must tolerate explicit nulls (gemini-code-assist
|
||||
# PR review). securityConfig=null was the parens-pattern crash
|
||||
# the helper review caught.
|
||||
for null_path in ("global.seaweedfs.securityConfig",
|
||||
"global.seaweedfs.securityConfig.jwtSigning"):
|
||||
try:
|
||||
out = render({null_path: "null"})
|
||||
except subprocess.CalledProcessError as e:
|
||||
failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
|
||||
continue
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append(f"{null_path}=null: should not render configmap")
|
||||
else:
|
||||
print(f"✓ {null_path}=null: render tolerates explicit null")
|
||||
|
||||
if failed:
|
||||
print("\nFAIL:", file=sys.stderr)
|
||||
for f in failed:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
echo "✓ IAM gRPC decoupling tests passed"
|
||||
|
||||
echo "=== Testing with monitoring enabled ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.monitoring.enabled=true \
|
||||
@@ -346,106 +217,6 @@ jobs:
|
||||
print("✓ No blank lines in security+S3 command blocks")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
|
||||
# Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
|
||||
# the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
|
||||
# them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
|
||||
# makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
|
||||
# HTTP request to an HTTPS server" into the pod log.
|
||||
#
|
||||
# When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
|
||||
# together so the opt-in HTTPS listener actually has credentials.
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import subprocess, sys, yaml
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def script_of(manifest, kind_name):
|
||||
for doc in yaml.safe_load_all(manifest):
|
||||
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
if doc["metadata"]["name"] != kind_name:
|
||||
continue
|
||||
for c in doc["spec"]["template"]["spec"]["containers"]:
|
||||
cmd = c.get("command", [])
|
||||
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
|
||||
return cmd[2]
|
||||
raise AssertionError(f"no container script for {kind_name}")
|
||||
|
||||
cases = [
|
||||
# (values, workload-name, httpsPort-set?, arg-prefix)
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true"},
|
||||
"test-seaweedfs-s3", False, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true",
|
||||
"s3.httpsPort": "8443"},
|
||||
"test-seaweedfs-s3", True, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true"},
|
||||
"test-seaweedfs-filer", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true",
|
||||
"filer.s3.httpsPort": "8444"},
|
||||
"test-seaweedfs-filer", True, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true"},
|
||||
"test-seaweedfs-all-in-one", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true",
|
||||
"allInOne.s3.httpsPort": "8445"},
|
||||
"test-seaweedfs-all-in-one", True, "s3."),
|
||||
]
|
||||
|
||||
failed = False
|
||||
for values, name, https_on, prefix in cases:
|
||||
script = script_of(render(values), name)
|
||||
cert_flag = f"-{prefix}cert.file="
|
||||
key_flag = f"-{prefix}key.file="
|
||||
https_flag = f"-{prefix}port.https="
|
||||
has_cert = cert_flag in script
|
||||
has_key = key_flag in script
|
||||
has_https = https_flag in script
|
||||
label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
|
||||
if https_on:
|
||||
if not (has_cert and has_key and has_https):
|
||||
print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: cert/key/https args emitted together")
|
||||
else:
|
||||
if has_cert or has_key or has_https:
|
||||
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
|
||||
f"main S3 -port would silently become HTTPS and break HTTP probes "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
|
||||
|
||||
# bash -n: pin down that the rendered script parses. Guards against
|
||||
# a future helper change that leaves a dangling `\` with nothing
|
||||
# after it (every current caller already exits cleanly because
|
||||
# bash treats trailing `\<newline><EOF>` as line-continuation to
|
||||
# an empty line — but keep the contract explicit).
|
||||
parse = subprocess.run(["bash", "-n"], input=script, text=True,
|
||||
capture_output=True)
|
||||
if parse.returncode != 0:
|
||||
print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
|
||||
file=sys.stderr)
|
||||
failed = True
|
||||
|
||||
sys.exit(1 if failed else 0)
|
||||
PYEOF
|
||||
|
||||
echo "✅ All template rendering tests passed!"
|
||||
|
||||
- name: Create kind cluster
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
name: "helm: release"
|
||||
name: "helm: manual release"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -445,7 +445,7 @@ jobs:
|
||||
# Test consumer group functionality with explicit timeout
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
name: "Multi-Master Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/multi-master-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run multi-master failover tests
|
||||
# The tests in test/multi_master spin up their own 3-node master raft
|
||||
# cluster (using the freshly-installed `weed` binary) and exercise
|
||||
# leader-election, failover and recovery scenarios. The shared
|
||||
# test/testutil port-allocator regression test runs alongside since it
|
||||
# is a prerequisite for the cluster fixtures.
|
||||
run: |
|
||||
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/multi_master/cluster.go creates per-test dirs via
|
||||
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
|
||||
# node's log into <baseDir>/logs/master*.log.
|
||||
echo "Collecting per-node master logs from temp directories..."
|
||||
mkdir -p /tmp/multi-master-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: multi-master-test-logs
|
||||
path: /tmp/multi-master-logs/
|
||||
retention-days: 7
|
||||
@@ -1,137 +0,0 @@
|
||||
name: "NFS Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/nfs-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
nfs-integration:
|
||||
name: NFS Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run NFS Integration Tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running NFS integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run the protocol-layer tests. The kernel-mount tests require root
|
||||
# for mount(2) and are exercised in their own privileged step below;
|
||||
# skip them here so a "skipped because not root" line doesn't show
|
||||
# up as noise on every CI run.
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "NFS integration tests completed"
|
||||
|
||||
- name: Install kernel NFS client
|
||||
run: |
|
||||
# nfs-common provides mount.nfs; netbase provides /etc/protocols
|
||||
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nfs-common netbase
|
||||
|
||||
- name: Run kernel-mount E2E tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running kernel-mount end-to-end tests..."
|
||||
echo "These mount the running 'weed nfs' subprocess via the actual"
|
||||
echo "Linux NFS client to catch protocol regressions invisible to"
|
||||
echo "the go-nfs-client-based tests above."
|
||||
echo "============================================"
|
||||
|
||||
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
|
||||
# binary that test/nfs/framework.go locates via $PATH) resolve
|
||||
# correctly under sudo, and pass through the Go module/cache dirs
|
||||
# so we don't redownload modules under root.
|
||||
sudo env "PATH=$PATH" \
|
||||
GOMODCACHE="$(go env GOMODCACHE)" \
|
||||
GOCACHE="$(go env GOCACHE)" \
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Kernel-mount E2E tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
|
||||
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
|
||||
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
|
||||
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
|
||||
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1,118 +0,0 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-e2e
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build pjdfstest image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/pjdfstest
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-harness
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-harness
|
||||
|
||||
- name: Tag pjdfstest image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -80,7 +80,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
@@ -156,7 +156,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -169,7 +169,7 @@ jobs:
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ jobs:
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@@ -48,7 +47,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -84,7 +83,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
@@ -119,7 +118,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -155,7 +154,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
|
||||
@@ -39,19 +39,11 @@ jobs:
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -88,8 +80,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
@@ -97,15 +88,6 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -130,7 +112,7 @@ jobs:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -165,8 +147,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
@@ -174,15 +155,6 @@ jobs:
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -198,7 +170,7 @@ jobs:
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
@@ -234,20 +206,10 @@ jobs:
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
name: "S3 ETag and ACL Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-etag-acl
|
||||
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
|
||||
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-etag-acl \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
# Wait for the S3 endpoint to come up (returns 403 unauth before any
|
||||
# request is signed; that's fine — it means the server is listening).
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run ETag tests
|
||||
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
|
||||
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
|
||||
# SDK for Java v2 rejects the latter on the PutObject path.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/etag/...
|
||||
|
||||
- name: Run ACL versioning tests
|
||||
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
|
||||
# PutObjectAcl with and without versionId, modifying ACLs on different
|
||||
# versions independently.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/acl/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-etag-acl-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -245,132 +245,6 @@ jobs:
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# One job per test so each gets a fresh `weed mini` server, avoiding
|
||||
# the cross-test volume-pool exhaustion that surfaced when several
|
||||
# TTL-pinned bucket collections piled up in a single run.
|
||||
test:
|
||||
- TestLifecycleAbortIncompleteMultipartUpload
|
||||
- TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
|
||||
- TestLifecycleBootstrapWalkOnExistingObjects
|
||||
- TestLifecycleConfigUpdateBetweenSweeps
|
||||
- TestLifecycleDeleteBucketLifecycleStopsDispatching
|
||||
- TestLifecycleDisabledRuleSkipsObject
|
||||
- TestLifecycleEmptyBucketSweepIsNoOp
|
||||
- TestLifecycleExpirationDateInThePast
|
||||
- TestLifecycleExpirationFiresOnBackdatedObject
|
||||
- TestLifecycleExpiredDeleteMarkerCleanup
|
||||
- TestLifecycleMultipleBucketsInOneSweep
|
||||
- TestLifecycleMultipleRulesInOneBucket
|
||||
- TestLifecycleNewerNoncurrentVersions
|
||||
- TestLifecycleNoncurrentVersionExpiration
|
||||
- TestLifecycleSizeFilterGreaterThan
|
||||
- TestLifecycleSkipsObjectLockedObjects
|
||||
- TestLifecycleSuspendedVersioningExpiration
|
||||
- TestLifecycleTagFilter
|
||||
- TestLifecycleVersionedBucketCreatesDeleteMarker
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run ${{ matrix.test }}
|
||||
timeout-minutes: 8
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Last 200 lines of server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-lifecycle-test-logs-${{ matrix.test }}
|
||||
path: test/s3/lifecycle/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Checksum Tests
|
||||
timeout-minutes: 16
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-checksum-test-logs
|
||||
path: test/s3/checksum/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -138,23 +138,6 @@ jobs:
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
"sts")
|
||||
echo "Running STS and service account tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
# SigV4-signed STS calls need admin credentials matching test_config.json.
|
||||
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
|
||||
export STS_TEST_ACCESS_KEY=test-access-key
|
||||
export STS_TEST_SECRET_KEY=test-secret-key
|
||||
# The use_service_account_credentials subtest is excluded because
|
||||
# newly-created service-account access keys are not currently
|
||||
# persisted to the filer after CreateServiceAccount — a
|
||||
# pre-existing sync issue tracked separately from the
|
||||
# GetFederationToken routing fix this PR addresses.
|
||||
go test -v -timeout 15m \
|
||||
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
|
||||
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
|
||||
./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
name: "S3 Mutation Regression Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/delete/**'
|
||||
- 'test/s3/distributed_lock/**'
|
||||
- 'test/s3/versioning/**'
|
||||
- 'test/volume_server/framework/**'
|
||||
- 'docker/compose/s3.json'
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-mutation-regression-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload versioning logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-regression-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload delete logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-delete-regression-logs
|
||||
path: test/s3/delete/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run distributed lock regressions
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
||||
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
||||
|
||||
- name: Upload distributed lock logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-distributed-lock-regression-logs
|
||||
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
||||
retention-days: 3
|
||||
@@ -71,10 +71,8 @@ jobs:
|
||||
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
|
||||
S3_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
# Check logs first for the readiness line. weed mini's progress
|
||||
# board prints " S3 ready (Xs)"; older builds and the
|
||||
# standalone S3 binary log "S3 (gateway|service) ... ready".
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
|
||||
# Check logs first for startup message (weed mini says "S3 service is ready")
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)"; then
|
||||
echo "SeaweedFS S3 gateway is ready"
|
||||
S3_READY=1
|
||||
break
|
||||
|
||||
@@ -73,12 +73,8 @@ jobs:
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
else
|
||||
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
|
||||
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
|
||||
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
|
||||
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
|
||||
# same change that fixed them.
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
|
||||
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
fi
|
||||
|
||||
- name: Show server logs on failure
|
||||
|
||||
@@ -2,6 +2,10 @@ name: "S3 Tables Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-tables-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -190,148 +194,6 @@ jobs:
|
||||
path: test/s3tables/catalog_trino/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
dremio-iceberg-catalog-tests:
|
||||
name: Dremio Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Dremio image
|
||||
run: docker pull dremio/dremio-oss:25.2.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Dremio Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Dremio Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Dremio Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|dremio)" || true
|
||||
echo "=== Dremio containers ==="
|
||||
docker ps -a --filter "name=seaweed-dremio" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dremio-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_dremio/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
doris-iceberg-catalog-tests:
|
||||
name: Doris Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Doris image
|
||||
run: docker pull apache/doris:doris-all-in-one-2.1.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Doris Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 30
|
||||
working-directory: test/s3tables/catalog_doris
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Doris Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
|
||||
echo "Doris Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_doris
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|doris)" || true
|
||||
echo "=== Doris containers ==="
|
||||
docker ps -a --filter "name=seaweed-doris" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: doris-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_doris/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -662,77 +524,6 @@ jobs:
|
||||
path: test/s3tables/lakekeeper/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
unity-catalog-integration-tests:
|
||||
name: Unity Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Unity Catalog image
|
||||
run: docker pull unitycatalog/unitycatalog:v0.4.0
|
||||
|
||||
- name: Pre-pull Python image for delta-rs writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Unity Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Unity Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Unity Catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
|
||||
echo "=== Unity Catalog containers ==="
|
||||
docker ps -a --filter "name=seaweed-unity-catalog" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: unity-catalog-integration-test-logs
|
||||
path: test/s3tables/unity_catalog/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -37,8 +37,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -132,49 +130,7 @@ jobs:
|
||||
done
|
||||
|
||||
echo "✅ S3 server is responding, starting tests..."
|
||||
|
||||
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
|
||||
# something driving deletions. The s3tests build tag rescales one
|
||||
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
|
||||
# the upload's mtime; -dispatch / -checkpoint defaults are already
|
||||
# tightened under the same build tag.
|
||||
LC_LOG=/tmp/lifecycle-worker.log
|
||||
# -debug routes glog to stderr so the bootstrap walker's progress
|
||||
# shows up in $LC_LOG; without it weed shell silences glog.
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9333 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
# Aliveness check: a bad shell command exits in <1s and the suite
|
||||
# would otherwise just timeout the expiration tests with no signal.
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
# bash -e exits the step on the first tox failure, so move teardown
|
||||
# into a trap to guarantee the worker log + data dir reach the runner.
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -354,8 +310,11 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
@@ -380,8 +339,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -550,8 +507,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -775,8 +730,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -997,39 +950,6 @@ jobs:
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Spawn the lifecycle worker (see basic-tests block for context).
|
||||
LC_LOG=/tmp/lifecycle-worker-sql.log
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9337 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -1209,7 +1129,10 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "TLS Rotation Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run TLS Rotation Integration Tests
|
||||
working-directory: test/tls_rotation
|
||||
run: |
|
||||
go test -v -count=1 -timeout 5m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting master logs from temp directories..."
|
||||
mkdir -p /tmp/tls-rotation-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tls-rotation-test-logs
|
||||
path: /tmp/tls-rotation-test-logs/
|
||||
retention-days: 14
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "Vacuum Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run Vacuum Integration Tests
|
||||
working-directory: test/vacuum
|
||||
run: |
|
||||
go test -v -timeout 10m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/vacuum-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vacuum-integration-test-logs
|
||||
path: /tmp/vacuum-test-logs/
|
||||
retention-days: 14
|
||||
@@ -2,7 +2,6 @@
|
||||
vendor
|
||||
tags
|
||||
*.swp
|
||||
.claude/
|
||||
### OSX template
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"setup": [],
|
||||
"teardown": [],
|
||||
"run": []
|
||||
}
|
||||
@@ -42,7 +42,8 @@ test: admin-generate
|
||||
|
||||
# Admin component targets
|
||||
admin-generate:
|
||||
@cd $(ADMIN_DIR) && $(MAKE) generate
|
||||
@echo "Generating admin component templates..."
|
||||
@cd $(ADMIN_DIR) && templ generate ./view
|
||||
|
||||
admin-build: admin-generate
|
||||
@echo "Building admin component..."
|
||||
|
||||
@@ -56,6 +56,7 @@ Table of Contents
|
||||
* [Quick Start](#quick-start)
|
||||
* [Quick Start with weed mini](#quick-start-with-weed-mini)
|
||||
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
|
||||
* [Quick Start with Single Binary](#quick-start-with-single-binary)
|
||||
* [Introduction](#introduction)
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
@@ -79,41 +80,39 @@ Table of Contents
|
||||
|
||||
|
||||
## Quick Start with weed mini ##
|
||||
The easiest way to get started with SeaweedFS for development and testing:
|
||||
|
||||
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file, or run `go install github.com/seaweedfs/seaweedfs/weed@latest`. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
AWS_ACCESS_KEY_ID=admin \
|
||||
AWS_SECRET_ACCESS_KEY=secret \
|
||||
S3_BUCKET=my-bucket \
|
||||
# remove quarantine on macOS
|
||||
# xattr -d com.apple.quarantine ./weed
|
||||
|
||||
./weed mini -dir=/data
|
||||
```
|
||||
|
||||
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
|
||||
|
||||
The same command starts everything else too:
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
This single command starts a complete SeaweedFS setup with:
|
||||
- **Master UI**: http://localhost:9333
|
||||
- **Volume Server**: http://localhost:9340
|
||||
- **Filer UI**: http://localhost:8888
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
- **WebDAV**: http://localhost:7333
|
||||
- **Admin UI**: http://localhost:23646
|
||||
|
||||
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
|
||||
|
||||
Perfect for development, testing, learning SeaweedFS, and single-node deployments. To scale out, add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, on another machine, or on thousands of machines.
|
||||
Perfect for development, testing, learning SeaweedFS, and single node deployments!
|
||||
|
||||
## Quick Start for S3 API on Docker ##
|
||||
|
||||
```bash
|
||||
docker run -p 8333:8333 \
|
||||
-e AWS_ACCESS_KEY_ID=admin \
|
||||
-e AWS_SECRET_ACCESS_KEY=secret \
|
||||
-e S3_BUCKET=my-bucket \
|
||||
chrislusf/seaweedfs
|
||||
```
|
||||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
|
||||
Same behavior as the `weed mini` command above — the S3 endpoint is at http://localhost:8333 with `my-bucket` pre-created. Drop the env vars to run anonymously for development.
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
|
||||
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
# Introduction #
|
||||
|
||||
@@ -146,11 +145,6 @@ SeaweedFS can achieve both fast local access time and elastic cloud storage capa
|
||||
What's more, the cloud storage access API cost is minimized.
|
||||
Faster and cheaper than direct cloud storage!
|
||||
|
||||
SeaweedFS also ships a built-in **Iceberg REST Catalog**, turning the same cluster into a self-contained lakehouse.
|
||||
Spark, Trino, Dremio, DuckDB, and RisingWave can query Iceberg tables directly — no Hive Metastore, Glue, or
|
||||
external catalog service required. Storage and table metadata live in one system, simplifying on-prem and
|
||||
small-team analytics stacks.
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
# Features #
|
||||
@@ -187,13 +181,6 @@ small-team analytics stacks.
|
||||
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
|
||||
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
|
||||
|
||||
## Data Lakehouse Features ##
|
||||
* [S3 Table Buckets][S3TableBucket] expose a dedicated namespace for Iceberg tables with strict layout validation.
|
||||
* Built-in [Iceberg REST Catalog][IcebergCatalog] runs alongside the S3 endpoint — no external metastore needed.
|
||||
* Native integrations with [Apache Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], and [RisingWave][RisingWaveIceberg].
|
||||
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan removal, manifest rewriting.
|
||||
* Granular IAM at the bucket, namespace, and table level via standard S3 bucket policies.
|
||||
|
||||
## Kubernetes ##
|
||||
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
|
||||
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
|
||||
@@ -217,14 +204,6 @@ small-team analytics stacks.
|
||||
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
|
||||
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
|
||||
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
|
||||
[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket
|
||||
[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog
|
||||
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
|
||||
[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration
|
||||
[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration
|
||||
[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration
|
||||
[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration
|
||||
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
|
||||
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
-24
@@ -1,24 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you find a security issue in SeaweedFS, please report it privately:
|
||||
|
||||
- Email: support@seaweedfs.com
|
||||
- Do not open a public GitHub issue
|
||||
|
||||
Please include:
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce (if possible)
|
||||
- Affected versions
|
||||
|
||||
## Response
|
||||
|
||||
- We will respond as soon as possible (usually within 1 business day)
|
||||
- We will investigate and work on a fix
|
||||
- We may coordinate disclosure with you
|
||||
|
||||
## Notes
|
||||
|
||||
- Please allow time for a fix before public disclosure
|
||||
- If you’re unsure whether something is a security issue, feel free to reach out
|
||||
@@ -2,15 +2,13 @@ FROM ubuntu:22.04
|
||||
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
|
||||
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
|
||||
# Use faster mirrors and optimize package installation
|
||||
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
|
||||
apt-get -o Acquire::http::Timeout=15 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
|
||||
@@ -16,22 +16,15 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
# Rust volume server: use pre-built binary from CI when available (placed in
|
||||
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
|
||||
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
|
||||
# for non-native architectures.
|
||||
FROM alpine:3.23 as rust_builder
|
||||
# Rust volume server builder (amd64/arm64 only)
|
||||
FROM rust:1-alpine as rust_builder
|
||||
ARG TARGETARCH
|
||||
ARG TAGS
|
||||
COPY weed-volume-prebuilt/ /prebuilt/
|
||||
RUN apk add musl-dev protobuf-dev git
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/proto /build/proto
|
||||
WORKDIR /build/seaweed-volume
|
||||
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
|
||||
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
|
||||
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
|
||||
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
|
||||
ARG TAGS
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
if [ "$TAGS" = "5BytesOffset" ]; then \
|
||||
cargo build --release; \
|
||||
else \
|
||||
@@ -56,8 +49,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@@ -89,8 +81,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -7,8 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@@ -40,8 +39,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -34,8 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@@ -68,8 +67,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -17,8 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "steward",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "steward-key",
|
||||
"secretKey": "steward-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "le001",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "le001-key",
|
||||
"secretKey": "le001-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+2
-15
@@ -37,21 +37,17 @@ if [ "$(id -u)" = "0" ]; then
|
||||
fi
|
||||
|
||||
isArgPassed() {
|
||||
# Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9
|
||||
# library accepts both, and users may pick either form on the CLI.
|
||||
arg="$1"
|
||||
argWithEqualSign="$1="
|
||||
argDouble="-$1"
|
||||
argDoubleWithEqualSign="-$1="
|
||||
shift
|
||||
while [ $# -gt 0 ]; do
|
||||
passedArg="$1"
|
||||
shift
|
||||
case $passedArg in
|
||||
"$arg"|"$argDouble")
|
||||
"$arg")
|
||||
return 0
|
||||
;;
|
||||
"$argWithEqualSign"*|"$argDoubleWithEqualSign"*)
|
||||
"$argWithEqualSign"*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
@@ -99,15 +95,6 @@ case "$1" in
|
||||
exec /usr/bin/weed -logtostderr=true server $ARGS $@
|
||||
;;
|
||||
|
||||
'mini')
|
||||
ARGS="-dir=/data"
|
||||
if isArgPassed "-dir" "$@"; then
|
||||
ARGS=""
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true mini $ARGS $@
|
||||
;;
|
||||
|
||||
'filer')
|
||||
ARGS=""
|
||||
shift
|
||||
|
||||
@@ -4,8 +4,8 @@ go 1.25.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.2
|
||||
cloud.google.com/go/storage v1.62.1
|
||||
cloud.google.com/go/pubsub v1.50.1
|
||||
cloud.google.com/go/storage v1.60.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -34,20 +34,20 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/jackc/pgx/v5 v5.8.0
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/klauspost/reedsolomon v1.14.0
|
||||
github.com/klauspost/compress v1.18.5
|
||||
github.com/klauspost/reedsolomon v1.13.3
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.7
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
@@ -61,14 +61,14 @@ require (
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/procfs v0.20.1
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.8
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/seaweedfs/raft v1.1.7
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/viper v1.21.0
|
||||
@@ -83,82 +83,81 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/viant/ptrie v1.0.1
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.etcd.io/etcd/client/v3 v3.6.7
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.50.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/image v0.39.0
|
||||
golang.org/x/net v0.53.0
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
|
||||
golang.org/x/image v0.36.0
|
||||
golang.org/x/net v0.51.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.43.0
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/tools v0.44.0 // indirect
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.278.0
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/api v0.267.0
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.49.1
|
||||
modernc.org/sqlite v1.46.1
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.31.0
|
||||
cloud.google.com/go/kms v1.25.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.1001
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
|
||||
github.com/a-h/templ v0.3.977
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
|
||||
github.com/apache/iceberg-go v0.5.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.6
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.9.0
|
||||
github.com/getsentry/sentry-go v0.43.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.28.0
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.11.0
|
||||
github.com/rclone/rclone v1.74.1
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.73.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.3
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2
|
||||
github.com/shirou/gopsutil/v4 v4.26.2
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.4
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
@@ -171,19 +170,18 @@ require (
|
||||
atomicgo.dev/cursor v0.2.0 // indirect
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.9.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.8.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.23.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -191,7 +189,8 @@ require (
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
@@ -212,6 +211,7 @@ require (
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
@@ -224,7 +224,7 @@ require (
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -257,10 +257,9 @@ require (
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
@@ -285,62 +284,62 @@ require (
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.35.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth v0.18.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.7.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
filippo.io/edwards25519 v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.10.3 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -348,13 +347,13 @@ require (
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
@@ -368,27 +367,27 @@ require (
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/errors v0.22.4 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.28.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
@@ -405,7 +404,7 @@ require (
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
|
||||
github.com/jonboulle/clockwork v0.5.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
@@ -419,9 +418,9 @@ require (
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.22 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
@@ -436,8 +435,8 @@ require (
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.5 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
@@ -453,7 +452,7 @@ require (
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/rfjakob/eme v1.2.0 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
@@ -466,11 +465,11 @@ require (
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.6.3 // indirect
|
||||
github.com/tinylib/msgp v1.5.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
@@ -479,7 +478,7 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@@ -487,40 +486,35 @@ require (
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.42.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/libc v1.68.0 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
storj.io/picobuf v0.0.4 // indirect
|
||||
storj.io/uplink v1.14.0 // indirect
|
||||
storj.io/uplink v1.13.1 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
// apache/thrift v0.23.0 uses math.MaxUint32 as an untyped int constant in
|
||||
// lib/go/thrift/framed_transport.go, which overflows int on 32-bit GOARCHes
|
||||
// (e.g. openbsd/arm, linux/arm). Pin to v0.22.0 until upstream fixes it.
|
||||
replace github.com/apache/thrift => github.com/apache/thrift v0.22.0
|
||||
|
||||
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
|
||||
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
|
||||
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA=
|
||||
cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q=
|
||||
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
||||
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
|
||||
@@ -283,8 +283,8 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE
|
||||
cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
|
||||
cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY=
|
||||
cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
|
||||
cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U=
|
||||
cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY=
|
||||
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
|
||||
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
|
||||
cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc=
|
||||
cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A=
|
||||
cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk=
|
||||
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
|
||||
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
|
||||
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
|
||||
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
|
||||
cloud.google.com/go/kms v1.31.0 h1:LS8N92OxFDgOLg5NCo3OmbvjtQAIVT5gUHVLKIDHaFE=
|
||||
cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U=
|
||||
cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
|
||||
cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
|
||||
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
|
||||
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
|
||||
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
|
||||
@@ -310,13 +310,13 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA=
|
||||
cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY=
|
||||
cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E=
|
||||
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
|
||||
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
|
||||
cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE=
|
||||
cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM=
|
||||
cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA=
|
||||
@@ -391,10 +391,10 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp
|
||||
cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI=
|
||||
cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0=
|
||||
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
|
||||
cloud.google.com/go/pubsub v1.50.2 h1:54Up97HnThdP4H8jjWJSSQ/mnYG2EKon7ZSNETRq0tM=
|
||||
cloud.google.com/go/pubsub v1.50.2/go.mod h1:jyCWeZdGFqd4mitSsBERnJcpqaHBsxQoPkNvjj4sp0w=
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 h1:oMKNiBQpXImRWnHYla9uSU66ZzByZwBSCJOEs/pTKVg=
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0/go.mod h1:2lS/XQKq5qtOMs6kHBK+WX1ytUC36kLl2ig3zqsGUx8=
|
||||
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
|
||||
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
|
||||
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
|
||||
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
|
||||
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
|
||||
@@ -485,8 +485,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq
|
||||
cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
|
||||
cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
|
||||
cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
|
||||
cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8=
|
||||
cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA=
|
||||
cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
|
||||
cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
|
||||
cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
|
||||
cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I=
|
||||
cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4=
|
||||
@@ -569,16 +569,16 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiIS75iTVAKIUxrST5X2dVHMTptL4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M=
|
||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A=
|
||||
github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
|
||||
@@ -593,20 +593,20 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 h1:2RfP0d2QgkFH64BjZSWd59aMsc28IyWsUuHqU0txBtY=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82/go.mod h1:IPk80dOmc7VFC0DJ85xMTPmre+8xoXX6kGHAkf5jRRw=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 h1:W8S9TrAyZ4//3PXsU6+Bi+fe/6uIL986GyS7PVzIDL4=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 h1:lMHTplAYI9FtmCo/QOcpRxmPA5REVAct1r2riQmDQKw=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264/go.mod h1:wGqkOzRu/ClJibvDgcfuJNAqI2nLhe8g91tPlDKRCdE=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 h1:mJ5QbLPOm4g5qhZiVB6wbSllfpeUExftGoyPek2hk4M=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2/go.mod h1:ngpMgwkjur1VNUjqn11LPk3o5eCyOCRbcfg/0YAY7Hc=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 h1:DUnYhvC4SoC8T84rx5omnhY3+xcQg/Whyoa3mDPIMkk=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0/go.mod h1:Q3BYO6iDA2zweQPDGbNTtqft5tDcEpm6RTuqMlPcvbw=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1/go.mod h1:HbOjEdu/yzCJ/mjTF6wEOJNbAUpHfU2UOA2hVD4CNFg=
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
|
||||
@@ -633,16 +633,16 @@ github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk=
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
|
||||
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
|
||||
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
|
||||
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
|
||||
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
|
||||
github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI=
|
||||
github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s=
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
|
||||
github.com/Sereal/Sereal/Go/sereal v0.0.0-20231009093132-b9187f1a92c6/go.mod h1:JwrycNnC8+sZPDyzM3MQ86LvaGzSpfxg885KOOwFRW4=
|
||||
github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
|
||||
github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
|
||||
@@ -652,18 +652,16 @@ github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfih
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
|
||||
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb h1:wumPkzt4zaxO4rHPBrjDK8iZMR41C1qs7njNqlacwQg=
|
||||
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb/go.mod h1:QiYsIBRQEO+Z4Rz7GoI+dsHVneZNONvhczuA+llOZNM=
|
||||
github.com/a-h/templ v0.3.1001 h1:yHDTgexACdJttyiyamcTHXr2QkIeVF1MukLy44EAhMY=
|
||||
github.com/a-h/templ v0.3.1001/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
|
||||
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
|
||||
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 h1:iLDOF0rdGTrol/q8OfPIIs5kLD8XvA2q75o6Uq/tgak=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0/go.mod h1:DrEWcQJjz7t5iF2duaiyhg4jyoF0kxOD6LtECNGkZ/Q=
|
||||
github.com/aalpar/deheap v1.1.2 h1:MABHLcnjqsffb8GLkUFDigqpBBxOMz0DoKM9QfELeTw=
|
||||
github.com/aalpar/deheap v1.1.2/go.mod h1:A+nfkD4JbS05sewV0he/MYgR/90vfqyMoNNROgs+rmA=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 h1:hhdWprfSpFbN7lz3W1gM40vOgvSh1WCSMxYD6gGB4Hs=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3/go.mod h1:XaUnRxSCYgL3kkgX0QHIV0D+znljPIDImxlv2kbGv0Y=
|
||||
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
|
||||
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
|
||||
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
|
||||
github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
|
||||
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
|
||||
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
|
||||
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
|
||||
@@ -694,10 +692,11 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 h1:r/gg2gzUXiXoy72VU3jnODh8l/5rL0aslnTAbtmai/U=
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87/go.mod h1:IJTMBTlHe7cDOhRh0ioGuEKBl5iTR6xPfl5BN4AgirU=
|
||||
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq/RrkjCcxCnqUxhwf8=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0 h1:Omnzb1Z/P90Dr2TbVNu54ICQL7TKVIIsJO231w484HU=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
|
||||
github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
|
||||
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
|
||||
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
|
||||
@@ -715,50 +714,50 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -828,13 +827,15 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 h1:iLoIwb7BJECHTbNcmIhYDsQhoZiACWGNvEpyqQy97Dk=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 h1:ugiQwb7DwpWQnete2AZkTh94MonZKmxD7hDGy1qTzDs=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=
|
||||
@@ -904,9 +905,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
|
||||
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
|
||||
github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
|
||||
github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=
|
||||
github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 h1:tLWBZgPg6TV67oe76W4p+aUQEWIa52wbcuiz8GFd3vo=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
|
||||
github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
|
||||
@@ -1047,12 +1047,12 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
|
||||
github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
|
||||
github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
@@ -1071,13 +1071,13 @@ github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3
|
||||
github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
|
||||
github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA=
|
||||
github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw=
|
||||
github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM=
|
||||
github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
@@ -1096,8 +1096,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
|
||||
github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM=
|
||||
github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
|
||||
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
|
||||
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
|
||||
@@ -1116,8 +1116,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
|
||||
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
|
||||
github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
|
||||
github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
|
||||
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
|
||||
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
|
||||
@@ -1126,8 +1126,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
|
||||
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
|
||||
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
|
||||
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
|
||||
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
@@ -1136,8 +1136,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
|
||||
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
|
||||
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
|
||||
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
|
||||
@@ -1383,8 +1383,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
|
||||
github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
|
||||
github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
|
||||
github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
|
||||
github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
@@ -1397,14 +1397,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 h1:PV6ipOJN0JIek4dqPqsTtenQmjI1SZntCUgPzhfk9gM=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160/go.mod h1:yRuPDnHgGmsbvopF0amMqXxr4n32GynzX5GTwFYDHaw=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c h1:r+KtxPyrhsYeNbsfeqTfEM8xRdwgV6LuNhLZxpXecb4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
|
||||
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jaegertracing/jaeger v1.47.0 h1:XXxTMO+GxX930gxKWsg90rFr6RswkCRIW0AgWFnTYsg=
|
||||
@@ -1429,8 +1429,8 @@ github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
|
||||
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb h1:6vkM8gO+zFV2m21QzGYyUSq5TP0VQgP2Xz3UQyCN2kI=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb/go.mod h1:H1+whwD0Qe3YOunlXIWhh3rlvzW5cZfkMDYGQPg+KAM=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
@@ -1475,15 +1475,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
|
||||
github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
|
||||
github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
|
||||
@@ -1523,8 +1523,8 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
|
||||
github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
|
||||
github.com/lpar/date v1.0.0/go.mod h1:KjYe0dDyMQTgpqcUz4LEIeM5VZwhggjVx/V2dtc8NSo=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
|
||||
@@ -1544,8 +1544,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk=
|
||||
github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
|
||||
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
@@ -1657,12 +1657,12 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
|
||||
github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 h1:eDkWg6ZN0uKwWzSekoFcQJhR+C+F/aVdTwr+lGHU9Qk=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0/go.mod h1:8ZzvzuEG/cFLFZhxg/Mg1w19KqyXBKO3c17QIc5PkGs=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 h1:l9awEvzWvxmYhy/97A0hZ87pa7BncYXmcO/S8+rvgK0=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0/go.mod h1:oB8jFGVc/7/zJ+DbleE8MzGHjhs2ioCz5stRTdZdIcY=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
||||
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
|
||||
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
|
||||
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
|
||||
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
|
||||
github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
|
||||
@@ -1755,8 +1755,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8=
|
||||
github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
@@ -1779,18 +1779,16 @@ github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
|
||||
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
|
||||
github.com/rclone/go-proton-api v1.0.2/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.74.1 h1:QQI//tl4O5uooTowdKwNKcv5qCQvJ3LIBG00+TKeXw0=
|
||||
github.com/rclone/rclone v1.74.1/go.mod h1:44epaKf5w/1ZQounefr76wx4+y9NZmgfMKle9T3pzj0=
|
||||
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.73.1 h1:FCGZUd7sXO8+nIpbuXGPkd+f23o/ZACefrHtRGDAiDM=
|
||||
github.com/rclone/rclone v1.73.1/go.mod h1:T9HeSXv/SI9qJFkYLqsjY/5ljIFWJTVmTrG7H/4JrPY=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
@@ -1809,8 +1807,8 @@ github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=
|
||||
github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
@@ -1840,12 +1838,14 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3 h1:rJufGrHImTx7yoGUmetUi+To4LrmTQJROJnBHWt92ic=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/seaweedfs/raft v1.1.8 h1:a5f+wSnriRjINLqLqMY7DDNNfU1+kv6TKBuOZa0KDwU=
|
||||
github.com/seaweedfs/raft v1.1.8/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/seaweedfs/raft v1.1.7 h1:3mVJZ2p4rdvBtbbrHROPjYKtH+q5qjMBc56G6VRu1kA=
|
||||
github.com/seaweedfs/raft v1.1.7/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
@@ -1856,15 +1856,15 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
|
||||
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
|
||||
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
|
||||
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
|
||||
@@ -1887,8 +1887,8 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
@@ -1930,8 +1930,8 @@ github.com/substrait-io/substrait-protobuf/go v0.81.0 h1:/qC1XYKuO4oPdTwLYySuVZ6
|
||||
github.com/substrait-io/substrait-protobuf/go v0.81.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c h1:dtcOwRimeiBFrlutmF6K94l0rxYFARNFMA+lSQ41C+M=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 h1:rrGZv6xYk37hx0tW2sYfgbO0PqStbHqz6Bq6oc9Hurg=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/tarantool/go-iproto v1.1.0 h1:HULVOIHsiehI+FnHfM7wMDntuzUddO09DKqu2WnFQ5A=
|
||||
github.com/tarantool/go-iproto v1.1.0/go.mod h1:LNCtdyZxojUed8SbOiYHoc3v9NvaZTB7p96hUySMlIo=
|
||||
@@ -1961,8 +1961,8 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GR
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA=
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g=
|
||||
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
|
||||
github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s=
|
||||
github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tinylib/msgp v1.5.0 h1:GWnqAE54wmnlFazjq2+vgr736Akg58iiHImh+kPY2pc=
|
||||
github.com/tinylib/msgp v1.5.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
@@ -2028,10 +2028,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
|
||||
github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
@@ -2042,8 +2038,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@@ -2064,14 +2060,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 h1:avIdi8eGXjKbn1WLokNR1Ofnz1k8t7tJ88YQLD/iCi8=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2 h1:Bzra5hUQIWrxknQ0oV6HE5q/eZU1VhHMKSZ/XWpqMgM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2/go.mod h1:VYUUkRJkKuQPkIpgtZJj6+58Fa2g8ccAqdmaaK6HP5k=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3 h1:hTwpF+PdbuR7vcixN+4AC6yu4asaUIAljQCxL51eDII=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
|
||||
@@ -2098,23 +2094,21 @@ github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
|
||||
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
|
||||
github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
|
||||
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/mwc v0.0.7 h1:0NerGhCww6ZQx+/xCx5iwznftveokvto1KILpYfENZk=
|
||||
github.com/zeebo/mwc v0.0.7/go.mod h1:0B32or6moOig1YGuqMoimBpU9QK9uYaGG2bBOuddqtE=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
|
||||
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
|
||||
go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
|
||||
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
@@ -2130,14 +2124,14 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 h1:0tY123n7CdWMem7MOVdKOt0YfshufLCwfE5Bob+hQuM=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
|
||||
@@ -2148,18 +2142,18 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4D
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
@@ -2216,8 +2210,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -2233,8 +2227,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
@@ -2248,8 +2242,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
|
||||
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2283,8 +2277,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
|
||||
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
|
||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2352,8 +2346,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -2511,11 +2505,11 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4=
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2531,8 +2525,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
|
||||
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -2553,15 +2547,15 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -2632,8 +2626,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
|
||||
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -2711,8 +2705,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E=
|
||||
google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ=
|
||||
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
||||
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -2846,12 +2840,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -2892,8 +2886,8 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
|
||||
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
|
||||
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
|
||||
google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
|
||||
@@ -2983,19 +2977,19 @@ modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
|
||||
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=
|
||||
modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8=
|
||||
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
|
||||
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
|
||||
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
|
||||
modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
|
||||
modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU=
|
||||
modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0=
|
||||
modernc.org/ccgo/v4 v4.30.2 h1:4yPaaq9dXYXZ2V8s1UgrC3KIj580l2N4ClrLwnbv2so=
|
||||
modernc.org/ccgo/v4 v4.30.2/go.mod h1:yZMnhWEdW0qw3EtCndG1+ldRrVGS+bIwyWmAWzS0XEw=
|
||||
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
|
||||
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
|
||||
@@ -3010,8 +3004,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
|
||||
modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
|
||||
modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
|
||||
modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
|
||||
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
|
||||
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
|
||||
modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ=
|
||||
modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0=
|
||||
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
@@ -3030,8 +3024,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
|
||||
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
|
||||
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU=
|
||||
modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
@@ -3054,8 +3048,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aN
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
|
||||
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
|
||||
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a h1:7gSBQY3vhQMIqi3vfaEXR7mreRjcBLfVsYY0rHIN7P0=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a/go.mod h1:kyxwKwlfH4paBZCZt/szvRB770ieAIJF73iDy2DpEHw=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c h1:UDXSrdeLJe3QFouavSW10fYdpclK0YNu3KvQHzqq2+k=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c/go.mod h1:XNX7uykja6aco92y2y8RuqaXIDRPpt1YA2OQDKlKEUk=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 h1:8OE12DvUnB9lfZcHe7IDGsuhjrY9GBAr964PVHmhsro=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 h1:5MZ0CyMbG6Pi0rRzUWVG6dvpXjbBYEX2oyXuj+tT+sk=
|
||||
@@ -3064,7 +3058,7 @@ storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q=
|
||||
storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs=
|
||||
storj.io/picobuf v0.0.4 h1:qswHDla+YZ2TovGtMnU4astjvrADSIz84FXRn0qgP6o=
|
||||
storj.io/picobuf v0.0.4/go.mod h1:hSMxmZc58MS/2qSLy1I0idovlO7+6K47wIGUyRZa6mg=
|
||||
storj.io/uplink v1.14.0 h1:J1yXlt0aRr6kgLTHWXOWosNCFVfbamlcyd+CSxyIczo=
|
||||
storj.io/uplink v1.14.0/go.mod h1:2ysmjzd/1Xtz4VKoErNcSqBQz3UC9WKTVuLMV1cNu6E=
|
||||
storj.io/uplink v1.13.1 h1:C8RdW/upALoCyuF16Lod9XGCXEdbJAS+ABQy9JO/0pA=
|
||||
storj.io/uplink v1.13.1/go.mod h1:x0MQr4UfFsQBwgVWZAtEsLpuwAn6dg7G0Mpne1r516E=
|
||||
tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc=
|
||||
tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.26"
|
||||
appVersion: "4.17"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.26.0
|
||||
version: 4.17.0
|
||||
|
||||
@@ -49,35 +49,6 @@ CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
#### RocksDB variant
|
||||
|
||||
The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
|
||||
To use this image with the Helm chart, override the image on all three components and disable
|
||||
the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
|
||||
override the image's built-in RocksDB configuration:
|
||||
|
||||
```yaml
|
||||
# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
|
||||
master:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
volume:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
filer:
|
||||
enablePVC: true
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
extraEnvironmentVars:
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
* `master` and `volume` use the same image tag so that all components share a consistent
|
||||
SeaweedFS build; RocksDB itself is only used by the filer.
|
||||
* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
|
||||
so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.admin.image" . }}
|
||||
image: {{ template "admin.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
{{- $adminAuthEnabled := or .Values.admin.secret.existingSecret .Values.admin.secret.adminPassword }}
|
||||
{{- $urlPrefix := .Values.admin.urlPrefix }}
|
||||
@@ -135,14 +135,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- $secretExtraEnvironmentVars := .Values.admin.secretExtraEnvironmentVars }}
|
||||
{{- if $secretExtraEnvironmentVars }}
|
||||
{{- range $key := keys $secretExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $secretExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
valueFrom: {{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -188,13 +180,11 @@ spec:
|
||||
- name: admin-logs
|
||||
mountPath: /logs
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -252,7 +242,7 @@ spec:
|
||||
securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.admin.data.type "hostPath" }}
|
||||
@@ -285,12 +275,10 @@ spec:
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.admin.logs.claimName }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
@@ -315,7 +303,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.admin.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.admin.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "admin.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.admin.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
{{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
|
||||
@@ -247,8 +247,8 @@ spec:
|
||||
{{- $httpsPort := .Values.allInOne.s3.httpsPort | default .Values.s3.httpsPort }}
|
||||
{{- if $httpsPort }}
|
||||
-s3.port.https={{ $httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
|
||||
@@ -330,13 +330,11 @@ spec:
|
||||
mountPath: /etc/seaweedfs/master.toml
|
||||
subPath: master.toml
|
||||
readOnly: true
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
readOnly: true
|
||||
@@ -420,7 +418,7 @@ spec:
|
||||
{{- omit .Values.allInOne.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: data
|
||||
@@ -463,12 +461,10 @@ spec:
|
||||
- name: master-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master-config
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -117,13 +117,11 @@ spec:
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -166,7 +164,7 @@ spec:
|
||||
securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.cosi.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: socket
|
||||
@@ -181,12 +179,10 @@ spec:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-s3-secret
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.filer.image" . }}
|
||||
image: {{ template "filer.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -200,8 +200,8 @@ spec:
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- if .Values.filer.s3.httpsPort }}
|
||||
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/seaweedfs_s3_config \
|
||||
@@ -234,13 +234,11 @@ spec:
|
||||
mountPath: /etc/seaweedfs/notification.toml
|
||||
subPath: notification.toml
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -276,8 +274,7 @@ spec:
|
||||
name: swfs-s3-tls
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $jwt := (.Values.global.seaweedfs.securityConfig).jwtSigning | default dict }}
|
||||
{{- $isJwtEnabled := or $jwt.filerWrite $jwt.filerRead }}
|
||||
{{- $isJwtEnabled := or .Values.global.seaweedfs.securityConfig.jwtSigning.filerWrite .Values.global.seaweedfs.securityConfig.jwtSigning.filerRead }}
|
||||
{{- if .Values.filer.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
{{- if or $isJwtEnabled .Values.filer.readinessProbe.tcpSocket }}
|
||||
@@ -320,7 +317,7 @@ spec:
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
@@ -371,12 +368,10 @@ spec:
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-notification-config
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
@@ -418,7 +413,7 @@ spec:
|
||||
storageClassName: {{ .Values.filer.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.filer.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "filer.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -80,7 +80,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -188,13 +188,11 @@ spec:
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/master.toml
|
||||
subPath: master.toml
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -253,7 +251,7 @@ spec:
|
||||
securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
@@ -289,12 +287,10 @@ spec:
|
||||
- name: master-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master-config
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
@@ -316,7 +312,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.master.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "master.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.master.data.type "persistentVolumeClaim"}}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.s3.image" . }}
|
||||
image: {{ template "s3.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -127,8 +127,8 @@ spec:
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
-port.https={{ .Values.s3.httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.domainName }}
|
||||
-domainName={{ .Values.s3.domainName }} \
|
||||
@@ -156,13 +156,11 @@ spec:
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -228,7 +226,7 @@ spec:
|
||||
securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.s3.enableAuth }}
|
||||
@@ -251,12 +249,10 @@ spec:
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
{{- $access_key_admin := $adminCreds.accessKey -}}
|
||||
{{- $secret_key_admin := $adminCreds.secretKey -}}
|
||||
{{- if not (and $access_key_admin $secret_key_admin) -}}
|
||||
{{- $access_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
{{- $readCreds := $creds.read | default dict -}}
|
||||
{{- $access_key_read := $readCreds.accessKey -}}
|
||||
{{- $secret_key_read := $readCreds.secretKey -}}
|
||||
{{- if not (and $access_key_read $secret_key_read) -}}
|
||||
{{- $access_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- /*
|
||||
The seaweedfs-s3 Service only gains a "metrics" port when the standalone S3
|
||||
gateway is enabled. With only the embedded filer S3 gateway, metrics live on
|
||||
the filer process and are already scraped by the filer ServiceMonitor.
|
||||
*/ -}}
|
||||
{{- if and .Values.s3.enabled .Values.s3.metricsPort .Values.global.seaweedfs.monitoring.enabled }}
|
||||
{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
|
||||
{{- if .Values.s3.metricsPort }}
|
||||
{{- if .Values.global.seaweedfs.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
@@ -34,3 +31,5 @@ spec:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.sftp.image" . }}
|
||||
image: {{ template "sftp.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -176,13 +176,11 @@ spec:
|
||||
- mountPath: /etc/sw/ssh
|
||||
name: config-ssh
|
||||
readOnly: true
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -235,7 +233,7 @@ spec:
|
||||
securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.sftp.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.sftp.enableAuth }}
|
||||
@@ -266,12 +264,10 @@ spec:
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{{- if or .Values.sftp.enabled .Values.allInOne.enabled }}
|
||||
{{- $secretName := printf "%s-sftp-secret" (include "seaweedfs.fullname" .) }}
|
||||
{{- $admin_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" "length" 20) -}}
|
||||
{{- $read_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" "length" 20) -}}
|
||||
{{- $public_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" "length" 20) -}}
|
||||
{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" 20) -}}
|
||||
{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" 20) -}}
|
||||
{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" 20) -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
type: Opaque
|
||||
|
||||
@@ -72,77 +72,77 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper filer image */}}
|
||||
{{- define "seaweedfs.filer.image" -}}
|
||||
{{- define "filer.image" -}}
|
||||
{{- if .Values.filer.imageOverride -}}
|
||||
{{- $imageOverride := .Values.filer.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper master image */}}
|
||||
{{- define "seaweedfs.master.image" -}}
|
||||
{{- define "master.image" -}}
|
||||
{{- if .Values.master.imageOverride -}}
|
||||
{{- $imageOverride := .Values.master.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper s3 image */}}
|
||||
{{- define "seaweedfs.s3.image" -}}
|
||||
{{- define "s3.image" -}}
|
||||
{{- if .Values.s3.imageOverride -}}
|
||||
{{- $imageOverride := .Values.s3.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper sftp image */}}
|
||||
{{- define "seaweedfs.sftp.image" -}}
|
||||
{{- define "sftp.image" -}}
|
||||
{{- if .Values.sftp.imageOverride -}}
|
||||
{{- $imageOverride := .Values.sftp.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper admin image */}}
|
||||
{{- define "seaweedfs.admin.image" -}}
|
||||
{{- define "admin.image" -}}
|
||||
{{- if .Values.admin.imageOverride -}}
|
||||
{{- $imageOverride := .Values.admin.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper worker image */}}
|
||||
{{- define "seaweedfs.worker.image" -}}
|
||||
{{- define "worker.image" -}}
|
||||
{{- if .Values.worker.imageOverride -}}
|
||||
{{- $imageOverride := .Values.worker.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper volume image */}}
|
||||
{{- define "seaweedfs.volume.image" -}}
|
||||
{{- define "volume.image" -}}
|
||||
{{- if .Values.volume.imageOverride -}}
|
||||
{{- $imageOverride := .Values.volume.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Computes the container image name for all components (if they are not overridden) */}}
|
||||
{{- define "seaweedfs.image" -}}
|
||||
{{- define "common.image" -}}
|
||||
{{- $registryName := default .Values.image.registry .Values.global.imageRegistry | toString -}}
|
||||
{{- $repositoryName := default .Values.image.repository .Values.global.seaweedfs.image.repository | toString -}}
|
||||
{{- $name := .Values.global.seaweedfs.image.name | toString -}}
|
||||
@@ -160,7 +160,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Volume PVC exists */}}
|
||||
{{- define "seaweedfs.volume.pvc_exists" -}}
|
||||
{{- define "volume.pvc_exists" -}}
|
||||
{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -169,7 +169,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Filer PVC exists */}}
|
||||
{{- define "seaweedfs.filer.pvc_exists" -}}
|
||||
{{- define "filer.pvc_exists" -}}
|
||||
{{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -178,7 +178,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Master PVC exists */}}
|
||||
{{- define "seaweedfs.master.pvc_exists" -}}
|
||||
{{- define "master.pvc_exists" -}}
|
||||
{{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -187,7 +187,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Admin PVC exists */}}
|
||||
{{- define "seaweedfs.admin.pvc_exists" -}}
|
||||
{{- define "admin.pvc_exists" -}}
|
||||
{{- if or (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -196,7 +196,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any InitContainers exist for Volumes */}}
|
||||
{{- define "seaweedfs.volume.initContainers_exists" -}}
|
||||
{{- define "volume.initContainers_exists" -}}
|
||||
{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -225,10 +225,10 @@ imagePullSecrets:
|
||||
{{/*
|
||||
Renders a value that contains template perhaps with scope if the scope is present.
|
||||
Usage:
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.tplvalues.render" -}}
|
||||
{{- define "common.tplvalues.render" -}}
|
||||
{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
|
||||
{{- if contains "{{" (toJson .value) }}
|
||||
{{- if .scope }}
|
||||
@@ -245,9 +245,9 @@ Usage:
|
||||
Converts a Kubernetes quantity like "256Mi" or "2G" to a float64 in base units,
|
||||
handling both binary (Ki, Mi, Gi) and decimal (m, k, M) suffixes; numeric inputs
|
||||
Usage:
|
||||
{{ include "seaweedfs.resource-quantity" "10Gi" }}
|
||||
{{ include "common.resource-quantity" "10Gi" }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.resource-quantity" -}}
|
||||
{{- define "common.resource-quantity" -}}
|
||||
{{- $value := . -}}
|
||||
{{- $unit := 1.0 -}}
|
||||
{{- if typeIs "string" . -}}
|
||||
@@ -267,7 +267,7 @@ Usage:
|
||||
getOrGeneratePassword will check if a password exists in a secret and return it,
|
||||
or generate a new random password if it doesn't exist.
|
||||
*/}}
|
||||
{{- define "seaweedfs.getOrGeneratePassword" -}}
|
||||
{{- define "getOrGeneratePassword" -}}
|
||||
{{- $params := . -}}
|
||||
{{- $namespace := $params.namespace -}}
|
||||
{{- $secretName := $params.secretName -}}
|
||||
@@ -332,16 +332,6 @@ Create the name of the service account to use
|
||||
{{- .Values.global.seaweedfs.serviceAccountName | default "seaweedfs" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* True when security.toml should be rendered and mounted. volumeWrite is
|
||||
excluded since it defaults to true. */}}
|
||||
{{- define "seaweedfs.securityConfigEnabled" -}}
|
||||
{{- $sec := (.Values.global.seaweedfs).securityConfig | default dict -}}
|
||||
{{- $jwt := $sec.jwtSigning | default dict -}}
|
||||
{{- if or .Values.global.seaweedfs.enableSecurity $jwt.volumeRead $jwt.filerWrite $jwt.filerRead -}}
|
||||
true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}}
|
||||
{{- define "seaweedfs.s3.tlsArgs" -}}
|
||||
{{- $prefix := .prefix -}}
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
{{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
|
||||
containers:
|
||||
- name: post-install-job
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
env:
|
||||
- name: WEED_CLUSTER_DEFAULT
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
@@ -55,7 +55,6 @@ data:
|
||||
key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
# all grpc tls authentications are mutual
|
||||
# the values for the following ca, cert, and key are paths to the PERM files.
|
||||
[grpc]
|
||||
@@ -95,5 +94,4 @@ data:
|
||||
[https.volume]
|
||||
cert = ""
|
||||
key = ""
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -35,8 +35,8 @@
|
||||
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
|
||||
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
|
||||
{{- if and $currentPVC }}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
|
||||
{{- $oldSize := include "common.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "common.resource-quantity" $desiredSize }}
|
||||
{{- if gt $newSize $oldSize }}
|
||||
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
|
||||
{{- end }}
|
||||
|
||||
@@ -71,12 +71,12 @@ spec:
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
|
||||
{{- $initContainers_exists := include "seaweedfs.volume.initContainers_exists" $ -}}
|
||||
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
|
||||
{{- if $initContainers_exists }}
|
||||
initContainers:
|
||||
{{- if $volume.idx }}
|
||||
- name: seaweedfs-vol-move-idx
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
command: [ '/bin/sh', '-c' ]
|
||||
args: [ '{{range $dir := $volume.dataDirs }}if ls /{{$dir.name}}/*.idx >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
|
||||
@@ -104,7 +104,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" $.Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_NAME
|
||||
@@ -211,13 +211,11 @@ spec:
|
||||
- name: idx
|
||||
mountPath: "/idx/"
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" $ }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -276,7 +274,7 @@ spec:
|
||||
securityContext: {{- omit $volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if $volume.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
|
||||
@@ -335,12 +333,10 @@ spec:
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" $ }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" $ }}-security-config
|
||||
{{- end }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" $ }}-ca-cert
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.worker.image" . }}
|
||||
image: {{ template "worker.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -149,13 +149,11 @@ spec:
|
||||
- name: worker-logs
|
||||
mountPath: /logs
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -221,7 +219,7 @@ spec:
|
||||
securityContext: {{- omit .Values.worker.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.worker.data.type "hostPath" }}
|
||||
@@ -254,12 +252,10 @@ spec:
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.worker.logs.claimName }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -18,8 +18,6 @@ global:
|
||||
loggingLevel: 1
|
||||
enableSecurity: false
|
||||
masterServer: null
|
||||
# filerWrite: true mounts security.toml on filer + admin without needing
|
||||
# enableSecurity (mTLS); required for the Admin UI Users tab.
|
||||
securityConfig:
|
||||
jwtSigning:
|
||||
volumeWrite: true
|
||||
@@ -1288,13 +1286,6 @@ admin:
|
||||
|
||||
extraEnvironmentVars: {}
|
||||
|
||||
# secret env variables (e.g. for injecting OIDC client secret from a Kubernetes Secret)
|
||||
secretExtraEnvironmentVars: {}
|
||||
# WEED_ADMIN_OIDC_CLIENT_SECRET:
|
||||
# secretKeyRef:
|
||||
# name: seaweedfs-admin-oidc
|
||||
# key: client_secret
|
||||
|
||||
# Health checks
|
||||
livenessProbe:
|
||||
enabled: true
|
||||
@@ -1348,14 +1339,9 @@ worker:
|
||||
# Admin server to connect to
|
||||
adminServer: ""
|
||||
|
||||
# Worker job types - categories or comma-separated list of names/aliases
|
||||
# Categories: all, default, heavy
|
||||
# default: vacuum, volume_balance, ec_balance, admin_script
|
||||
# heavy: erasure_coding, iceberg_maintenance
|
||||
# all: every registered job type
|
||||
# Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
|
||||
# Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
|
||||
jobType: "all"
|
||||
# Worker job types - comma-separated list
|
||||
# Available: vacuum, volume_balance, erasure_coding
|
||||
jobType: "vacuum,volume_balance,erasure_coding"
|
||||
|
||||
# Maximum number of concurrent detection requests
|
||||
maxDetect: 1
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
MARP ?= npx @marp-team/marp-cli
|
||||
SRC := seaweedfs-production-setup.md
|
||||
PDF := $(SRC:.md=.pdf)
|
||||
HTML := $(SRC:.md=.html)
|
||||
PPTX := $(SRC:.md=.pptx)
|
||||
FLAGS := --allow-local-files
|
||||
|
||||
.PHONY: all pdf html pptx watch clean
|
||||
|
||||
all: pdf
|
||||
|
||||
pdf: $(PDF)
|
||||
|
||||
html: $(HTML)
|
||||
|
||||
pptx: $(PPTX)
|
||||
|
||||
$(PDF): $(SRC)
|
||||
$(MARP) $(SRC) --pdf $(FLAGS) -o $@
|
||||
|
||||
$(HTML): $(SRC)
|
||||
$(MARP) $(SRC) --html $(FLAGS) -o $@
|
||||
|
||||
$(PPTX): $(SRC)
|
||||
$(MARP) $(SRC) --pptx $(FLAGS) -o $@
|
||||
|
||||
watch:
|
||||
$(MARP) $(SRC) --watch --html $(FLAGS)
|
||||
|
||||
clean:
|
||||
rm -f $(PDF) $(HTML) $(PPTX)
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 738 KiB |
@@ -1,118 +0,0 @@
|
||||
---
|
||||
marp: true
|
||||
theme: default
|
||||
paginate: true
|
||||
---
|
||||
|
||||
# SeaweedFS Production Setup
|
||||
|
||||
A walkthrough of the recommended deployment topology
|
||||
|
||||
- Storage layer
|
||||
- File access layer
|
||||
- Backend operations
|
||||
- Erasure coding for durability
|
||||
|
||||
---
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
# Storage Layer
|
||||
|
||||
**3 Masters + Volume Servers**
|
||||
|
||||
- **Masters (run 3)**
|
||||
- Form a Raft quorum; tolerate 1 failure
|
||||
- Track volume locations and cluster topology
|
||||
- Lightweight; not on the data path for reads/writes after lookup
|
||||
|
||||
- **Volume Servers (scale out)**
|
||||
- Hold the actual needle data on local disks
|
||||
- Add more servers for capacity and throughput
|
||||
- Heartbeat to masters; clients talk to them directly for I/O
|
||||
|
||||
Rule of thumb: start with 3 masters, then grow volume servers with data.
|
||||
|
||||
---
|
||||
|
||||
# File Access Layer
|
||||
|
||||
**Filer + DB for metadata**
|
||||
|
||||
- Filer serves POSIX-like directory and file metadata
|
||||
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
|
||||
- Run **3 filers** for HA
|
||||
- Filers are stateless relative to each other — state lives in the DB
|
||||
|
||||
**S3 Servers**
|
||||
|
||||
- Expose the S3 API on top of the filer
|
||||
- Scale the count based on throughput requirements
|
||||
- Can be co-located with filers or run independently
|
||||
|
||||
---
|
||||
|
||||
# Backend Operations
|
||||
|
||||
**1 Admin Server + a few Workers**
|
||||
|
||||
- **Admin server (1 is enough)**
|
||||
- Coordinates background jobs: balancing, EC encoding, vacuum, replication fixes
|
||||
- **Not on the data path** — a restart does not affect object store reads/writes
|
||||
- No HA required for normal operation
|
||||
|
||||
- **Workers (a few)**
|
||||
- Execute long-running tasks issued by the admin server
|
||||
- Horizontally scalable based on maintenance workload
|
||||
|
||||
Keeps housekeeping off the hot path.
|
||||
|
||||
---
|
||||
|
||||
# Production Topology Summary
|
||||
|
||||
| Component | Count | Role |
|
||||
|----------------|----------------------|-----------------------------------|
|
||||
| Master | 3 | Raft quorum, topology, volume map |
|
||||
| Volume Server | N (scale out) | Stores data needles |
|
||||
| Filer | 3 | Metadata gateway |
|
||||
| Metadata DB | HA cluster | Persists filer metadata |
|
||||
| S3 Server | N (by throughput) | S3 API frontend |
|
||||
| Admin | 1 | Background job coordinator |
|
||||
| Worker | A few | Executes admin-scheduled tasks |
|
||||
|
||||
---
|
||||
|
||||
# Erasure Coding for Durability
|
||||
|
||||
Prefer **many volume servers** so EC shards spread across failure domains.
|
||||
|
||||
**Example: 8 volume servers, EC 5+3**
|
||||
|
||||
- Each volume becomes 5 data shards + 3 parity shards
|
||||
- Shards are placed on 8 distinct volume servers
|
||||
- **Tolerates up to 3 simultaneous volume server failures**
|
||||
- Storage overhead: 1.6x vs 3x for full replication
|
||||
|
||||
```
|
||||
Volume -> [D1][D2][D3][D4][D5] + [P1][P2][P3]
|
||||
| | | | | | | |
|
||||
VS1 VS2 VS3 VS4 VS5 VS6 VS7 VS8
|
||||
```
|
||||
|
||||
More volume servers = more EC layouts available (e.g. 10+4, 6+3).
|
||||
|
||||
---
|
||||
|
||||
# Getting Started Checklist
|
||||
|
||||
1. Provision 3 master nodes (small boxes are fine)
|
||||
2. Provision volume servers sized to your data footprint
|
||||
3. Stand up an HA metadata DB, then 3 filers pointing at it
|
||||
4. Add S3 servers sized to required throughput
|
||||
5. Run 1 admin + a few workers for background maintenance
|
||||
6. Once you have enough volume servers, enable EC (e.g. 5+3 on 8 servers)
|
||||
|
||||
Scale each layer independently as usage grows.
|
||||
@@ -84,15 +84,6 @@ service SeaweedFiler {
|
||||
// distributed lock management internal use only
|
||||
rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
|
||||
}
|
||||
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
|
||||
}
|
||||
|
||||
// Peer chunk sharing — tier 1: mount-server registry.
|
||||
// See design-weed-mount-peer-chunk-sharing.md for details.
|
||||
rpc MountRegister (MountRegisterRequest) returns (MountRegisterResponse) {
|
||||
}
|
||||
rpc MountList (MountListRequest) returns (MountListResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
@@ -204,10 +195,6 @@ message FuseAttributes {
|
||||
bytes md5 = 14;
|
||||
uint32 rdev = 16;
|
||||
uint64 inode = 17;
|
||||
int64 ctime = 18; // unix time in seconds, inode change time
|
||||
int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
|
||||
int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
|
||||
int32 crtime_ns = 21; // nanosecond component of crtime (0-999999999)
|
||||
}
|
||||
|
||||
message CreateEntryRequest {
|
||||
@@ -305,7 +292,6 @@ message AssignVolumeRequest {
|
||||
string rack = 7;
|
||||
string data_node = 9;
|
||||
string disk_type = 8;
|
||||
uint64 expected_data_size = 10; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message AssignVolumeResponse {
|
||||
@@ -536,7 +522,6 @@ message LockResponse {
|
||||
string lock_owner = 2;
|
||||
string lock_host_moved_to = 3;
|
||||
string error = 4;
|
||||
int64 generation = 5;
|
||||
}
|
||||
message UnlockRequest {
|
||||
string name = 1;
|
||||
@@ -559,26 +544,12 @@ message Lock {
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_backup = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message TransferLocksRequest {
|
||||
repeated Lock locks = 1;
|
||||
}
|
||||
message TransferLocksResponse {
|
||||
}
|
||||
message ReplicateLockRequest {
|
||||
string name = 1;
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_unlock = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message ReplicateLockResponse {
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
|
||||
@@ -608,31 +579,3 @@ message StreamMutateEntryResponse {
|
||||
string error = 7; // human-readable error message when the operation failed
|
||||
int32 errno = 8; // POSIX errno (e.g. ENOENT=2, ENOTEMPTY=66) for direct FUSE status mapping
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Peer chunk sharing — mount-server registry
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
message MountRegisterRequest {
|
||||
string peer_addr = 1; // host:port where this mount serves peer chunk requests
|
||||
string rack = 2; // locality label (rack); used for peer ranking
|
||||
int32 ttl_seconds = 3; // how long the filer should keep this entry without a heartbeat
|
||||
string data_center = 4; // locality label (data center); coarser than rack
|
||||
}
|
||||
|
||||
message MountRegisterResponse {
|
||||
}
|
||||
|
||||
message MountListRequest {
|
||||
}
|
||||
|
||||
message MountListResponse {
|
||||
repeated MountInfo mounts = 1;
|
||||
}
|
||||
|
||||
message MountInfo {
|
||||
string peer_addr = 1;
|
||||
string rack = 2;
|
||||
int64 last_seen_ns = 3;
|
||||
string data_center = 4;
|
||||
}
|
||||
|
||||
Generated
+312
-152
File diff suppressed because it is too large
Load Diff
@@ -20,7 +20,7 @@ default = ["5bytes"]
|
||||
[dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
tokio-stream = "0.1"
|
||||
tokio-io-timeout = "1"
|
||||
|
||||
# gRPC + protobuf
|
||||
@@ -65,6 +65,8 @@ reed-solomon-erasure = "6"
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
pprof = { version = "0.15", features = ["prost-codec"] }
|
||||
|
||||
# Config
|
||||
toml = "0.8"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -82,7 +84,7 @@ memmap2 = "0.9"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
# HTTP client (for proxying, remote fetch)
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
reqwest = { version = "0.12", features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
|
||||
# Content hashing
|
||||
md-5 = "0.10"
|
||||
@@ -125,10 +127,6 @@ aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4
|
||||
aws-credential-types = "1"
|
||||
aws-types = "1"
|
||||
|
||||
# pprof is Unix-only (requires libc/nix APIs not available on Windows)
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
pprof = { version = "0.15", features = ["prost-codec"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
|
||||
@@ -145,11 +145,6 @@ message VolumeEcShardInformationMessage {
|
||||
uint64 expire_at_sec = 5; // used to record the destruction time of ec volume
|
||||
uint32 disk_id = 6;
|
||||
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
|
||||
uint64 file_count = 8; // total needles in the .ecx index (live + tombstoned)
|
||||
uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
|
||||
// fields 10-19 reserved for future upstream open-source additions.
|
||||
// fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
|
||||
// and must not be used here without coordination.
|
||||
}
|
||||
|
||||
message StorageBackend {
|
||||
@@ -236,7 +231,6 @@ message AssignRequest {
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
uint32 writable_volume_count = 9;
|
||||
string disk_type = 10;
|
||||
uint64 expected_data_size = 11; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message VolumeGrowRequest {
|
||||
|
||||
@@ -247,9 +247,6 @@ message VolumeUnmountResponse {
|
||||
message VolumeDeleteRequest {
|
||||
uint32 volume_id = 1;
|
||||
bool only_empty = 2;
|
||||
// when true, do not remove the cloud-tier object backing the volume.
|
||||
// used for moves where another server is taking over the same .vif.
|
||||
bool keep_remote_data = 3;
|
||||
}
|
||||
message VolumeDeleteResponse {
|
||||
}
|
||||
@@ -342,7 +339,6 @@ message ReceiveFileInfo {
|
||||
bool is_ec_volume = 4;
|
||||
uint32 shard_id = 5;
|
||||
uint64 file_size = 6;
|
||||
uint32 disk_id = 7; // EC shard disk; 0 = auto-select (see VolumeEcShardsCopyRequest.disk_id)
|
||||
}
|
||||
|
||||
message ReceiveFileResponse {
|
||||
@@ -458,7 +454,6 @@ message VolumeEcShardsMountRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
|
||||
}
|
||||
message VolumeEcShardsMountResponse {
|
||||
}
|
||||
@@ -641,7 +636,6 @@ message FetchAndWriteNeedleRequest {
|
||||
}
|
||||
repeated Replica replicas = 6;
|
||||
string auth = 7;
|
||||
int32 download_concurrency = 8; // multipart download concurrency if supported by the remote storage client; for S3, 0 = default (5)
|
||||
// remote conf
|
||||
remote_pb.RemoteConf remote_conf = 15;
|
||||
remote_pb.RemoteStorageLocation remote_location = 16;
|
||||
|
||||
@@ -63,9 +63,7 @@ pub struct Cli {
|
||||
#[arg(long = "rack", default_value = "")]
|
||||
pub rack: String,
|
||||
|
||||
/// Choose [memory|redb|redbMedium|redbLarge] mode for memory~performance balance.
|
||||
/// `leveldb`/`leveldbMedium`/`leveldbLarge` are accepted as aliases for the
|
||||
/// corresponding redb backends (Rust volume server uses redb under the hood).
|
||||
/// Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.
|
||||
#[arg(long = "index", default_value = "memory")]
|
||||
pub index: String,
|
||||
|
||||
@@ -703,16 +701,14 @@ fn resolve_config(cli: Cli) -> VolumeServerConfig {
|
||||
ip.clone()
|
||||
};
|
||||
|
||||
// Parse index type. Accept both `redb*` (preferred — what the volume server
|
||||
// actually uses) and the legacy `leveldb*` names as aliases.
|
||||
// Parse index type
|
||||
let index_type = match cli.index.as_str() {
|
||||
"memory" => NeedleMapKind::InMemory,
|
||||
"redb" | "leveldb" => NeedleMapKind::Redb,
|
||||
"redbMedium" | "leveldbMedium" => NeedleMapKind::RedbMedium,
|
||||
"redbLarge" | "leveldbLarge" => NeedleMapKind::RedbLarge,
|
||||
"leveldb" => NeedleMapKind::LevelDb,
|
||||
"leveldbMedium" => NeedleMapKind::LevelDbMedium,
|
||||
"leveldbLarge" => NeedleMapKind::LevelDbLarge,
|
||||
other => panic!(
|
||||
"Unknown index type: {}. Use memory|redb|redbMedium|redbLarge \
|
||||
(leveldb/leveldbMedium/leveldbLarge accepted as aliases)",
|
||||
"Unknown index type: {}. Use memory|leveldb|leveldbMedium|leveldbLarge",
|
||||
other
|
||||
),
|
||||
};
|
||||
@@ -1378,23 +1374,6 @@ mod tests {
|
||||
assert_eq!(cfg.folders, vec![default_volume_dir()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_config_index_accepts_redb_and_leveldb_aliases() {
|
||||
let pairs = [
|
||||
("memory", NeedleMapKind::InMemory),
|
||||
("redb", NeedleMapKind::Redb),
|
||||
("leveldb", NeedleMapKind::Redb),
|
||||
("redbMedium", NeedleMapKind::RedbMedium),
|
||||
("leveldbMedium", NeedleMapKind::RedbMedium),
|
||||
("redbLarge", NeedleMapKind::RedbLarge),
|
||||
("leveldbLarge", NeedleMapKind::RedbLarge),
|
||||
];
|
||||
for (input, expected) in pairs {
|
||||
let cfg = resolve_config(Cli::parse_from(["bin", "--index", input]));
|
||||
assert_eq!(cfg.index_type, expected, "input={}", input);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_security_config_access_ui() {
|
||||
let _guard = process_state_lock();
|
||||
|
||||
+38
-92
@@ -10,11 +10,9 @@ use seaweed_volume::security::tls::{
|
||||
GrpcClientAuthPolicy, TlsPolicy,
|
||||
};
|
||||
use seaweed_volume::security::{Guard, SigningKey};
|
||||
#[cfg(unix)]
|
||||
use seaweed_volume::server::debug::build_debug_router;
|
||||
use seaweed_volume::server::grpc_client::load_outgoing_grpc_tls;
|
||||
use seaweed_volume::server::grpc_server::VolumeGrpcService;
|
||||
#[cfg(unix)]
|
||||
use seaweed_volume::server::profiling::CpuProfileSession;
|
||||
use seaweed_volume::server::request_id::GrpcRequestIdLayer;
|
||||
use seaweed_volume::server::volume_server::{
|
||||
@@ -26,11 +24,6 @@ use seaweed_volume::storage::types::DiskType;
|
||||
|
||||
use tokio_rustls::TlsAcceptor;
|
||||
|
||||
#[cfg(unix)]
|
||||
type CpuProfileParam = Option<CpuProfileSession>;
|
||||
#[cfg(not(unix))]
|
||||
type CpuProfileParam = Option<()>;
|
||||
|
||||
const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
|
||||
const GRPC_KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
const GRPC_KEEPALIVE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
|
||||
@@ -49,7 +42,6 @@ fn main() {
|
||||
|
||||
let config = config::parse_cli();
|
||||
seaweed_volume::server::server_stats::init_process_start();
|
||||
#[cfg(unix)]
|
||||
let cpu_profile = match CpuProfileSession::start(&config) {
|
||||
Ok(session) => session,
|
||||
Err(e) => {
|
||||
@@ -57,8 +49,6 @@ fn main() {
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let cpu_profile: Option<()> = None;
|
||||
info!(
|
||||
"SeaweedFS Volume Server (Rust) v{}",
|
||||
seaweed_volume::version::full_version()
|
||||
@@ -267,7 +257,7 @@ where
|
||||
|
||||
async fn run(
|
||||
config: VolumeServerConfig,
|
||||
#[allow(unused_variables)] cpu_profile: CpuProfileParam,
|
||||
cpu_profile: Option<CpuProfileSession>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Initialize the store
|
||||
let mut store = Store::new(config.index_type);
|
||||
@@ -288,10 +278,6 @@ async fn run(
|
||||
config.jwt_read_signing_expires_seconds,
|
||||
);
|
||||
let master_url = config.masters.first().cloned().unwrap_or_default();
|
||||
// Defensive-copy the configured seed masters before freezing the lookup
|
||||
// set, so any later mutation of config.masters cannot desync them.
|
||||
let master_urls: Vec<String> = config.masters.clone();
|
||||
let seed_master_set = VolumeServerState::build_seed_master_set(&master_urls);
|
||||
let self_url = format!("{}:{}", config.ip, config.port);
|
||||
let (http_client, outgoing_http_scheme) = build_outgoing_http_client(&config)?;
|
||||
let outgoing_grpc_tls = load_outgoing_grpc_tls(&config)?;
|
||||
@@ -328,9 +314,7 @@ async fn run(
|
||||
),
|
||||
read_mode: config.read_mode,
|
||||
master_url,
|
||||
master_urls,
|
||||
seed_master_set,
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
master_urls: config.masters.clone(),
|
||||
self_url,
|
||||
http_client,
|
||||
outgoing_http_scheme,
|
||||
@@ -447,12 +431,10 @@ async fn run(
|
||||
}
|
||||
|
||||
// Build HTTP routers
|
||||
#[allow(unused_mut)]
|
||||
let mut admin_router = seaweed_volume::server::volume_server::build_admin_router_with_ui(
|
||||
state.clone(),
|
||||
config.ui_enabled,
|
||||
);
|
||||
#[cfg(unix)]
|
||||
if config.pprof {
|
||||
admin_router = admin_router.merge(build_debug_router());
|
||||
}
|
||||
@@ -606,35 +588,31 @@ async fn run(
|
||||
})
|
||||
};
|
||||
|
||||
// Bind the gRPC listener before spawning to propagate bind errors at startup.
|
||||
let grpc_listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let grpc_local_addr = grpc_listener
|
||||
.local_addr()
|
||||
.unwrap_or_else(|e| panic!("Failed to get gRPC local addr: {}", e));
|
||||
|
||||
let grpc_handle = {
|
||||
let grpc_state = state.clone();
|
||||
let grpc_addr = grpc_addr.clone();
|
||||
let grpc_tls_acceptor = grpc_tls_acceptor.clone();
|
||||
let mut shutdown_rx = shutdown_tx.subscribe();
|
||||
let shutdown_tx_grpc = shutdown_tx.clone();
|
||||
tokio::spawn(async move {
|
||||
let addr = grpc_addr.parse().expect("Invalid gRPC address");
|
||||
let grpc_service = VolumeGrpcService {
|
||||
state: grpc_state.clone(),
|
||||
};
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
let result = if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let incoming = grpc_tls_incoming(grpc_listener, tls_acceptor);
|
||||
info!("gRPC server listening on {} (TLS enabled)", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
if let Some(tls_acceptor) = grpc_tls_acceptor {
|
||||
let listener = tokio::net::TcpListener::bind(&grpc_addr)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to bind gRPC to {}: {}", grpc_addr, e));
|
||||
let incoming = grpc_tls_incoming(listener, tls_acceptor);
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {} (TLS enabled)", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
@@ -643,25 +621,32 @@ async fn run(
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
} else {
|
||||
let incoming =
|
||||
tokio_stream::wrappers::TcpListenerStream::new(grpc_listener);
|
||||
info!("gRPC server listening on {}", grpc_local_addr);
|
||||
build_grpc_server_builder()
|
||||
let reflection_v1 = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1()
|
||||
.expect("Failed to build gRPC reflection v1 service");
|
||||
let reflection_v1alpha = tonic_reflection::server::Builder::configure()
|
||||
.register_encoded_file_descriptor_set(seaweed_volume::pb::FILE_DESCRIPTOR_SET)
|
||||
.build_v1alpha()
|
||||
.expect("Failed to build gRPC reflection v1alpha service");
|
||||
info!("gRPC server listening on {}", addr);
|
||||
if let Err(e) = build_grpc_server_builder()
|
||||
.layer(GrpcRequestIdLayer)
|
||||
.add_service(reflection_v1)
|
||||
.add_service(reflection_v1alpha)
|
||||
.add_service(build_volume_grpc_service(grpc_service))
|
||||
.serve_with_incoming_shutdown(incoming, async move {
|
||||
.serve_with_shutdown(addr, async move {
|
||||
let _ = shutdown_rx.recv().await;
|
||||
})
|
||||
.await
|
||||
};
|
||||
if let Err(ref e) = result {
|
||||
error!("gRPC server error: {}", e);
|
||||
let _ = shutdown_tx_grpc.send(());
|
||||
{
|
||||
error!("gRPC server error: {}", e);
|
||||
}
|
||||
}
|
||||
result
|
||||
})
|
||||
};
|
||||
|
||||
@@ -736,7 +721,6 @@ async fn run(
|
||||
None
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
let debug_handle = if config.debug {
|
||||
let debug_addr = format!("0.0.0.0:{}", config.debug_port);
|
||||
info!("Debug pprof server listening on {}", debug_addr);
|
||||
@@ -758,8 +742,6 @@ async fn run(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let debug_handle: Option<tokio::task::JoinHandle<()>> = None;
|
||||
|
||||
let metrics_push_handle = {
|
||||
let push_state = state.clone();
|
||||
@@ -770,40 +752,9 @@ async fn run(
|
||||
}))
|
||||
};
|
||||
|
||||
// Wait for servers. Use select! with &mut so the losing handle is not
|
||||
// dropped, then await it explicitly afterward.
|
||||
let mut server_err: Option<String> = None;
|
||||
let mut http_handle = http_handle;
|
||||
let mut grpc_handle = grpc_handle;
|
||||
let grpc_finished_first = tokio::select! {
|
||||
_ = &mut http_handle => false,
|
||||
_ = &mut grpc_handle => true,
|
||||
};
|
||||
// Inspect the gRPC result (already resolved if it finished first,
|
||||
// otherwise await it now).
|
||||
let grpc_result = if grpc_finished_first {
|
||||
grpc_handle.await
|
||||
} else {
|
||||
// HTTP finished first; gRPC is still running. Await it.
|
||||
grpc_handle.await
|
||||
};
|
||||
match grpc_result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(e)) => {
|
||||
let msg = format!("gRPC server exited with error: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
// serve error already sent shutdown inside the task
|
||||
}
|
||||
Err(e) => {
|
||||
let msg = format!("gRPC task panicked: {}", e);
|
||||
error!("{}", msg);
|
||||
server_err = Some(msg);
|
||||
let _ = shutdown_tx.send(());
|
||||
}
|
||||
}
|
||||
// Ensure the HTTP handle completes too.
|
||||
// Wait for all servers
|
||||
let _ = http_handle.await;
|
||||
let _ = grpc_handle.await;
|
||||
if let Some(h) = public_handle {
|
||||
let _ = h.await;
|
||||
}
|
||||
@@ -823,15 +774,10 @@ async fn run(
|
||||
// Close all volumes (flush and release file handles) matching Go's Shutdown()
|
||||
state.store.write().unwrap().close();
|
||||
|
||||
#[cfg(unix)]
|
||||
if let Some(cpu_profile) = cpu_profile {
|
||||
cpu_profile.finish().map_err(std::io::Error::other)?;
|
||||
}
|
||||
|
||||
if let Some(err_msg) = server_err {
|
||||
return Err(std::io::Error::other(err_msg).into());
|
||||
}
|
||||
|
||||
info!("Volume server stopped.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -142,35 +142,6 @@ lazy_static::lazy_static! {
|
||||
&["code"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Scrubbing metrics (Go: VolumeServerScrub*) ----
|
||||
|
||||
/// Last scrub execution time, as seconds since UNIX epoch, with label `mode`.
|
||||
pub static ref SCRUB_LAST_TIME_SECONDS: GaugeVec = GaugeVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_last_time_seconds",
|
||||
"Last scrub execution time, as seconds since UNIX epoch.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Counter of overall volumes with issues detected during scrubbing, with label `mode`.
|
||||
pub static ref SCRUB_VOLUME_FAILURES: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_volume_failures",
|
||||
"Counter of overall volumes with issues detected during scrubbing.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Counter of overall EC shards with issues detected during scrubbing, with label `mode`.
|
||||
pub static ref SCRUB_SHARD_FAILURES: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_shard_failures",
|
||||
"Counter of overall EC shards with issues detected during scrubbing.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Legacy aliases for backward compat with existing code ----
|
||||
|
||||
/// Total number of volumes on this server (flat gauge).
|
||||
@@ -280,9 +251,6 @@ pub fn register_metrics() {
|
||||
Box::new(INFLIGHT_DOWNLOAD_SIZE.clone()),
|
||||
Box::new(INFLIGHT_UPLOAD_SIZE.clone()),
|
||||
Box::new(UPLOAD_ERROR_COUNTER.clone()),
|
||||
Box::new(SCRUB_LAST_TIME_SECONDS.clone()),
|
||||
Box::new(SCRUB_VOLUME_FAILURES.clone()),
|
||||
Box::new(SCRUB_SHARD_FAILURES.clone()),
|
||||
// Legacy metrics
|
||||
Box::new(VOLUMES_TOTAL.clone()),
|
||||
Box::new(DISK_SIZE_BYTES.clone()),
|
||||
|
||||
@@ -486,11 +486,6 @@ impl S3TierRegistry {
|
||||
self.backends.keys().cloned().collect()
|
||||
}
|
||||
|
||||
/// Remove a backend by name.
|
||||
pub fn remove(&mut self, name: &str) {
|
||||
self.backends.remove(name);
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.backends.clear();
|
||||
}
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.6 KiB After Width: | Height: | Size: 70 B |
@@ -107,57 +107,6 @@ pub fn build_grpc_endpoint(
|
||||
Ok(endpoint)
|
||||
}
|
||||
|
||||
/// Parse a SeaweedFS server address (`"ip:port.grpcPort"` or
|
||||
/// `"ip:port"`) into the `host:grpcPort` form `build_grpc_endpoint`
|
||||
/// expects. With the trailing `.grpcPort` segment, that segment IS
|
||||
/// the gRPC port; without it, the gRPC port is `port + 10000`
|
||||
/// (SeaweedFS's HTTP↔gRPC port-offset convention).
|
||||
///
|
||||
/// Shared between `grpc_server.rs` and the distributed-EC-read path
|
||||
/// in `store_ec.rs` — keep this as the single source of truth so the
|
||||
/// HTTP↔gRPC port translation can't drift between callers.
|
||||
pub fn parse_grpc_address(source: &str) -> Result<String, String> {
|
||||
let colon_idx = source
|
||||
.rfind(':')
|
||||
.ok_or_else(|| format!("cannot parse address: {}", source))?;
|
||||
let host = &source[..colon_idx];
|
||||
let port_part = &source[colon_idx + 1..];
|
||||
|
||||
if let Some(dot_idx) = port_part.rfind('.') {
|
||||
// Format: "ip:port.grpcPort". Validate BOTH ports as u16
|
||||
// so a malformed HTTP port (e.g. `host:abc.18080`) is
|
||||
// rejected here rather than tripping a downstream
|
||||
// `build_grpc_endpoint` URI parse failure with a less
|
||||
// useful error.
|
||||
let http_port = &port_part[..dot_idx];
|
||||
let grpc_port = &port_part[dot_idx + 1..];
|
||||
http_port
|
||||
.parse::<u16>()
|
||||
.map_err(|e| format!("invalid http port {:?}: {}", http_port, e))?;
|
||||
grpc_port
|
||||
.parse::<u16>()
|
||||
.map_err(|e| format!("invalid grpc port {:?}: {}", grpc_port, e))?;
|
||||
return Ok(format!("{}:{}", host, grpc_port));
|
||||
}
|
||||
|
||||
// Format: "ip:port" → grpc = port + 10000. Reject inputs whose
|
||||
// implicit grpc port would overflow the TCP port range (e.g.
|
||||
// `host:60000` produces 70000 — invalid). Without this check
|
||||
// the cast silently wraps and the endpoint call later fails
|
||||
// with an opaque connection error.
|
||||
let port: u16 = port_part
|
||||
.parse()
|
||||
.map_err(|e| format!("invalid port {:?}: {}", port_part, e))?;
|
||||
let grpc_port = port as u32 + 10000;
|
||||
if grpc_port > u16::MAX as u32 {
|
||||
return Err(format!(
|
||||
"implicit grpc port out of range: {} + 10000 = {}",
|
||||
port, grpc_port
|
||||
));
|
||||
}
|
||||
Ok(format!("{}:{}", host, grpc_port))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_grpc_endpoint, grpc_endpoint_uri, load_outgoing_grpc_tls};
|
||||
@@ -254,50 +203,4 @@ mod tests {
|
||||
let endpoint = build_grpc_endpoint("127.0.0.1:19333", None).unwrap();
|
||||
assert_eq!(endpoint.uri().scheme_str(), Some("http"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_dotted_form() {
|
||||
use super::parse_grpc_address;
|
||||
assert_eq!(
|
||||
parse_grpc_address("127.0.0.1:8080.18080").unwrap(),
|
||||
"127.0.0.1:18080"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_implicit_form_adds_10000() {
|
||||
use super::parse_grpc_address;
|
||||
assert_eq!(
|
||||
parse_grpc_address("127.0.0.1:8080").unwrap(),
|
||||
"127.0.0.1:18080"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_non_numeric_http_port_in_dotted_form() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("host:abc.18080").unwrap_err();
|
||||
assert!(err.contains("invalid http port"), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_non_numeric_grpc_port_in_dotted_form() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("host:8080.xyz").unwrap_err();
|
||||
assert!(err.contains("invalid grpc port"), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_implicit_port_that_overflows() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("127.0.0.1:60000").unwrap_err();
|
||||
assert!(err.contains("out of range"), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_input_without_colon() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("hostname").unwrap_err();
|
||||
assert!(err.contains("cannot parse"), "{}", err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ use crate::pb::master_pb;
|
||||
use crate::pb::master_pb::seaweed_client::SeaweedClient;
|
||||
use crate::pb::volume_server_pb;
|
||||
use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
use crate::storage::needle::needle::{self, Needle};
|
||||
use crate::storage::types::*;
|
||||
|
||||
@@ -29,42 +28,6 @@ fn volume_is_remote_only(dat_path: &str, has_remote_file: bool) -> bool {
|
||||
has_remote_file && !std::path::Path::new(dat_path).exists()
|
||||
}
|
||||
|
||||
/// Map a numeric `VolumeScrubMode` to its proto enum name, matching Go's
|
||||
/// `req.GetMode().String()` used for the Prometheus `mode` label.
|
||||
fn scrub_mode_label(mode: i32) -> &'static str {
|
||||
match mode {
|
||||
0 => "UNKNOWN",
|
||||
1 => "INDEX",
|
||||
2 => "FULL",
|
||||
3 => "LOCAL",
|
||||
_ => "UNKNOWN",
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_now_seconds() -> f64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as f64)
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// Record scrub metrics. `broken_shards` is `Some` only for EC scrubs so the
|
||||
/// shard-failures family stays untouched on regular volume scrubs (matching Go).
|
||||
fn emit_scrub_metrics(mode: i32, broken_volumes: usize, broken_shards: Option<usize>) {
|
||||
let mode_label = scrub_mode_label(mode);
|
||||
crate::metrics::SCRUB_LAST_TIME_SECONDS
|
||||
.with_label_values(&[mode_label])
|
||||
.set(unix_now_seconds());
|
||||
crate::metrics::SCRUB_VOLUME_FAILURES
|
||||
.with_label_values(&[mode_label])
|
||||
.inc_by(broken_volumes as u64);
|
||||
if let Some(n) = broken_shards {
|
||||
crate::metrics::SCRUB_SHARD_FAILURES
|
||||
.with_label_values(&[mode_label])
|
||||
.inc_by(n as u64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Persist VolumeServerState to a state.pb file (matches Go's State.save).
|
||||
fn save_state_file(
|
||||
path: &str,
|
||||
@@ -145,37 +108,6 @@ pub struct VolumeGrpcService {
|
||||
}
|
||||
|
||||
impl VolumeGrpcService {
|
||||
/// Verifies the gRPC caller is allowed to invoke a destructive admin
|
||||
/// operation. Mirrors the Go side's checkGrpcAdminAuth: an empty
|
||||
/// whitelist accepts everyone (insecure-by-default for tests and
|
||||
/// upgrades), a populated whitelist accepts only matching peer IPs.
|
||||
///
|
||||
/// `remote_addr()` on a real gRPC connection always yields the peer's
|
||||
/// SocketAddr; if it is somehow None we deny, matching "if we don't
|
||||
/// know who the caller is, refuse."
|
||||
fn check_grpc_admin_auth<T>(&self, request: &Request<T>) -> Result<(), Status> {
|
||||
let remote = match request.remote_addr() {
|
||||
Some(addr) => addr,
|
||||
None => {
|
||||
tracing::warn!("gRPC admin auth failed: no peer info");
|
||||
return Err(Status::permission_denied("no peer info"));
|
||||
}
|
||||
};
|
||||
let host = remote.ip().to_string();
|
||||
let guard = self.state.guard.read().unwrap();
|
||||
if !guard.check_whitelist(&host) {
|
||||
tracing::warn!(
|
||||
"gRPC admin auth failed: {} is not whitelisted (remote: {})",
|
||||
host,
|
||||
remote,
|
||||
);
|
||||
return Err(Status::permission_denied(format!(
|
||||
"not authorized: {host}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn notify_master_volume_readonly(
|
||||
&self,
|
||||
info: &MasterVolumeInfo,
|
||||
@@ -490,7 +422,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCompactRequest>,
|
||||
) -> Result<Response<Self::VacuumVolumeCompactStream>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -550,7 +481,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCommitRequest>,
|
||||
) -> Result<Response<volume_server_pb::VacuumVolumeCommitResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
|
||||
@@ -586,7 +516,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCleanupRequest>,
|
||||
) -> Result<Response<volume_server_pb::VacuumVolumeCleanupResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
@@ -602,7 +531,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::DeleteCollectionRequest>,
|
||||
) -> Result<Response<volume_server_pb::DeleteCollectionResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let collection = &request.into_inner().collection;
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
store
|
||||
@@ -615,7 +543,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::AllocateVolumeRequest>,
|
||||
) -> Result<Response<volume_server_pb::AllocateVolumeResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -764,7 +691,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMountRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMountResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
@@ -781,7 +707,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeUnmountRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeUnmountResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
// Go returns nil when volume is not found (idempotent unmount)
|
||||
@@ -795,7 +720,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeDeleteRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeDeleteResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -809,7 +733,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
store
|
||||
.delete_volume(vid, req.only_empty, req.keep_remote_data)
|
||||
.delete_volume(vid, req.only_empty)
|
||||
.map_err(|e| Status::internal(e.to_string()))?;
|
||||
self.state.volume_state_notify.notify_one();
|
||||
Ok(Response::new(volume_server_pb::VolumeDeleteResponse {}))
|
||||
@@ -819,7 +743,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMarkReadonlyRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMarkReadonlyResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
// Go: volume lookup (L239-241) happens before maintenance check (L166 in makeVolumeReadonly)
|
||||
@@ -839,7 +762,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMarkWritableRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMarkWritableResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
let info = {
|
||||
@@ -889,7 +811,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeConfigureRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeConfigureResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -1043,10 +964,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
if store.find_volume(vid).is_some() {
|
||||
drop(store);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
// keep remote data: the inbound copy carries a .vif that may
|
||||
// point at the same cloud-tier object the existing volume
|
||||
// references.
|
||||
store.delete_volume(vid, false, true).map_err(|e| {
|
||||
store.delete_volume(vid, false).map_err(|e| {
|
||||
Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
|
||||
})?;
|
||||
self.state.volume_state_notify.notify_one();
|
||||
@@ -1507,47 +1425,13 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Determine file path
|
||||
let path = if info.is_ec_volume {
|
||||
let store = self.state.store.read().unwrap();
|
||||
// std::fs::File::create truncates in place; a mounted
|
||||
// EcVolume holds fds on the same inodes, so overwriting
|
||||
// corrupts live readers.
|
||||
if store.has_ec_volume(VolumeId(info.volume_id)) {
|
||||
let mounted_disks =
|
||||
store.find_ec_volume_disk_ids(VolumeId(info.volume_id));
|
||||
resp_error = Some(format!(
|
||||
"ec volume {} is mounted on disk_ids:{:?}; unmount before ReceiveFile",
|
||||
info.volume_id, mounted_disks
|
||||
));
|
||||
break;
|
||||
}
|
||||
// disk_id=0 means "unset" (protobuf default), so auto-select
|
||||
// using the same primitive as volume_ec_shards_copy: prefer
|
||||
// a disk that has the EC volume mounted, then a disk that
|
||||
// owns the .ecx on disk (volume not yet mounted — relevant
|
||||
// when shards stream in mid-rebuild before any
|
||||
// VolumeEcShardsMount has happened; see #9212), then any
|
||||
// HDD, then any disk. Pass the build's default data-shard
|
||||
// count for free-slot maths; the helper takes it as a
|
||||
// parameter so custom-ratio builds can swap it.
|
||||
let vid = VolumeId(info.volume_id);
|
||||
let dir = if info.disk_id > 0 {
|
||||
let count = store.locations.len();
|
||||
if (info.disk_id as usize) >= count {
|
||||
resp_error = Some(format!(
|
||||
"invalid disk_id {}: only have {} disks",
|
||||
info.disk_id, count
|
||||
));
|
||||
break;
|
||||
}
|
||||
Some(store.locations[info.disk_id as usize].directory.clone())
|
||||
} else {
|
||||
store
|
||||
.find_ec_shard_target_location(
|
||||
&info.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
)
|
||||
.map(|i| store.locations[i].directory.clone())
|
||||
};
|
||||
// Go prefers a HardDriveType location, then falls back to first
|
||||
let dir = store
|
||||
.locations
|
||||
.iter()
|
||||
.find(|loc| loc.disk_type == DiskType::HardDrive)
|
||||
.or_else(|| store.locations.first())
|
||||
.map(|loc| loc.directory.clone());
|
||||
drop(store);
|
||||
let dir = match dir {
|
||||
Some(d) => d,
|
||||
@@ -1983,7 +1867,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
{
|
||||
let needle_header = resp.needle_header;
|
||||
let mut needle_body = resp.needle_body;
|
||||
let resp_version = resp.version;
|
||||
|
||||
if needle_header.is_empty() {
|
||||
continue;
|
||||
@@ -2008,36 +1891,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Parse needle from header + body
|
||||
let mut n = Needle::default();
|
||||
n.read_header(&needle_header);
|
||||
|
||||
if n.size.0 < 0 {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"unexpected negative needle size {} for needle {}",
|
||||
n.size.0, n.id.0
|
||||
)));
|
||||
} else if n.size.0 > 0 {
|
||||
// Normal needle: parse the body fields (DataSize, Data, flags, etc.)
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
} else {
|
||||
// Delete tombstone (size == 0): body is checksum + timestamp
|
||||
// (V3) or checksum only (V2) + padding. Validate minimum
|
||||
// footer length for the protocol version.
|
||||
use crate::storage::types::{
|
||||
NEEDLE_CHECKSUM_SIZE, TIMESTAMP_SIZE, VERSION_3, Version,
|
||||
};
|
||||
let version = Version(resp_version as u8);
|
||||
let min_footer = if version >= VERSION_3 {
|
||||
NEEDLE_CHECKSUM_SIZE + TIMESTAMP_SIZE
|
||||
} else {
|
||||
NEEDLE_CHECKSUM_SIZE
|
||||
};
|
||||
if needle_body.len() < min_footer {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"tombstone needle {} body too short: got {} bytes, need >= {} for version {}",
|
||||
n.id.0, needle_body.len(), min_footer, resp_version
|
||||
)));
|
||||
}
|
||||
}
|
||||
n.read_body_v2(&needle_body)
|
||||
.map_err(|e| Status::internal(format!("parse needle body: {}", e)))?;
|
||||
|
||||
// Write needle to local volume
|
||||
let mut store = state.store.write().unwrap();
|
||||
@@ -2099,9 +1954,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
// Check existing .vif for EC shard config (matching Go's MaybeLoadVolumeInfo)
|
||||
let (data_shards, parity_shards) =
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
|
||||
&dir, &idx_dir, collection, vid,
|
||||
);
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(&dir, collection, vid);
|
||||
|
||||
if let Err(e) = crate::storage::erasure_coding::ec_encoder::write_ec_files(
|
||||
&dir,
|
||||
@@ -2255,7 +2108,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let (data_shards, parity_shards) =
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
|
||||
&rebuild_dir,
|
||||
&rebuild_idx_dir,
|
||||
collection,
|
||||
vid,
|
||||
);
|
||||
@@ -2339,17 +2191,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
// Select target location:
|
||||
// When disk_id > 0: use that specific location.
|
||||
// When disk_id == 0 (unset): auto-select via
|
||||
// find_ec_shard_target_location, which prefers a disk that
|
||||
// already has the EC volume mounted, then a disk that owns the
|
||||
// .ecx on disk (volume not yet mounted — relevant for
|
||||
// ec.rebuild, where only the first shard carries .ecx and
|
||||
// subsequent shards must land on the same disk; see #9212),
|
||||
// then any HDD, then any disk. Pass the build's default
|
||||
// data-shard count; the helper takes it as a parameter so
|
||||
// custom-ratio builds can swap it.
|
||||
// Select target location matching Go's 3-tier fallback:
|
||||
// When disk_id > 0: use that specific location
|
||||
// When disk_id == 0 (unset): (1) location with existing EC shards, (2) any HDD, (3) any
|
||||
let (dest_dir, dest_idx_dir) = {
|
||||
let store = self.state.store.read().unwrap();
|
||||
let count = store.locations.len();
|
||||
@@ -2365,11 +2209,20 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let loc = &store.locations[req.disk_id as usize];
|
||||
(loc.directory.clone(), loc.idx_directory.clone())
|
||||
} else {
|
||||
match store.find_ec_shard_target_location(
|
||||
&req.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
) {
|
||||
// Auto-select: prefer location with existing EC shards for this volume
|
||||
let loc_idx = store
|
||||
.find_free_location_predicate(|loc| loc.has_ec_volume(vid))
|
||||
.or_else(|| {
|
||||
// Fall back to any HDD location
|
||||
store.find_free_location_predicate(|loc| {
|
||||
loc.disk_type == DiskType::HardDrive
|
||||
})
|
||||
})
|
||||
.or_else(|| {
|
||||
// Fall back to any location
|
||||
store.find_free_location_predicate(|_| true)
|
||||
});
|
||||
match loc_idx {
|
||||
Some(i) => {
|
||||
let loc = &store.locations[i];
|
||||
(loc.directory.clone(), loc.idx_directory.clone())
|
||||
@@ -2589,7 +2442,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeEcShardsDeleteRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeEcShardsDeleteResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -2614,7 +2466,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
for &shard_id in &req.shard_ids {
|
||||
store
|
||||
.mount_ec_shard(vid, &req.collection, shard_id, &req.source_disk_type)
|
||||
.mount_ec_shard(vid, &req.collection, shard_id)
|
||||
.map_err(|e| {
|
||||
Status::internal(format!("mount {}.{}: {}", req.volume_id, shard_id, e))
|
||||
})?;
|
||||
@@ -2658,19 +2510,12 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
let store = self.state.store.read().unwrap();
|
||||
// Reconciled EC volumes can have their shards split across
|
||||
// disks (e.g. shards 0/12 on disk 0, shard 1 on disk 1), so
|
||||
// resolve the EcVolume from the *shard*'s home location
|
||||
// rather than first-match `find_ec_volume(vid)` which would
|
||||
// miss shards that live on a sibling. Mirrors Go's findEcShard.
|
||||
let ec_vol = store
|
||||
.find_ec_volume_with_shard(vid, req.shard_id)
|
||||
.ok_or_else(|| {
|
||||
Status::not_found(format!(
|
||||
"ec volume {} shard {} not found",
|
||||
req.volume_id, req.shard_id
|
||||
))
|
||||
})?;
|
||||
let ec_vol = store.find_ec_volume(vid).ok_or_else(|| {
|
||||
Status::not_found(format!(
|
||||
"ec volume {} shard {} not found",
|
||||
req.volume_id, req.shard_id
|
||||
))
|
||||
})?;
|
||||
|
||||
// Check if the requested needle is deleted (via .ecx index, matching Go)
|
||||
if req.file_key > 0 {
|
||||
@@ -2689,8 +2534,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
|
||||
// Read from the shard. Guaranteed to be present because
|
||||
// find_ec_volume_with_shard already verified it.
|
||||
// Read from the shard
|
||||
let shard = ec_vol
|
||||
.shards
|
||||
.get(req.shard_id as usize)
|
||||
@@ -2780,13 +2624,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
let store = self.state.store.read().unwrap();
|
||||
// Aggregate per-shard data dirs across all locations so the
|
||||
// shard-presence check + decoder both see the union for
|
||||
// cross-disk reconciled volumes (#9252). Mirrors Go's
|
||||
// CollectEcShards.
|
||||
let max_shard_count = crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT;
|
||||
let (ec_vol, shard_dirs) = store
|
||||
.collect_ec_shard_dirs(vid, max_shard_count)
|
||||
let ec_vol = store
|
||||
.find_ec_volume(vid)
|
||||
.ok_or_else(|| Status::not_found(format!("ec volume {} not found", req.volume_id)))?;
|
||||
|
||||
if ec_vol.collection != req.collection {
|
||||
@@ -2800,6 +2639,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let data_shards = ec_vol.data_shards as usize;
|
||||
|
||||
// Validate data shard count range (matches Go's VolumeEcShardsToVolume)
|
||||
let max_shard_count = crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT;
|
||||
if data_shards == 0 || data_shards > max_shard_count {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"invalid data shard count {} for volume {} (must be 1..{})",
|
||||
@@ -2807,9 +2647,14 @@ impl VolumeServer for VolumeGrpcService {
|
||||
)));
|
||||
}
|
||||
|
||||
// Check that all data shards are present somewhere on this server.
|
||||
// Check that all data shards are present
|
||||
for shard_id in 0..data_shards {
|
||||
if shard_dirs[shard_id].is_none() {
|
||||
if ec_vol
|
||||
.shards
|
||||
.get(shard_id)
|
||||
.map(|s| s.is_none())
|
||||
.unwrap_or(true)
|
||||
{
|
||||
return Err(Status::internal(format!(
|
||||
"ec volume {} missing shard {}",
|
||||
req.volume_id, shard_id
|
||||
@@ -2841,46 +2686,29 @@ impl VolumeServer for VolumeGrpcService {
|
||||
)));
|
||||
}
|
||||
|
||||
// Reconstruct the volume from EC shards. Use the EcVolume's
|
||||
// own dir for the produced .dat (matches the volume's home
|
||||
// disk) and its `ecx_actual_dir` for the .ecx lookup, while
|
||||
// reading each shard from its real on-disk location.
|
||||
let dat_dir = ec_vol.dir.clone();
|
||||
let ecx_dir = ec_vol.ecx_actual_dir().to_string();
|
||||
// Reconstruct the volume from EC shards
|
||||
let dir = ec_vol.dir.clone();
|
||||
let collection = ec_vol.collection.clone();
|
||||
// shard_dirs[i] is guaranteed Some for i in 0..data_shards by
|
||||
// the check above; collect concrete dirs for the decoder.
|
||||
let per_shard_dirs: Vec<String> = shard_dirs[..data_shards]
|
||||
.iter()
|
||||
.map(|d| d.clone().unwrap())
|
||||
.collect();
|
||||
drop(store);
|
||||
|
||||
// Calculate .dat file size from .ecx entries (.ec00 lives on
|
||||
// its own disk, .ecx on the index disk).
|
||||
// Calculate .dat file size from .ecx entries
|
||||
let dat_file_size =
|
||||
crate::storage::erasure_coding::ec_decoder::find_dat_file_size_with_dirs(
|
||||
&per_shard_dirs[0],
|
||||
&ecx_dir,
|
||||
&collection,
|
||||
vid,
|
||||
)
|
||||
.map_err(|e| Status::internal(format!("FindDatFileSize: {}", e)))?;
|
||||
crate::storage::erasure_coding::ec_decoder::find_dat_file_size(&dir, &collection, vid)
|
||||
.map_err(|e| Status::internal(format!("FindDatFileSize: {}", e)))?;
|
||||
|
||||
// Write .dat file using block-interleaved reading from shards.
|
||||
crate::storage::erasure_coding::ec_decoder::write_dat_file_from_shards_with_dirs(
|
||||
&dat_dir,
|
||||
// Write .dat file using block-interleaved reading from shards
|
||||
crate::storage::erasure_coding::ec_decoder::write_dat_file_from_shards(
|
||||
&dir,
|
||||
&collection,
|
||||
vid,
|
||||
dat_file_size,
|
||||
data_shards,
|
||||
&per_shard_dirs,
|
||||
)
|
||||
.map_err(|e| Status::internal(format!("WriteDatFile: {}", e)))?;
|
||||
|
||||
// Write .idx file from .ecx and .ecj files (lives on idx dir).
|
||||
// Write .idx file from .ecx and .ecj files
|
||||
crate::storage::erasure_coding::ec_decoder::write_idx_file_from_ec_index(
|
||||
&dat_dir,
|
||||
&dir,
|
||||
&collection,
|
||||
vid,
|
||||
)
|
||||
@@ -3288,9 +3116,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
async fn volume_server_leave(
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeServerLeaveRequest>,
|
||||
_request: Request<volume_server_pb::VolumeServerLeaveRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeServerLeaveResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
*self.state.is_stopping.write().unwrap() = true;
|
||||
self.state.is_heartbeating.store(false, Ordering::Relaxed);
|
||||
// Wake heartbeat loop to send deregistration.
|
||||
@@ -3440,7 +3267,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::ScrubVolumeRequest>,
|
||||
) -> Result<Response<volume_server_pb::ScrubVolumeResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
|
||||
// Validate mode
|
||||
@@ -3505,8 +3331,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
// Match Go: if mark_broken_volumes_readonly, call makeVolumeReadonly on each broken volume.
|
||||
// Collect errors via errors.Join semantics (return joined error if any fail).
|
||||
let mut errs: Vec<String> = Vec::new();
|
||||
if req.mark_broken_volumes_readonly {
|
||||
let mut errs: Vec<String> = Vec::new();
|
||||
for vid in &broken_vids {
|
||||
match self.make_volume_readonly(*vid, true).await {
|
||||
Ok(()) => {
|
||||
@@ -3518,14 +3344,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Record metrics before the post-scrub error check so scrub failures are
|
||||
// persisted even when a follow-up admin action (mark-readonly) fails.
|
||||
emit_scrub_metrics(mode, broken_vids.len(), None);
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(Status::internal(errs.join("\n")));
|
||||
if !errs.is_empty() {
|
||||
return Err(Status::internal(errs.join("\n")));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Response::new(volume_server_pb::ScrubVolumeResponse {
|
||||
@@ -3595,15 +3416,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// LOCAL (2) / FULL (3): verify EC shard data
|
||||
let files = ecv.walk_ecx_stats().map(|(f, _, _)| f).unwrap_or(0);
|
||||
|
||||
// After cross-disk reconciliation, an EcVolume can
|
||||
// legitimately have ecv.dir != ecv.dir_idx (shards
|
||||
// on one disk, .ecx / .ecj / .vif on a sibling).
|
||||
// Use the EcVolume's own dirs rather than collapsing
|
||||
// both args to find_ec_dir's single answer, otherwise
|
||||
// read_ec_shard_config falls back to the wrong .vif
|
||||
// location for split-disk volumes (#9252).
|
||||
let dir = ecv.dir.clone();
|
||||
let idx_dir = ecv.dir_idx.clone();
|
||||
let dir = store
|
||||
.find_ec_dir(*vid, &collection)
|
||||
.unwrap_or_else(|| String::from(""));
|
||||
if dir.is_empty() {
|
||||
continue;
|
||||
}
|
||||
@@ -3613,7 +3428,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let (data_shards, parity_shards) =
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
|
||||
&dir,
|
||||
&idx_dir,
|
||||
&collection,
|
||||
*vid,
|
||||
);
|
||||
@@ -3651,12 +3465,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
|
||||
emit_scrub_metrics(
|
||||
mode,
|
||||
broken_volume_ids.len(),
|
||||
Some(broken_shard_infos.len()),
|
||||
);
|
||||
|
||||
Ok(Response::new(volume_server_pb::ScrubEcVolumeResponse {
|
||||
total_volumes,
|
||||
total_files,
|
||||
@@ -3900,24 +3708,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
let start = now_ns();
|
||||
|
||||
// Empty target is a self-liveness probe and stays unauthenticated.
|
||||
// Otherwise gate the dial on cluster membership: volume servers only
|
||||
// know masters, so any other target type is refused. Mirrors Go's
|
||||
// volume_grpc_admin.go Ping admission check. tonic forbids returning
|
||||
// a body alongside an error, so we surface the InvalidArgument status
|
||||
// alone — behaviour-identical to Go's status.Errorf return.
|
||||
if !req.target.is_empty()
|
||||
&& !self
|
||||
.state
|
||||
.is_known_ping_target(&req.target, &req.target_type)
|
||||
.await
|
||||
{
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"unknown ping target {} of type {}",
|
||||
req.target, req.target_type
|
||||
)));
|
||||
}
|
||||
|
||||
// Route ping based on target type (matches Go's volume_grpc_admin.go Ping)
|
||||
let remote_time_ns = if req.target_type == "volumeServer" {
|
||||
match ping_volume_server_target(&req.target, self.state.outgoing_grpc_tls.as_ref())
|
||||
@@ -4053,12 +3843,29 @@ async fn ping_filer_target(
|
||||
Ok(resp.into_inner().start_time_ns)
|
||||
}
|
||||
|
||||
// parse_grpc_address moved to super::grpc_client::parse_grpc_address
|
||||
// for sharing with the distributed-EC-read path in server/store_ec.rs.
|
||||
// In-file callers below still write `parse_grpc_address(...)`; this
|
||||
// `use` makes them resolve to the new home without churning every
|
||||
// call site.
|
||||
use super::grpc_client::parse_grpc_address;
|
||||
/// Parse a SeaweedFS server address ("ip:port.grpcPort" or "ip:port") into a gRPC address.
|
||||
fn parse_grpc_address(source: &str) -> Result<String, String> {
|
||||
if let Some(colon_idx) = source.rfind(':') {
|
||||
let port_part = &source[colon_idx + 1..];
|
||||
if let Some(dot_idx) = port_part.rfind('.') {
|
||||
// Format: "ip:port.grpcPort"
|
||||
let host = &source[..colon_idx];
|
||||
let grpc_port = &port_part[dot_idx + 1..];
|
||||
grpc_port
|
||||
.parse::<u16>()
|
||||
.map_err(|e| format!("invalid grpc port: {}", e))?;
|
||||
return Ok(format!("{}:{}", host, grpc_port));
|
||||
}
|
||||
// Format: "ip:port" → grpc = port + 10000
|
||||
let port: u16 = port_part
|
||||
.parse()
|
||||
.map_err(|e| format!("invalid port: {}", e))?;
|
||||
let grpc_port = port as u32 + 10000;
|
||||
let host = &source[..colon_idx];
|
||||
return Ok(format!("{}:{}", host, grpc_port));
|
||||
}
|
||||
Err(format!("cannot parse address: {}", source))
|
||||
}
|
||||
|
||||
/// Set the modification time of a file from nanoseconds since Unix epoch.
|
||||
fn set_file_mtime(path: &str, modified_ts_ns: i64) {
|
||||
@@ -4240,12 +4047,11 @@ fn find_last_append_at_ns(idx_path: &str, dat_path: &str, version: u32) -> Optio
|
||||
let mut header = [0u8; 16];
|
||||
dat_file.read_exact(&mut header).ok()?;
|
||||
let needle_size = i32::from_be_bytes([header[12], header[13], header[14], header[15]]);
|
||||
if needle_size < 0 {
|
||||
if needle_size <= 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Seek to tail: offset + 16 (header) + size -> checksum (4) + timestamp (8)
|
||||
// For delete needles (size == 0), the tail is right after the header.
|
||||
let tail_offset = actual_offset as u64 + 16 + needle_size as u64;
|
||||
dat_file.seek(SeekFrom::Start(tail_offset)).ok()?;
|
||||
|
||||
@@ -4356,7 +4162,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -4418,7 +4223,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -4428,8 +4232,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -4475,9 +4277,7 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
let tier_config = S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4489,16 +4289,14 @@ mod tests {
|
||||
};
|
||||
{
|
||||
let mut registry = global_s3_tier_registry().write().unwrap();
|
||||
registry.register(
|
||||
"s3.incr_copy_test".to_string(),
|
||||
S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register("s3.default".to_string(), S3TierBackend::new(&tier_config));
|
||||
registry.register("s3".to_string(), S3TierBackend::new(&tier_config));
|
||||
}
|
||||
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
files: vec![crate::storage::volume::VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "incr_copy_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4563,8 +4361,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -4667,8 +4463,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -4711,210 +4505,7 @@ mod tests {
|
||||
assert_eq!(copied, dat_bytes[super_block_size as usize..]);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.incr_copy_test");
|
||||
}
|
||||
|
||||
/// Build a bare-bones service with no on-disk store but a configurable
|
||||
/// seed master set, for Ping admission tests. Matches the structure of
|
||||
/// `make_local_service_with_volume` minus the volume bits.
|
||||
fn make_service_with_seed_masters(seeds: &[&str]) -> (VolumeGrpcService, TempDir) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let master_urls: Vec<String> = seeds.iter().map(|s| (*s).to_string()).collect();
|
||||
let seed_master_set =
|
||||
crate::server::volume_server::VolumeServerState::build_seed_master_set(&master_urls);
|
||||
|
||||
let state = Arc::new(VolumeServerState {
|
||||
store: RwLock::new(store),
|
||||
guard: RwLock::new(Guard::new(
|
||||
&[],
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
)),
|
||||
is_stopping: RwLock::new(false),
|
||||
maintenance: std::sync::atomic::AtomicBool::new(false),
|
||||
state_version: std::sync::atomic::AtomicU32::new(0),
|
||||
concurrent_upload_limit: 0,
|
||||
concurrent_download_limit: 0,
|
||||
inflight_upload_data_timeout: std::time::Duration::from_secs(60),
|
||||
inflight_download_data_timeout: std::time::Duration::from_secs(60),
|
||||
inflight_upload_bytes: std::sync::atomic::AtomicI64::new(0),
|
||||
inflight_download_bytes: std::sync::atomic::AtomicI64::new(0),
|
||||
upload_notify: tokio::sync::Notify::new(),
|
||||
download_notify: tokio::sync::Notify::new(),
|
||||
data_center: String::new(),
|
||||
rack: String::new(),
|
||||
file_size_limit_bytes: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
|
||||
has_master: false,
|
||||
pre_stop_seconds: 0,
|
||||
volume_state_notify: tokio::sync::Notify::new(),
|
||||
write_queue: std::sync::OnceLock::new(),
|
||||
s3_tier_registry: std::sync::RwLock::new(
|
||||
crate::remote_storage::s3_tier::S3TierRegistry::new(),
|
||||
),
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: master_urls.first().cloned().unwrap_or_default(),
|
||||
master_urls,
|
||||
seed_master_set,
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
outgoing_grpc_tls: None,
|
||||
metrics_runtime: std::sync::RwLock::new(
|
||||
crate::server::volume_server::RuntimeMetricsConfig::default(),
|
||||
),
|
||||
metrics_notify: tokio::sync::Notify::new(),
|
||||
fix_jpg_orientation: false,
|
||||
has_slow_read: false,
|
||||
read_buffer_size_bytes: 1024 * 1024,
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
});
|
||||
|
||||
(VolumeGrpcService { state }, tmp)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_empty_target_is_self_probe() {
|
||||
// Empty target stays unauthenticated and returns Ok with timing fields
|
||||
// populated — it is the local liveness probe path.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&[]);
|
||||
let response = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: String::new(),
|
||||
target_type: String::new(),
|
||||
}))
|
||||
.await
|
||||
.expect("empty target ping must succeed");
|
||||
let inner = response.into_inner();
|
||||
assert!(inner.start_time_ns > 0);
|
||||
assert!(inner.stop_time_ns >= inner.start_time_ns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_seed_master_target_passes_admission() {
|
||||
// A target that matches a configured seed master clears admission.
|
||||
// The dial itself may or may not succeed depending on what's listening
|
||||
// on the loopback; either way, the response must not be the
|
||||
// InvalidArgument the gate would surface.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9333".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
assert!(
|
||||
!err.message().contains("unknown ping target"),
|
||||
"admission gate should have allowed this target: {}",
|
||||
err.message()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_unknown_master_target_rejected() {
|
||||
// A master-type target not in the seed list and not the current
|
||||
// master is refused with InvalidArgument.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let err = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9999".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await
|
||||
.expect_err("unknown master target must be rejected");
|
||||
assert_eq!(err.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
"unknown ping target localhost:9999 of type master"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_volume_server_target_always_rejected() {
|
||||
// Volume servers do not maintain a peer-volume list, so volumeServer
|
||||
// pings are refused regardless of address.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let err = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:8080".to_string(),
|
||||
target_type: "volumeServer".to_string(),
|
||||
}))
|
||||
.await
|
||||
.expect_err("volumeServer target must be rejected");
|
||||
assert_eq!(err.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
"unknown ping target localhost:8080 of type volumeServer"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_current_master_target_passes_admission() {
|
||||
// A target that matches the current (post-leader-change) master also
|
||||
// clears admission, even if it is not in the seed list. Pick a port
|
||||
// that is extremely unlikely to be live so the test does not flake on
|
||||
// a developer machine that happens to be running a real master.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
// Simulate the heartbeat goroutine having moved to a new leader.
|
||||
*service.state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "127.0.0.1:1".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
assert!(
|
||||
!err.message().contains("unknown ping target"),
|
||||
"leader-change master should be admitted: {}",
|
||||
err.message()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_target_with_grpc_port_suffix_is_normalised() {
|
||||
// Seed masters in pb.ServerAddress form (`host:port.grpcPort`) must
|
||||
// match a Ping target sent in plain `host:port` form, since the gate
|
||||
// normalises both sides through to_http_address.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333.19333"]);
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9333".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
}
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,7 +16,7 @@ use axum::response::{IntoResponse, Response};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::grpc_client::{build_grpc_endpoint, GRPC_MAX_MESSAGE_SIZE};
|
||||
use super::volume_server::{normalize_outgoing_http_url, to_http_address, VolumeServerState};
|
||||
use super::volume_server::{normalize_outgoing_http_url, VolumeServerState};
|
||||
use crate::config::ReadMode;
|
||||
use crate::metrics;
|
||||
use crate::pb::volume_server_pb;
|
||||
@@ -357,12 +357,9 @@ async fn lookup_volume(
|
||||
master_url: &str,
|
||||
volume_id: u32,
|
||||
) -> Result<Vec<VolumeLocation>, String> {
|
||||
// master_url may be in SeaweedFS's "host:httpPort.grpcPort" form (mirrors
|
||||
// Go's pb.ServerAddress); strip the gRPC port before building the HTTP URL.
|
||||
let master_http = to_http_address(master_url);
|
||||
let url = normalize_outgoing_http_url(
|
||||
scheme,
|
||||
&format!("{}/dir/lookup?volumeId={}", master_http, volume_id),
|
||||
&format!("{}/dir/lookup?volumeId={}", master_url, volume_id),
|
||||
)?;
|
||||
let resp = client
|
||||
.get(&url)
|
||||
@@ -531,12 +528,9 @@ async fn do_replicated_request(
|
||||
|
||||
let mut futures = Vec::new();
|
||||
for loc in remote_locations {
|
||||
// Defensively strip a `.grpcPort` suffix in case loc.url carries it
|
||||
// (e.g. from an older master); see lookup_volume for the same fix.
|
||||
let peer_http = to_http_address(&loc.url);
|
||||
let url = normalize_outgoing_http_url(
|
||||
&state.outgoing_http_scheme,
|
||||
&format!("{}{}?{}", peer_http, path, new_query),
|
||||
&format!("{}{}?{}", loc.url, path, new_query),
|
||||
)?;
|
||||
let client = state.http_client.clone();
|
||||
|
||||
@@ -704,15 +698,13 @@ async fn proxy_request(
|
||||
) -> Response {
|
||||
// Build target URL, adding proxied=true query param
|
||||
let path = info.path.trim_start_matches('/');
|
||||
// Defensively strip a `.grpcPort` suffix in case target.url carries it.
|
||||
let target_http = to_http_address(&target.url);
|
||||
|
||||
let raw_target = if info.original_query.is_empty() {
|
||||
format!("{}/{}?proxied=true", target_http, path)
|
||||
format!("{}/{}?proxied=true", target.url, path)
|
||||
} else {
|
||||
format!(
|
||||
"{}/{}?{}&proxied=true",
|
||||
target_http, path, info.original_query
|
||||
target.url, path, info.original_query
|
||||
)
|
||||
};
|
||||
let target_url = match normalize_outgoing_http_url(&state.outgoing_http_scheme, &raw_target) {
|
||||
@@ -779,11 +771,9 @@ fn redirect_request(info: &ProxyRequestInfo, target: &VolumeLocation, scheme: &s
|
||||
query_params.push("proxied=true".to_string());
|
||||
let query = query_params.join("&");
|
||||
|
||||
// Defensively strip a `.grpcPort` suffix in case target.url carries it.
|
||||
let target_http = to_http_address(&target.url);
|
||||
let raw_target = format!(
|
||||
"{}/{},{}?{}",
|
||||
target_http, &info.vid_str, &info.fid_str, query
|
||||
target.url, &info.vid_str, &info.fid_str, query
|
||||
);
|
||||
let location = match normalize_outgoing_http_url(scheme, &raw_target) {
|
||||
Ok(url) => url,
|
||||
@@ -1019,44 +1009,43 @@ async fn get_or_head_handler_inner(
|
||||
|
||||
if has_ec_volume && !has_volume {
|
||||
// ---- EC volume read path (always full read, no streaming) ----
|
||||
//
|
||||
// The distributed read path already does a local-first pass
|
||||
// in its Snapshot phase under the same store read lock the
|
||||
// legacy code would have taken — so calling it directly
|
||||
// serves both the "all shards local" fast case and the
|
||||
// "some intervals need peer fetch + reconstruct" general
|
||||
// case without paying for the local interval reads twice.
|
||||
match crate::server::store_ec::read_ec_shard_needle_distributed(
|
||||
&state, vid, needle_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(ec_needle)) => {
|
||||
n = ec_needle;
|
||||
}
|
||||
Ok(None) => {
|
||||
let store = state.store.read().unwrap();
|
||||
match store.find_ec_volume(vid) {
|
||||
Some(ecv) => match ecv.read_ec_shard_needle(needle_id) {
|
||||
Ok(Some(ec_needle)) => {
|
||||
n = ec_needle;
|
||||
}
|
||||
Ok(None) => {
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
|
||||
.inc();
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
Err(e) => {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
|
||||
.inc();
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_INTERNAL])
|
||||
.inc();
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("read ec error: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
},
|
||||
None => {
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
|
||||
.inc();
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
Err(e) => {
|
||||
let kind = if e.kind() == std::io::ErrorKind::NotFound {
|
||||
metrics::ERROR_GET_NOT_FOUND
|
||||
} else {
|
||||
metrics::ERROR_GET_INTERNAL
|
||||
};
|
||||
metrics::HANDLER_COUNTER.with_label_values(&[kind]).inc();
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("ec read: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
drop(store);
|
||||
|
||||
// Validate cookie (matches Go behavior after ReadEcShardNeedle)
|
||||
if n.cookie != cookie {
|
||||
@@ -3931,81 +3920,4 @@ mod tests {
|
||||
"https://volume.internal:8080/3,01637037d6?collection=photos&proxied=true"
|
||||
);
|
||||
}
|
||||
|
||||
/// Defensive coverage for peer-to-peer redirects: if a master ever returns
|
||||
/// `Location.url` already encoded in `host:httpPort.grpcPort` form,
|
||||
/// `redirect_request` must still emit a valid HTTP Location header.
|
||||
#[test]
|
||||
fn test_redirect_request_strips_grpc_port_from_target_url() {
|
||||
let info = ProxyRequestInfo {
|
||||
original_headers: HeaderMap::new(),
|
||||
original_query: String::new(),
|
||||
path: "/3,01637037d6".to_string(),
|
||||
vid_str: "3".to_string(),
|
||||
fid_str: "01637037d6".to_string(),
|
||||
};
|
||||
let target = VolumeLocation {
|
||||
url: "volume.internal:8080.18080".to_string(),
|
||||
public_url: "volume.public:8080.18080".to_string(),
|
||||
grpc_port: 18080,
|
||||
};
|
||||
|
||||
let response = redirect_request(&info, &target, "http");
|
||||
assert_eq!(response.status(), StatusCode::MOVED_PERMANENTLY);
|
||||
assert_eq!(
|
||||
response.headers().get(header::LOCATION).unwrap(),
|
||||
"http://volume.internal:8080/3,01637037d6?proxied=true"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression test for issue #9274.
|
||||
///
|
||||
/// SeaweedFS encodes a server's gRPC port by appending `.grpcPort` to
|
||||
/// the HTTP port: e.g. `master.local:9333.19333` (mirrors Go's
|
||||
/// `pb.ServerAddress`). When the volume server's `--mserver` flag is
|
||||
/// configured this way, replication writes look up the peer locations
|
||||
/// from the master over HTTP. Prior to the fix, the lookup URL became
|
||||
/// `http://host:9333.19333/dir/lookup?volumeId=...` — an invalid port
|
||||
/// that reqwest rejects with a "builder error", causing every
|
||||
/// replicated write to fail.
|
||||
#[tokio::test]
|
||||
async fn test_lookup_volume_strips_grpc_port_from_master_url() {
|
||||
use axum::{routing::get, Router};
|
||||
|
||||
let app = Router::new().route(
|
||||
"/dir/lookup",
|
||||
get(|axum::extract::Query(params): axum::extract::Query<std::collections::HashMap<String, String>>| async move {
|
||||
assert_eq!(params.get("volumeId").map(String::as_str), Some("31"));
|
||||
axum::Json(serde_json::json!({
|
||||
"volumeOrFileId": "31",
|
||||
"locations": [
|
||||
{"url": "10.0.0.2:5301", "publicUrl": "10.0.0.2:5301", "grpcPort": 5311}
|
||||
]
|
||||
}))
|
||||
}),
|
||||
);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
// Mirrors how main.rs receives masters: a `host:httpPort.grpcPort` token.
|
||||
// The grpc-port part is fictitious here (the master is HTTP-only in
|
||||
// this test) — what matters is that the volume server must not pass
|
||||
// it through to reqwest.
|
||||
let master_url = format!("127.0.0.1:{}.{}", addr.port(), 19999);
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let locations = super::lookup_volume(&client, "http", &master_url, 31)
|
||||
.await
|
||||
.expect("lookup_volume should succeed even when master_url carries a .grpcPort suffix");
|
||||
|
||||
assert_eq!(locations.len(), 1);
|
||||
assert_eq!(locations[0].url, "10.0.0.2:5301");
|
||||
assert_eq!(locations[0].grpc_port, 5311);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,20 +169,10 @@ pub async fn run_heartbeat_with_state(
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a master address to a gRPC `host:port` target.
|
||||
///
|
||||
/// Mirrors Go's `pb.ServerToGrpcAddress()`:
|
||||
/// - `host:port.grpcPort` returns `host:grpcPort` (explicit gRPC port).
|
||||
/// - `host:port` returns `host:(port+10000)` (Go's default offset).
|
||||
/// - Anything that fails to parse is returned unchanged.
|
||||
/// Convert a master address "host:port" to a gRPC host:port target.
|
||||
/// The Go master uses port + 10000 for gRPC by default.
|
||||
pub fn to_grpc_address(master_addr: &str) -> String {
|
||||
if let Some((host, port_str)) = master_addr.rsplit_once(':') {
|
||||
// "host:port.grpcPort" — the part after the last '.' is the gRPC port.
|
||||
if let Some((_, grpc_port)) = port_str.rsplit_once('.') {
|
||||
if grpc_port.parse::<u16>().is_ok() {
|
||||
return format!("{}:{}", host, grpc_port);
|
||||
}
|
||||
}
|
||||
if let Ok(port) = port_str.parse::<u16>() {
|
||||
let grpc_port = port + 10000;
|
||||
return format!("{}:{}", host, grpc_port);
|
||||
@@ -344,12 +334,6 @@ fn diff_ec_shard_delta_messages(
|
||||
if !current.contains_key(key) {
|
||||
let mut deleted = message.clone();
|
||||
deleted.shard_sizes = vec![0];
|
||||
tracing::info!(
|
||||
volume_id = deleted.id,
|
||||
disk_id = deleted.disk_id,
|
||||
ec_index_bits = deleted.ec_index_bits,
|
||||
"deletes ec shards"
|
||||
);
|
||||
deleted_ec_shards.push(deleted);
|
||||
}
|
||||
}
|
||||
@@ -403,15 +387,6 @@ async fn do_heartbeat(
|
||||
let mut response_stream = client.send_heartbeat(stream).await?.into_inner();
|
||||
|
||||
info!("Heartbeat stream established with {}", grpc_addr);
|
||||
// Publish the master we're now talking to in canonical http host:port
|
||||
// form so Ping admission can recognise it once a leader change moves us
|
||||
// off the seed list. Mirrors Go's vs.setCurrentMaster(masterAddress).
|
||||
{
|
||||
let normalised =
|
||||
super::volume_server::to_http_address(current_master).into_owned();
|
||||
let mut guard = state.current_master_url.write().await;
|
||||
*guard = normalised;
|
||||
}
|
||||
if is_stopping(state) {
|
||||
state.is_heartbeating.store(false, Ordering::Relaxed);
|
||||
send_deregister_heartbeat(config, state, &tx).await;
|
||||
@@ -875,7 +850,7 @@ fn build_heartbeat_with_ec_status(
|
||||
}
|
||||
|
||||
for vid in delete_vids {
|
||||
let _ = loc.delete_volume(vid, false, false);
|
||||
let _ = loc.delete_volume(vid, false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1048,8 +1023,6 @@ mod tests {
|
||||
read_mode: ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -1065,26 +1038,6 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_grpc_address_default_offset() {
|
||||
assert_eq!(to_grpc_address("10.0.0.1:9333"), "10.0.0.1:19333");
|
||||
assert_eq!(to_grpc_address("localhost:9333"), "localhost:19333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_grpc_address_explicit_grpc_port() {
|
||||
// host:port.grpcPort form — gRPC port is what's after the dot.
|
||||
assert_eq!(to_grpc_address("10.85.183.6:5300.6300"), "10.85.183.6:6300");
|
||||
assert_eq!(to_grpc_address("master.local:9333.19333"), "master.local:19333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_grpc_address_returns_input_when_unparseable() {
|
||||
assert_eq!(to_grpc_address(""), "");
|
||||
assert_eq!(to_grpc_address("no-port"), "no-port");
|
||||
assert_eq!(to_grpc_address("host:not-a-port"), "host:not-a-port");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_heartbeat_includes_store_identity_and_disk_metadata() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
@@ -1258,7 +1211,7 @@ mod tests {
|
||||
let shard_path = format!("{}/ec_metrics_case_27.ec00", dir);
|
||||
std::fs::write(&shard_path, b"ec-shard").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0])
|
||||
.unwrap();
|
||||
|
||||
let state = test_state_with_store(store);
|
||||
@@ -1300,7 +1253,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/expired_heartbeat_ec_31.ec00", dir), b"expired").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0], "")
|
||||
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0])
|
||||
.unwrap();
|
||||
store
|
||||
.find_ec_volume_mut(VolumeId(31))
|
||||
@@ -1455,8 +1408,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_registers_s3_default_aliases() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently. Just register our entries and verify them.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1488,8 +1443,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_ignores_unsupported_types() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1500,17 +1457,12 @@ mod tests {
|
||||
}],
|
||||
);
|
||||
|
||||
// The per-state registry is freshly created and should have no entries
|
||||
// since "rclone" is unsupported.
|
||||
let registry = state.s3_tier_registry.read().unwrap();
|
||||
assert!(registry.names().is_empty());
|
||||
// Only check that the unsupported type was not added to the global
|
||||
// registry. Other tests may have their own entries present.
|
||||
let global_registry = crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.read()
|
||||
.unwrap();
|
||||
assert!(global_registry.get("rclone.default").is_none());
|
||||
assert!(global_registry.get("rclone").is_none());
|
||||
assert!(global_registry.names().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1603,7 +1555,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/ec_delta_case_81.ec00", dir), b"delta").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0])
|
||||
.unwrap();
|
||||
let current = collect_ec_shard_delta_messages(&store);
|
||||
let (new_ec_shards, deleted_ec_shards) =
|
||||
|
||||
@@ -1,15 +1,12 @@
|
||||
#[cfg(unix)]
|
||||
pub mod debug;
|
||||
pub mod grpc_client;
|
||||
pub mod grpc_server;
|
||||
pub mod handlers;
|
||||
pub mod heartbeat;
|
||||
pub mod memory_status;
|
||||
#[cfg(unix)]
|
||||
pub mod profiling;
|
||||
pub mod request_id;
|
||||
pub mod server_stats;
|
||||
pub mod store_ec;
|
||||
pub mod ui;
|
||||
pub mod volume_server;
|
||||
pub mod write_queue;
|
||||
|
||||
@@ -1,678 +0,0 @@
|
||||
//! Distributed EC read path. Mirror of `weed/storage/store_ec.go`'s
|
||||
//! `readEcShardIntervals` → `readOneEcShardInterval` →
|
||||
//! `readRemoteEcShardInterval` → `recoverOneRemoteEcShardInterval`
|
||||
//! chain.
|
||||
//!
|
||||
//! The existing `EcVolume::read_ec_shard_needle` reads only locally-
|
||||
//! mounted shards and returns `NotFound` if any interval requires a
|
||||
//! shard held on a peer server. In a standard RS(10,4)-across-14
|
||||
//! deployment each server holds one shard, so every read needs >=9
|
||||
//! peer fetches. This module fills the gap by:
|
||||
//!
|
||||
//! 1. Locating the needle in `.ecx` (under the Store read lock) and
|
||||
//! computing the per-interval (shard_id, shard_offset, size).
|
||||
//! 2. Reading the local-resident intervals while still holding the
|
||||
//! lock — same path the local-only helper uses.
|
||||
//! 3. Dropping the lock and, for any remaining intervals, fetching
|
||||
//! from peer volume servers via `VolumeEcShardRead`. If the
|
||||
//! direct peer read fails, fan-out reads to other shards at the
|
||||
//! same (shard_offset, size) and rebuild the missing shard via
|
||||
//! Reed-Solomon — exactly Go's flow.
|
||||
//! 4. Refreshing the per-EcVolume `shard_locations` cache from the
|
||||
//! master's `LookupEcVolume` RPC when the cached map is stale.
|
||||
//!
|
||||
//! All gRPC IO is async; the file IO portion runs under the sync
|
||||
//! Store read lock, matching Go's `readLocalEcShardInterval`. The
|
||||
//! cache write-back briefly reacquires the EcVolume's internal
|
||||
//! `RwLock` so we do not contend with the Store-level lock at all.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures::future::join_all;
|
||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||
use tonic::Request;
|
||||
|
||||
use crate::pb::master_pb::{self, seaweed_client::SeaweedClient, LookupEcVolumeRequest};
|
||||
use crate::pb::volume_server_pb::{
|
||||
volume_server_client::VolumeServerClient, VolumeEcShardReadRequest,
|
||||
};
|
||||
use crate::server::grpc_client::{build_grpc_endpoint, parse_grpc_address, GRPC_MAX_MESSAGE_SIZE};
|
||||
use crate::server::request_id::outgoing_request_id_interceptor;
|
||||
use crate::server::volume_server::VolumeServerState;
|
||||
use crate::storage::erasure_coding::ec_shard::ShardId;
|
||||
use crate::storage::needle::needle::{get_actual_size, Needle};
|
||||
use crate::storage::types::*;
|
||||
|
||||
/// One interval's data after Phase A.
|
||||
enum IntervalResult {
|
||||
/// Already read from a locally-mounted shard.
|
||||
Local(Vec<u8>),
|
||||
/// Shard not local (or local read failed). Must be fetched.
|
||||
NeedRemote {
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
},
|
||||
}
|
||||
|
||||
/// Snapshot extracted under the Store read lock so Phases B/C can run
|
||||
/// without holding any sync lock across `.await`.
|
||||
struct Snapshot {
|
||||
data_shards: u32,
|
||||
parity_shards: u32,
|
||||
version: Version,
|
||||
actual_size: usize,
|
||||
offset: Offset,
|
||||
size_for_parse: Size,
|
||||
intervals: Vec<IntervalResult>,
|
||||
cached_locations: HashMap<ShardId, Vec<String>>,
|
||||
cache_refreshed_at: Option<Instant>,
|
||||
}
|
||||
|
||||
/// Top-level entry point. Returns `Ok(None)` for "not found" (matches
|
||||
/// Go's `ReadEcShardNeedle`); errors propagate as `io::Error`.
|
||||
pub async fn read_ec_shard_needle_distributed(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<Option<Needle>> {
|
||||
// Phase A — under the Store read lock, locate the needle, compute
|
||||
// intervals, and read any locally-mounted shard intervals. We must
|
||||
// not `.await` while holding this guard (std::sync::RwLockReadGuard
|
||||
// is !Send).
|
||||
let snapshot = match snapshot_under_lock(state, vid, needle_id)? {
|
||||
Some(s) => s,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
// Phase B — refresh the shard_locations cache from the master if
|
||||
// it is stale. Do this lazily: if every needed interval was read
|
||||
// locally we can skip the master RPC entirely.
|
||||
let any_remote = snapshot
|
||||
.intervals
|
||||
.iter()
|
||||
.any(|r| matches!(r, IntervalResult::NeedRemote { .. }));
|
||||
let total_shards = (snapshot.data_shards + snapshot.parity_shards) as usize;
|
||||
|
||||
let mut shard_locations = snapshot.cached_locations.clone();
|
||||
if any_remote
|
||||
&& needs_refresh(
|
||||
&shard_locations,
|
||||
snapshot.cache_refreshed_at,
|
||||
snapshot.data_shards as usize,
|
||||
total_shards,
|
||||
)
|
||||
{
|
||||
match cached_lookup_ec_shard_locations(state, vid).await {
|
||||
Ok(fresh) => {
|
||||
shard_locations = fresh.clone();
|
||||
write_back_shard_locations(state, vid, fresh);
|
||||
}
|
||||
Err(e) => {
|
||||
// Lookup failed — proceed with cached values. If cache
|
||||
// is empty, the remote fetch below will fail and we
|
||||
// surface a NotFound (matching Go's behavior when no
|
||||
// locations are known).
|
||||
tracing::warn!(
|
||||
"ec lookup failed for volume {}: {} — using cached locations ({} entries)",
|
||||
vid.0,
|
||||
e,
|
||||
shard_locations.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase C — fetch missing intervals, reconstructing when the
|
||||
// direct peer read fails.
|
||||
let mut assembled: Vec<Vec<u8>> = Vec::with_capacity(snapshot.intervals.len());
|
||||
for res in snapshot.intervals {
|
||||
match res {
|
||||
IntervalResult::Local(buf) => assembled.push(buf),
|
||||
IntervalResult::NeedRemote {
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
} => {
|
||||
let buf = fetch_one_interval(
|
||||
state,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
&shard_locations,
|
||||
snapshot.data_shards as usize,
|
||||
snapshot.parity_shards as usize,
|
||||
)
|
||||
.await?;
|
||||
assembled.push(buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase D — assemble and parse the Needle. Mirrors the tail of
|
||||
// `EcVolume::read_ec_shard_needle`.
|
||||
let mut bytes = Vec::with_capacity(snapshot.actual_size);
|
||||
for chunk in assembled {
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
bytes.truncate(snapshot.actual_size);
|
||||
if bytes.len() < snapshot.actual_size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
format!(
|
||||
"read {} bytes but need {} for needle {}",
|
||||
bytes.len(),
|
||||
snapshot.actual_size,
|
||||
needle_id
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
let mut n = Needle::default();
|
||||
n.id = needle_id;
|
||||
n.read_bytes(
|
||||
&bytes,
|
||||
snapshot.offset.to_actual_offset(),
|
||||
snapshot.size_for_parse,
|
||||
snapshot.version,
|
||||
)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
|
||||
Ok(Some(n))
|
||||
}
|
||||
|
||||
fn snapshot_under_lock(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<Option<Snapshot>> {
|
||||
let store = state.store.read().unwrap();
|
||||
let ecv = match store.find_ec_volume(vid) {
|
||||
Some(v) => v,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
// Reuse EcVolume::locate_needle for offset/size resolution AND
|
||||
// the per-needle shard-interval math — it's the same routine the
|
||||
// local-only read path uses, so we stay byte-identical on the
|
||||
// shard-size + interval boundaries.
|
||||
let (offset, size, intervals) = match ecv.locate_needle(needle_id)? {
|
||||
Some(v) => v,
|
||||
None => return Ok(None),
|
||||
};
|
||||
if intervals.is_empty() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"no intervals for needle",
|
||||
));
|
||||
}
|
||||
let actual = get_actual_size(size, ecv.version);
|
||||
|
||||
// Phase A.local: for each interval read the local shard if we
|
||||
// hold it; otherwise fall through to remote. We accumulate the
|
||||
// results in interval order so the assembly step is just a
|
||||
// concat.
|
||||
let mut interval_results = Vec::with_capacity(intervals.len());
|
||||
for interval in &intervals {
|
||||
let (shard_id, shard_offset) = interval.to_shard_id_and_offset(ecv.data_shards);
|
||||
let buf_size = interval.size as usize;
|
||||
let local = ecv
|
||||
.shards
|
||||
.get(shard_id as usize)
|
||||
.and_then(|s| s.as_ref());
|
||||
match local {
|
||||
Some(shard) => {
|
||||
let mut buf = vec![0u8; buf_size];
|
||||
match shard.read_at(&mut buf, shard_offset as u64) {
|
||||
Ok(n) if n == buf_size => {
|
||||
interval_results.push(IntervalResult::Local(buf));
|
||||
}
|
||||
_ => interval_results.push(IntervalResult::NeedRemote {
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size: buf_size,
|
||||
}),
|
||||
}
|
||||
}
|
||||
None => interval_results.push(IntervalResult::NeedRemote {
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size: buf_size,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
let cached_locations = ecv.shard_locations.read().unwrap().clone();
|
||||
let cache_refreshed_at = *ecv.shard_locations_refresh_time.lock().unwrap();
|
||||
|
||||
Ok(Some(Snapshot {
|
||||
data_shards: ecv.data_shards,
|
||||
parity_shards: ecv.parity_shards,
|
||||
version: ecv.version,
|
||||
actual_size: actual as usize,
|
||||
offset,
|
||||
size_for_parse: size,
|
||||
intervals: interval_results,
|
||||
cached_locations,
|
||||
cache_refreshed_at,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Master `LookupEcVolume` freshness rules — match Go's
|
||||
/// `cachedLookupEcShardLocations` thresholds in store_ec.go.
|
||||
fn needs_refresh(
|
||||
locations: &HashMap<ShardId, Vec<String>>,
|
||||
refreshed_at: Option<Instant>,
|
||||
data_shards: usize,
|
||||
total_shards: usize,
|
||||
) -> bool {
|
||||
let now = Instant::now();
|
||||
let age = match refreshed_at {
|
||||
Some(t) => now.saturating_duration_since(t),
|
||||
None => return true,
|
||||
};
|
||||
let shard_count = locations.len();
|
||||
if shard_count < data_shards && age < Duration::from_secs(11) {
|
||||
return false;
|
||||
}
|
||||
if shard_count == total_shards && age < Duration::from_secs(37 * 60) {
|
||||
return false;
|
||||
}
|
||||
if shard_count >= data_shards && age < Duration::from_secs(7 * 60) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn cached_lookup_ec_shard_locations(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<HashMap<ShardId, Vec<String>>> {
|
||||
let master = {
|
||||
let live = state.current_master_url.read().await.clone();
|
||||
if !live.is_empty() {
|
||||
live
|
||||
} else {
|
||||
state.master_url.clone()
|
||||
}
|
||||
};
|
||||
if master.is_empty() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"no master configured for ec shard lookup",
|
||||
));
|
||||
}
|
||||
|
||||
let grpc_addr = parse_grpc_address(&master)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
|
||||
let endpoint = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref())
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
|
||||
let channel = endpoint
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.timeout(Duration::from_secs(10))
|
||||
.connect()
|
||||
.await
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("master connect: {}", e)))?;
|
||||
|
||||
let mut client = SeaweedClient::with_interceptor(channel, outgoing_request_id_interceptor)
|
||||
.max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE);
|
||||
|
||||
let resp = client
|
||||
.lookup_ec_volume(Request::new(LookupEcVolumeRequest { volume_id: vid.0 }))
|
||||
.await
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("lookup_ec_volume: {}", e)))?;
|
||||
let resp = resp.into_inner();
|
||||
|
||||
let mut out = HashMap::new();
|
||||
for entry in resp.shard_id_locations {
|
||||
let addrs: Vec<String> = entry
|
||||
.locations
|
||||
.iter()
|
||||
.map(format_location_as_server_address)
|
||||
.collect();
|
||||
out.insert(entry.shard_id as ShardId, addrs);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn write_back_shard_locations(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
locations: HashMap<ShardId, Vec<String>>,
|
||||
) {
|
||||
let store = state.store.read().unwrap();
|
||||
if let Some(ecv) = store.find_ec_volume(vid) {
|
||||
// Atomic swap + freshness stamp so a concurrent reader sees
|
||||
// either the prior cache or the fresh one — never an
|
||||
// intermediate half-replaced map with the freshness flag
|
||||
// already flipped.
|
||||
ecv.replace_shard_locations(locations);
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a SeaweedFS-style `host:httpPort.grpcPort` address from a
|
||||
/// master `Location` so the result is what `parse_grpc_address` (and
|
||||
/// the heartbeat path) already understand.
|
||||
fn format_location_as_server_address(loc: &master_pb::Location) -> String {
|
||||
let raw = loc
|
||||
.url
|
||||
.trim_start_matches("http://")
|
||||
.trim_start_matches("https://");
|
||||
if loc.grpc_port > 0 {
|
||||
if let Some((host, http_port)) = raw.rsplit_once(':') {
|
||||
return format!("{}:{}.{}", host, http_port, loc.grpc_port);
|
||||
}
|
||||
}
|
||||
raw.to_string()
|
||||
}
|
||||
|
||||
/// Try direct peer read; on failure, reconstruct via Reed-Solomon
|
||||
/// from the other shards. Mirrors `readOneEcShardInterval`'s tail.
|
||||
async fn fetch_one_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
shard_locations: &HashMap<ShardId, Vec<String>>,
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
// Direct peer read against the cached locations for this shard.
|
||||
if let Some(sources) = shard_locations.get(&shard_id) {
|
||||
if !sources.is_empty() {
|
||||
match read_remote_ec_shard_interval(
|
||||
state,
|
||||
sources,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(buf) => return Ok(buf),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
"direct read ec shard {}.{} from {:?} failed: {} — will reconstruct",
|
||||
vid.0,
|
||||
shard_id,
|
||||
sources,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reconstruct: fan-out reads to every other shard at the same
|
||||
// (shard_offset, size). Mirrors `recoverOneRemoteEcShardInterval`.
|
||||
recover_one_remote_ec_shard_interval(
|
||||
state,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
shard_locations,
|
||||
data_shards,
|
||||
parity_shards,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_remote_ec_shard_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
sources: &[String],
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let mut last_err: Option<io::Error> = None;
|
||||
for src in sources {
|
||||
match do_read_remote_ec_shard_interval(
|
||||
state,
|
||||
src,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(buf) => return Ok(buf),
|
||||
Err(e) => last_err = Some(e),
|
||||
}
|
||||
}
|
||||
Err(last_err.unwrap_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
format!("no source for ec shard {}.{}", vid.0, shard_id),
|
||||
)
|
||||
}))
|
||||
}
|
||||
|
||||
async fn do_read_remote_ec_shard_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
source: &str,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let grpc_addr =
|
||||
parse_grpc_address(source).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
|
||||
let endpoint = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref())
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
|
||||
let channel = endpoint
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.connect()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("connect to {}: {}", source, e),
|
||||
)
|
||||
})?;
|
||||
|
||||
// TODO(grpc-jwt): clusters with `jwt.signing.key` configured will
|
||||
// reject peer-to-peer VolumeEcShardRead calls until the Rust
|
||||
// crate grows an outgoing-JWT interceptor. The gap is shared
|
||||
// with every other peer gRPC call from this binary
|
||||
// (`copy_file_from_source`, `batch_delete`, …) — handling it
|
||||
// here in isolation would split the credential plumbing across
|
||||
// call sites. Re-visit when outgoing JWT signing lands as a
|
||||
// server-wide helper.
|
||||
let mut client = VolumeServerClient::with_interceptor(channel, outgoing_request_id_interceptor)
|
||||
.max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE);
|
||||
|
||||
let req = VolumeEcShardReadRequest {
|
||||
volume_id: vid.0,
|
||||
shard_id: shard_id as u32,
|
||||
offset: shard_offset,
|
||||
size: size as i64,
|
||||
file_key: needle_id.0,
|
||||
};
|
||||
let resp = client
|
||||
.volume_ec_shard_read(Request::new(req))
|
||||
.await
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("volume_ec_shard_read {}.{} from {}: {}", vid.0, shard_id, source, e),
|
||||
)
|
||||
})?;
|
||||
let mut stream = resp.into_inner();
|
||||
|
||||
let mut out = Vec::with_capacity(size);
|
||||
while let Some(msg) = stream
|
||||
.message()
|
||||
.await
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("recv: {}", e)))?
|
||||
{
|
||||
if !msg.data.is_empty() {
|
||||
out.extend_from_slice(&msg.data);
|
||||
}
|
||||
}
|
||||
if out.len() < size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
format!(
|
||||
"short read from {} for ec shard {}.{}: got {} want {}",
|
||||
source,
|
||||
vid.0,
|
||||
shard_id,
|
||||
out.len(),
|
||||
size
|
||||
),
|
||||
));
|
||||
}
|
||||
out.truncate(size);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn recover_one_remote_ec_shard_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id_to_recover: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
shard_locations: &HashMap<ShardId, Vec<String>>,
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let total_shards = data_shards + parity_shards;
|
||||
let rs = ReedSolomon::new(data_shards, parity_shards).map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("reed-solomon init: {:?}", e),
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut bufs: Vec<Option<Vec<u8>>> = vec![None; total_shards];
|
||||
|
||||
// Phase 0: seed bufs from LOCALLY mounted shards. If this node
|
||||
// already holds enough sibling shards, reconstruction completes
|
||||
// without any peer fan-out — and even with a cold/incomplete
|
||||
// shard_locations cache or a failed master lookup, local
|
||||
// survivors still contribute. Mirrors Go's
|
||||
// recoverOneRemoteEcShardInterval behaviour, which is implicitly
|
||||
// local-aware because the Store fan-out targets ALL known
|
||||
// locations (including the caller's own server address); the
|
||||
// Rust port had been remote-only, so reconstructing with a cold
|
||||
// cache failed even when enough siblings were on disk.
|
||||
{
|
||||
let store = state.store.read().unwrap();
|
||||
if let Some(ecv) = store.find_ec_volume(vid) {
|
||||
for sid in 0..total_shards {
|
||||
if sid as ShardId == shard_id_to_recover {
|
||||
continue;
|
||||
}
|
||||
if let Some(Some(shard)) = ecv.shards.get(sid) {
|
||||
let mut buf = vec![0u8; size];
|
||||
if shard.read_at(&mut buf, shard_offset as u64).map(|n| n == size).unwrap_or(false) {
|
||||
bufs[sid] = Some(buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 1: remote fan-out — one task per known shard location
|
||||
// we DON'T already have locally and DON'T need to recover.
|
||||
let mut tasks = Vec::new();
|
||||
for (sid, locs) in shard_locations {
|
||||
if *sid == shard_id_to_recover || locs.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if bufs[*sid as usize].is_some() {
|
||||
continue;
|
||||
}
|
||||
let sid = *sid;
|
||||
let locs = locs.clone();
|
||||
let state = state.clone();
|
||||
tasks.push(async move {
|
||||
let res = read_remote_ec_shard_interval(
|
||||
&state,
|
||||
&locs,
|
||||
vid,
|
||||
needle_id,
|
||||
sid,
|
||||
shard_offset,
|
||||
size,
|
||||
)
|
||||
.await;
|
||||
(sid, res)
|
||||
});
|
||||
}
|
||||
let results = join_all(tasks).await;
|
||||
|
||||
for (sid, res) in results {
|
||||
match res {
|
||||
Ok(buf) => {
|
||||
if (sid as usize) < total_shards {
|
||||
bufs[sid as usize] = Some(buf);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
"recover: read {}.{} for needle {} failed: {}",
|
||||
vid.0,
|
||||
sid,
|
||||
needle_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let available = bufs.iter().filter(|b| b.is_some()).count();
|
||||
if available < data_shards {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"cannot recover ec shard {}.{}: only {} shards available, need at least {}",
|
||||
vid.0, shard_id_to_recover, available, data_shards
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
rs.reconstruct(&mut bufs).map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"reed-solomon reconstruct ec shard {}.{}: {:?}",
|
||||
vid.0, shard_id_to_recover, e
|
||||
),
|
||||
)
|
||||
})?;
|
||||
|
||||
match bufs.into_iter().nth(shard_id_to_recover as usize).flatten() {
|
||||
Some(buf) => Ok(buf),
|
||||
None => Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"reconstructed buffer for shard {}.{} missing after RS reconstruct",
|
||||
vid.0, shard_id_to_recover
|
||||
),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// parse_grpc_address lives in `grpc_client.rs` and is re-exported
|
||||
// here via the use above so this module shares a single
|
||||
// HTTP↔gRPC port-translation routine with grpc_server.rs.
|
||||
@@ -81,16 +81,6 @@ pub struct VolumeServerState {
|
||||
pub master_url: String,
|
||||
/// Seed master addresses for UI rendering.
|
||||
pub master_urls: Vec<String>,
|
||||
/// Canonical http `host:port` form of every configured seed master.
|
||||
/// Built once at construction so Ping admission stays O(1). Mirrors
|
||||
/// Go's `seedMasterSet` on `VolumeServer`.
|
||||
pub seed_master_set: std::collections::HashSet<String>,
|
||||
/// Current master this server is heartbeating with, in canonical http
|
||||
/// `host:port` form. Empty when no heartbeat connection is active. The
|
||||
/// heartbeat goroutine writes; admission reads — the lock keeps them
|
||||
/// from racing on a leader change. Mirrors Go's `currentMaster` plus
|
||||
/// `currentMasterLock`.
|
||||
pub current_master_url: tokio::sync::RwLock<String>,
|
||||
/// This server's own address (ip:port) for filtering self from lookup results.
|
||||
pub self_url: String,
|
||||
/// HTTP client for proxy requests and master lookups.
|
||||
@@ -127,35 +117,6 @@ impl VolumeServerState {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build the seed master set from a list of raw `host:port[.grpcPort]`
|
||||
/// addresses, normalised the same way Go's `pb.ServerAddress.ToHttpAddress`
|
||||
/// does (drop the `.grpcPort` suffix, preserve everything else).
|
||||
pub fn build_seed_master_set(master_urls: &[String]) -> std::collections::HashSet<String> {
|
||||
master_urls
|
||||
.iter()
|
||||
.map(|m| to_http_address(m).into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns true iff `target` (normalised to canonical http `host:port`)
|
||||
/// is a master this server already knows about. Volume servers do not
|
||||
/// keep a peer-volume or peer-filer list, so Ping is scoped to masters.
|
||||
/// Mirrors Go's `VolumeServer.isKnownPingTarget`.
|
||||
pub async fn is_known_ping_target(&self, target: &str, target_type: &str) -> bool {
|
||||
if target_type != "master" {
|
||||
return false;
|
||||
}
|
||||
let key = to_http_address(target).into_owned();
|
||||
if key.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let current = self.current_master_url.read().await.clone();
|
||||
if !current.is_empty() && current == key {
|
||||
return true;
|
||||
}
|
||||
self.seed_master_set.contains(&key)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_metrics_router() -> Router {
|
||||
@@ -173,38 +134,6 @@ pub fn normalize_outgoing_http_url(scheme: &str, raw_target: &str) -> Result<Str
|
||||
Ok(format!("{}://{}", scheme, raw_target))
|
||||
}
|
||||
|
||||
/// Convert a SeaweedFS server address to its HTTP `host:port` form.
|
||||
///
|
||||
/// Mirrors Go's `pb.ServerAddress.ToHttpAddress()`: SeaweedFS encodes a
|
||||
/// server's gRPC port by appending `.grpcPort` to the HTTP port (e.g.
|
||||
/// `host:9333.19333`). For HTTP requests we want only the HTTP `host:port`.
|
||||
/// - `host:port.grpcPort` -> `host:port`
|
||||
/// - `host:port` -> `host:port` (unchanged)
|
||||
/// - Anything that does not look like `host:port[.grpcPort]` is returned unchanged.
|
||||
///
|
||||
/// Returns a `Cow<str>` so the common (no-suffix) case borrows from `addr`
|
||||
/// without allocating; only the rewrite branch produces a new `String`.
|
||||
pub fn to_http_address(addr: &str) -> std::borrow::Cow<'_, str> {
|
||||
let Some(ports_sep_index) = addr.rfind(':') else {
|
||||
return std::borrow::Cow::Borrowed(addr);
|
||||
};
|
||||
let ports = &addr[ports_sep_index + 1..];
|
||||
if let Some(dot_idx) = ports.rfind('.') {
|
||||
let http_port = &ports[..dot_idx];
|
||||
let grpc_port = &ports[dot_idx + 1..];
|
||||
// Only strip the suffix when both parts parse as real ports — leave
|
||||
// anything else (e.g. "host:abc.def") untouched so bad config surfaces
|
||||
// rather than being silently rewritten. Mirrors the validation already
|
||||
// done in `to_grpc_address` for the inverse direction.
|
||||
if let (Ok(_), Ok(_)) = (http_port.parse::<u16>(), grpc_port.parse::<u16>()) {
|
||||
return std::borrow::Cow::Owned(
|
||||
addr[..ports_sep_index + 1 + dot_idx].to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
std::borrow::Cow::Borrowed(addr)
|
||||
}
|
||||
|
||||
fn request_remote_addr(request: &Request) -> Option<SocketAddr> {
|
||||
request
|
||||
.extensions()
|
||||
@@ -463,78 +392,3 @@ pub fn build_public_router(state: Arc<VolumeServerState>) -> Router {
|
||||
.layer(middleware::from_fn(common_headers_middleware))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_strips_grpc_port_suffix() {
|
||||
assert_eq!(to_http_address("10.0.0.1:9333.19333"), "10.0.0.1:9333");
|
||||
assert_eq!(
|
||||
to_http_address("master.local:9333.19333"),
|
||||
"master.local:9333"
|
||||
);
|
||||
assert_eq!(to_http_address("10.85.183.6:5300.6300"), "10.85.183.6:5300");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_passthrough_without_grpc_suffix() {
|
||||
assert_eq!(to_http_address("10.0.0.1:9333"), "10.0.0.1:9333");
|
||||
assert_eq!(to_http_address("master.local:9333"), "master.local:9333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_returns_input_when_unparseable() {
|
||||
assert_eq!(to_http_address(""), "");
|
||||
assert_eq!(to_http_address("no-port"), "no-port");
|
||||
// Trailing colon: nothing after the separator, treat as unparseable.
|
||||
assert_eq!(to_http_address("host:"), "host:");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_borrows_when_unchanged_and_owns_when_stripped() {
|
||||
// The common case (no suffix) must not allocate.
|
||||
let result = to_http_address("10.0.0.1:9333");
|
||||
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
|
||||
// Stripping requires a new string.
|
||||
let result = to_http_address("10.0.0.1:9333.19333");
|
||||
assert!(matches!(result, std::borrow::Cow::Owned(_)));
|
||||
assert_eq!(result, "10.0.0.1:9333");
|
||||
// Unparseable / passthrough also borrows.
|
||||
let result = to_http_address("host:abc.def");
|
||||
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_keeps_non_numeric_dotted_suffix() {
|
||||
// The dotted form is only valid when both sides are real port numbers.
|
||||
// Otherwise the address is malformed config (e.g. a hostname like
|
||||
// "host:abc.def"), and silently rewriting it would just hide the bug.
|
||||
assert_eq!(to_http_address("host:abc.def"), "host:abc.def");
|
||||
assert_eq!(to_http_address("host:9333.notaport"), "host:9333.notaport");
|
||||
assert_eq!(to_http_address("host:notaport.19333"), "host:notaport.19333");
|
||||
// Out-of-range ports must not be silently truncated either.
|
||||
assert_eq!(to_http_address("host:99999.19333"), "host:99999.19333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_handles_bracketed_ipv6_literals() {
|
||||
// The function uses `rfind(':')`, so for bracketed IPv6 the port
|
||||
// separator is correctly identified as the colon AFTER the closing
|
||||
// bracket — making IPv4 and IPv6 behave the same.
|
||||
assert_eq!(to_http_address("[::1]:9333.19333"), "[::1]:9333");
|
||||
assert_eq!(
|
||||
to_http_address("[2001:db8::10]:5300.6300"),
|
||||
"[2001:db8::10]:5300"
|
||||
);
|
||||
// Plain bracketed IPv6 without a dotted suffix is borrowed unchanged.
|
||||
let result = to_http_address("[2001:db8::1]:9333");
|
||||
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
|
||||
assert_eq!(result, "[2001:db8::1]:9333");
|
||||
// Non-numeric / out-of-range / missing suffix all preserve the input.
|
||||
assert_eq!(to_http_address("[::1]:"), "[::1]:");
|
||||
assert_eq!(to_http_address("[::1]:abc.def"), "[::1]:abc.def");
|
||||
assert_eq!(to_http_address("[::1]:99999.19333"), "[::1]:99999.19333");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,8 +207,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
|
||||
@@ -202,15 +202,6 @@ impl DiskLocation {
|
||||
}
|
||||
}
|
||||
|
||||
// After regular volumes, auto-discover EC shards on disk so a
|
||||
// fresh restart picks up shards without an explicit
|
||||
// VolumeEcShardsMount RPC. Mirrors Go's loadExistingVolumes
|
||||
// calling loadAllEcShards. Failures here only log — they
|
||||
// shouldn't fail the whole disk's startup.
|
||||
if let Err(e) = self.load_all_ec_shards() {
|
||||
warn!(directory = %self.directory, error = %e, "load_all_ec_shards failed");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -304,11 +295,7 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
/// Remove all EC-related files for a volume.
|
||||
/// `pub(crate)` so the Store-level cross-disk passes in
|
||||
/// `store_ec_reconcile.rs` can call it to scrub partial EC artefacts
|
||||
/// when a healthy `.dat` for the same vid lives on a sibling disk
|
||||
/// (seaweedfs/seaweedfs#9478).
|
||||
pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
|
||||
fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
const MAX_SHARD_COUNT: usize = 32;
|
||||
@@ -390,20 +377,13 @@ impl DiskLocation {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove, close, and delete all files for a volume. When keep_remote_data
|
||||
/// is true the cloud-tier object backing the volume is left intact — used
|
||||
/// by moves where another server is taking over the same .vif.
|
||||
pub fn delete_volume(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
only_empty: bool,
|
||||
keep_remote_data: bool,
|
||||
) -> Result<(), VolumeError> {
|
||||
/// Remove, close, and delete all files for a volume.
|
||||
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
|
||||
if let Some(mut v) = self.volumes.remove(&vid) {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
v.destroy(only_empty, keep_remote_data)?;
|
||||
v.destroy(only_empty)?;
|
||||
Ok(())
|
||||
} else {
|
||||
Err(VolumeError::NotFound)
|
||||
@@ -424,7 +404,7 @@ impl DiskLocation {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
if let Err(e) = v.destroy(false, false) {
|
||||
if let Err(e) = v.destroy(false) {
|
||||
warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
|
||||
}
|
||||
}
|
||||
@@ -564,114 +544,28 @@ impl DiskLocation {
|
||||
self.ec_volumes.contains_key(&vid)
|
||||
}
|
||||
|
||||
/// Reports whether this disk has a sealed `.ecx` index file for the
|
||||
/// given (collection, vid). Unlike [`Self::has_ec_volume`] this does
|
||||
/// not require the EC volume to be mounted in memory, which makes it
|
||||
/// the right primitive for placement decisions during `ec.balance` /
|
||||
/// `ec.rebuild` flows where shards may arrive before any
|
||||
/// `VolumeEcShardsMount` has happened on the receiving disk. Without
|
||||
/// checking the on-disk state, auto-select can split shards from the
|
||||
/// `.ecx` that travels with the first shard, which is the source of
|
||||
/// the orphan-shard layout reported in seaweedfs/seaweedfs#9212.
|
||||
///
|
||||
/// Mirrors `DiskLocation.HasEcxFileOnDisk` in
|
||||
/// `weed/storage/disk_location_ec.go`. Skips entries that are
|
||||
/// directories so a stray dir named `<collection>_<vid>.ecx` doesn't
|
||||
/// register as a present index file.
|
||||
pub fn has_ecx_file_on_disk(&self, collection: &str, vid: VolumeId) -> bool {
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
let idx_path = format!("{}.ecx", idx_base);
|
||||
if let Ok(meta) = fs::metadata(&idx_path) {
|
||||
if !meta.is_dir() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if self.idx_directory != self.directory {
|
||||
let data_base = volume_file_name(&self.directory, collection, vid);
|
||||
let data_path = format!("{}.ecx", data_base);
|
||||
if let Ok(meta) = fs::metadata(&data_path) {
|
||||
if !meta.is_dir() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Remove an EC volume, returning it.
|
||||
pub fn remove_ec_volume(&mut self, vid: VolumeId) -> Option<EcVolume> {
|
||||
self.ec_volumes.remove(&vid)
|
||||
}
|
||||
|
||||
/// Mount EC shards for a volume on this location.
|
||||
///
|
||||
/// `source_disk_type` is the source volume's disk type carried on the
|
||||
/// `VolumeEcShardsMount` RPC. When non-empty it overrides the in-memory
|
||||
/// EC volume's reported disk type so heartbeats keep reporting under
|
||||
/// the source's disk type after EC encoding (#9423). Empty means "use
|
||||
/// this location's disk type" — used by disk-scan reload paths that
|
||||
/// have no orchestrator context.
|
||||
pub fn mount_ec_shards(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_ids: &[u32],
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
let idx_dir = self.idx_directory.clone();
|
||||
self.mount_ec_shards_with_idx_dir(vid, collection, shard_ids, &idx_dir, source_disk_type)
|
||||
}
|
||||
|
||||
/// Mount EC shards but explicitly specify the idx directory the
|
||||
/// EcVolume should pull `.ecx` / `.ecj` / `.vif` from.
|
||||
///
|
||||
/// Cross-disk reconcile (mirrors `loadEcShardsWithIdxDir` in
|
||||
/// `weed/storage/disk_location_ec.go`): when a volume's shards live
|
||||
/// on this disk but the index files were left on a sibling disk
|
||||
/// (the seaweedfs/seaweedfs#9212 orphan-shard layout), the
|
||||
/// reconciler creates the EcVolume here while pointing it at the
|
||||
/// sibling's idx dir. Each shard still ends up registered in this
|
||||
/// disk's `ec_volumes` map so heartbeat reporting carries the right
|
||||
/// disk_id per shard.
|
||||
pub fn mount_ec_shards_with_idx_dir(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_ids: &[u32],
|
||||
idx_dir: &str,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
let dir = self.directory.clone();
|
||||
// Avoid the entry().or_insert_with() pattern here: that closure
|
||||
// can't return a Result, so any EcVolume::new failure (e.g.
|
||||
// .ecx open error, .ecj create error, malformed .vif) would
|
||||
// have to panic via unwrap(). Build the EcVolume up front and
|
||||
// propagate the error to the caller.
|
||||
if !self.ec_volumes.contains_key(&vid) {
|
||||
let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid)
|
||||
.map_err(VolumeError::Io)?;
|
||||
self.ec_volumes.insert(vid, ec_vol);
|
||||
}
|
||||
let idx_dir = self.idx_directory.clone();
|
||||
let ec_vol = self
|
||||
.ec_volumes
|
||||
.get_mut(&vid)
|
||||
.expect("just inserted above");
|
||||
// When the orchestrator supplied a source disk type on the Mount
|
||||
// RPC, override the EC volume's disk type so heartbeats report
|
||||
// under the source volume's disk type (#9423). When the caller
|
||||
// passed "" (disk-scan reload, orphan-shard reconcile, restart)
|
||||
// we only default to this location's disk type for a fresh EC
|
||||
// volume; otherwise we leave whatever a prior RPC mount set in
|
||||
// place, so empty-source reloads don't reintroduce the drift.
|
||||
if !source_disk_type.is_empty() {
|
||||
ec_vol.set_disk_type(DiskType::from_string(source_disk_type));
|
||||
} else if ec_vol.shard_count() == 0 {
|
||||
ec_vol.set_disk_type(self.disk_type.clone());
|
||||
}
|
||||
.entry(vid)
|
||||
.or_insert_with(|| EcVolume::new(&dir, &idx_dir, collection, vid).unwrap());
|
||||
ec_vol.disk_type = self.disk_type.clone();
|
||||
|
||||
for &shard_id in shard_ids {
|
||||
let mut shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
|
||||
shard.disk_type = ec_vol.disk_type.clone();
|
||||
let shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
|
||||
ec_vol.add_shard(shard).map_err(VolumeError::Io)?;
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[collection, "ec_shards"])
|
||||
@@ -681,19 +575,10 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
/// Unmount EC shards for a volume on this location.
|
||||
///
|
||||
/// Only shards that are actually mounted decrement the per-shard
|
||||
/// metric — without this guard the gauge would underflow when the
|
||||
/// caller passes a shard that lives on a sibling disk
|
||||
/// (cross-disk reconcile makes that the common case for the same
|
||||
/// `vid` after reconciliation).
|
||||
pub fn unmount_ec_shards(&mut self, vid: VolumeId, shard_ids: &[u32]) {
|
||||
if let Some(ec_vol) = self.ec_volumes.get_mut(&vid) {
|
||||
let collection = ec_vol.collection.clone();
|
||||
for &shard_id in shard_ids {
|
||||
if !ec_vol.has_shard(shard_id as u8) {
|
||||
continue;
|
||||
}
|
||||
ec_vol.remove_shard(shard_id as u8);
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&collection, "ec_shards"])
|
||||
@@ -706,183 +591,6 @@ impl DiskLocation {
|
||||
}
|
||||
}
|
||||
|
||||
/// Auto-discover .ec?? shard files on disk and mount them.
|
||||
///
|
||||
/// Mirrors `loadAllEcShards` in `weed/storage/disk_location_ec.go`.
|
||||
/// Reads the data dir (and idx dir, if separate), groups shards by
|
||||
/// (collection, vid) using sorted iteration so a volume's shards
|
||||
/// land contiguously next to their `.ecx`. When the matching `.ecx`
|
||||
/// is encountered, validates and mounts the group.
|
||||
///
|
||||
/// Without this, an EC shard on disk is invisible after a Rust
|
||||
/// volume-server restart until something explicitly issues
|
||||
/// VolumeEcShardsMount — strict superset of seaweedfs/seaweedfs#9212.
|
||||
pub fn load_all_ec_shards(&mut self) -> io::Result<()> {
|
||||
// Use a HashSet during accumulation so a name appearing in
|
||||
// both data dir and idx dir (idempotent legacy layouts can
|
||||
// place .ecx in either) is processed exactly once. Without
|
||||
// dedup, mount_ec_shards' per-shard metric increment would
|
||||
// double-count for those filenames.
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
let mut entries: Vec<String> = Vec::new();
|
||||
for ent in fs::read_dir(&self.directory)? {
|
||||
let ent = ent?;
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
if seen.insert(name.clone()) {
|
||||
entries.push(name);
|
||||
}
|
||||
}
|
||||
if self.idx_directory != self.directory {
|
||||
for ent in fs::read_dir(&self.idx_directory)? {
|
||||
let ent = ent?;
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
if seen.insert(name.clone()) {
|
||||
entries.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
entries.sort();
|
||||
|
||||
let mut same_volume_shards: Vec<(String, u32)> = Vec::new(); // (filename, shard_id)
|
||||
let mut prev_vid: Option<VolumeId> = None;
|
||||
let mut prev_collection: String = String::new();
|
||||
|
||||
for name in entries {
|
||||
let Some(dot) = name.rfind('.') else {
|
||||
continue;
|
||||
};
|
||||
let (base, ext) = name.split_at(dot); // ext includes leading '.'
|
||||
let Some((collection, vid)) = parse_collection_volume_id(base) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if let Some(shard_id) = parse_ec_shard_extension(ext) {
|
||||
// Ignore zero-byte shards — same as Go.
|
||||
let path = format!("{}/{}", self.directory, name);
|
||||
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
|
||||
if size == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let same_group = match prev_vid {
|
||||
None => true,
|
||||
Some(p) => p == vid && prev_collection == collection,
|
||||
};
|
||||
if same_group {
|
||||
same_volume_shards.push((name, shard_id));
|
||||
} else {
|
||||
self.check_orphaned_shards(
|
||||
&same_volume_shards,
|
||||
&prev_collection,
|
||||
prev_vid.unwrap_or(VolumeId(0)),
|
||||
);
|
||||
same_volume_shards = vec![(name, shard_id)];
|
||||
}
|
||||
prev_vid = Some(vid);
|
||||
prev_collection = collection;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ext == ".ecx"
|
||||
&& prev_vid == Some(vid)
|
||||
&& prev_collection == collection
|
||||
&& !same_volume_shards.is_empty()
|
||||
{
|
||||
let shards = std::mem::take(&mut same_volume_shards);
|
||||
self.handle_found_ecx_file(&shards, &collection, vid);
|
||||
prev_vid = None;
|
||||
prev_collection.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Handle any trailing group that never saw a matching .ecx.
|
||||
self.check_orphaned_shards(
|
||||
&same_volume_shards,
|
||||
&prev_collection,
|
||||
prev_vid.unwrap_or(VolumeId(0)),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate + mount a (collection, vid) group when its `.ecx` is
|
||||
/// found. Mirrors `handleFoundEcxFile` in
|
||||
/// `weed/storage/disk_location_ec.go`.
|
||||
fn handle_found_ecx_file(&mut self, shards: &[(String, u32)], collection: &str, vid: VolumeId) {
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let dat_path = format!("{}.dat", base);
|
||||
let dat_exists = check_dat_file_exists(&dat_path);
|
||||
|
||||
if dat_exists && !self.validate_ec_volume(collection, vid) {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Incomplete or invalid EC volume: .dat exists but validation failed, cleaning up EC files",
|
||||
);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
return;
|
||||
}
|
||||
|
||||
let shard_ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
|
||||
if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids, "") {
|
||||
// mount_ec_shards adds shards one at a time and increments
|
||||
// the per-shard metric for each. If it fails halfway, plain
|
||||
// ec_volumes.remove(vid) would leak metric increments for
|
||||
// the shards that did mount. Drive cleanup through
|
||||
// unmount_ec_shards which mirror-decrements the metric, then
|
||||
// the empty EcVolume drops itself.
|
||||
if dat_exists {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Failed to load EC shards and .dat exists ({}), cleaning up EC files to use .dat",
|
||||
e,
|
||||
);
|
||||
self.unmount_ec_shards(vid, &shard_ids);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
} else {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Failed to load EC shards: {} (this may be normal for distributed EC volumes)",
|
||||
e,
|
||||
);
|
||||
self.unmount_ec_shards(vid, &shard_ids);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors `checkOrphanedShards`: if shards exist on disk without a
|
||||
/// matching `.ecx` AND the legacy `.dat` is still present, treat
|
||||
/// the encoding as interrupted before `.ecx` was written and clean
|
||||
/// up. If `.dat` is gone, leave files alone — they may be
|
||||
/// distributed-EC shards waiting for cross-disk reconciliation.
|
||||
fn check_orphaned_shards(
|
||||
&self,
|
||||
shards: &[(String, u32)],
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> bool {
|
||||
if shards.is_empty() || vid.0 == 0 {
|
||||
return false;
|
||||
}
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let dat_path = format!("{}.dat", base);
|
||||
if check_dat_file_exists(&dat_path) {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Found {} EC shards without .ecx file (incomplete encoding interrupted before .ecx), cleaning up",
|
||||
shards.len(),
|
||||
);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Total number of EC shards on this location.
|
||||
pub fn ec_shard_count(&self) -> usize {
|
||||
self.ec_volumes
|
||||
@@ -955,69 +663,6 @@ fn calculate_expected_shard_size(dat_file_size: i64) -> i64 {
|
||||
}
|
||||
|
||||
/// Parse a volume filename like "collection_42.dat" or "42.dat" into (collection, VolumeId).
|
||||
/// Parse a `<collection>_<vid>` or `<vid>` base name into its parts.
|
||||
/// Mirrors `parseCollectionVolumeId` in
|
||||
/// `weed/storage/disk_location.go`. Used when iterating raw filenames
|
||||
/// where the extension has already been stripped.
|
||||
///
|
||||
/// `pub(crate)` companion `parse_collection_volume_id_pub` is exposed
|
||||
/// so the cross-disk reconcile in `store_ec_reconcile.rs` can call it
|
||||
/// without re-implementing the parser.
|
||||
pub(crate) fn parse_collection_volume_id_pub(base: &str) -> Option<(String, VolumeId)> {
|
||||
parse_collection_volume_id(base)
|
||||
}
|
||||
|
||||
fn parse_collection_volume_id(base: &str) -> Option<(String, VolumeId)> {
|
||||
if let Some(pos) = base.rfind('_') {
|
||||
let collection = &base[..pos];
|
||||
let id_str = &base[pos + 1..];
|
||||
let id: u32 = id_str.parse().ok()?;
|
||||
Some((collection.to_string(), VolumeId(id)))
|
||||
} else {
|
||||
let id: u32 = base.parse().ok()?;
|
||||
Some((String::new(), VolumeId(id)))
|
||||
}
|
||||
}
|
||||
|
||||
/// `pub(crate)` re-export of [`parse_ec_shard_extension`] for the
|
||||
/// cross-disk reconcile in `store_ec_reconcile.rs`.
|
||||
pub(crate) fn is_ec_shard_extension(ext: &str) -> Option<u32> {
|
||||
parse_ec_shard_extension(ext)
|
||||
}
|
||||
|
||||
/// Recognise EC shard extensions `.ec00`–`.ec255` (the ShardId u8
|
||||
/// range — see `EcVolumeShard`'s typed shard id) and return the
|
||||
/// shard id. Returns `None` for any other extension.
|
||||
///
|
||||
/// Matches the regex `\.ec\d{2,3}` from the Go side, with the same
|
||||
/// per-shard `id <= 255` clamp Go's loader applies via
|
||||
/// `strconv.ParseInt(... 10, 64)` followed by the `if shardId < 0 ||
|
||||
/// shardId > 255` guard. The 3-digit form (`.ec100`–`.ec255`) is
|
||||
/// retained so the parser can still recognise shards from custom
|
||||
/// 32+ ratios that fit in a u8 even though OSS only ships 10+4.
|
||||
fn parse_ec_shard_extension(ext: &str) -> Option<u32> {
|
||||
let rest = ext.strip_prefix(".ec")?;
|
||||
if rest.len() < 2 || rest.len() > 3 {
|
||||
return None;
|
||||
}
|
||||
let id: u32 = rest.parse().ok()?;
|
||||
if id > 255 {
|
||||
return None;
|
||||
}
|
||||
Some(id)
|
||||
}
|
||||
|
||||
/// Robust `.dat` existence check: any unexpected stat error (permission,
|
||||
/// I/O) is treated as "exists" so we don't misclassify local EC as
|
||||
/// distributed EC. Mirrors `checkDatFileExists` in Go.
|
||||
fn check_dat_file_exists(path: &str) -> bool {
|
||||
match fs::metadata(path) {
|
||||
Ok(_) => true,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => false,
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_volume_filename(filename: &str) -> Option<(String, VolumeId)> {
|
||||
let stem = filename
|
||||
.strip_suffix(".dat")
|
||||
@@ -1189,7 +834,7 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(loc.volumes_len(), 2);
|
||||
|
||||
loc.delete_volume(VolumeId(1), false, false).unwrap();
|
||||
loc.delete_volume(VolumeId(1), false).unwrap();
|
||||
assert_eq!(loc.volumes_len(), 1);
|
||||
assert!(loc.find_volume(VolumeId(1)).is_none());
|
||||
}
|
||||
@@ -1262,7 +907,7 @@ mod tests {
|
||||
let shard_path = format!("{}/pics_7.ec00", dir);
|
||||
std::fs::write(&shard_path, b"ec-shard").unwrap();
|
||||
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0]).unwrap();
|
||||
assert!(loc.has_ec_volume(VolumeId(7)));
|
||||
assert!(std::path::Path::new(&shard_path).exists());
|
||||
assert!(std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
|
||||
@@ -1274,56 +919,6 @@ mod tests {
|
||||
assert!(!std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
|
||||
}
|
||||
|
||||
/// #9423: after an RPC `VolumeEcShardsMount` records the source
|
||||
/// volume's disk type on the EcVolume, a later empty-source mount
|
||||
/// call (disk-scan reload, orphan-shard reconcile, restart) must
|
||||
/// not clobber that override back to the physical location's disk
|
||||
/// type — otherwise heartbeat reporting drifts back to "hdd" on
|
||||
/// every reload.
|
||||
#[test]
|
||||
fn test_mount_ec_shards_empty_source_preserves_existing_disk_type() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Plant the first shard so the EC volume can mount, then call
|
||||
// mount_ec_shards with source_disk_type="ssd" — simulating the
|
||||
// VolumeEcShardsMount RPC path.
|
||||
std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd").unwrap();
|
||||
{
|
||||
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume mounted");
|
||||
assert_eq!(
|
||||
ec_vol.disk_type,
|
||||
DiskType::Ssd,
|
||||
"first RPC mount should have set disk type to source's value",
|
||||
);
|
||||
}
|
||||
|
||||
// Plant another shard and re-mount with an empty source —
|
||||
// matching what a disk-scan reload or orphan-shard reconcile
|
||||
// would pass. The previously-recorded "ssd" override must
|
||||
// survive.
|
||||
std::fs::write(format!("{}/pics_7.ec01", dir), b"ec-shard").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[1], "").unwrap();
|
||||
{
|
||||
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume still mounted");
|
||||
assert_eq!(
|
||||
ec_vol.disk_type,
|
||||
DiskType::Ssd,
|
||||
"empty-source remount must not reset disk type to the physical location's",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disk_location_persists_directory_uuid_and_tags() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -1353,152 +948,4 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(reloaded.directory_uuid, directory_uuid);
|
||||
}
|
||||
|
||||
/// Sanity: `parse_ec_shard_extension` accepts `.ec00` through
|
||||
/// `.ec999` and rejects anything else.
|
||||
#[test]
|
||||
fn test_parse_ec_shard_extension() {
|
||||
assert_eq!(parse_ec_shard_extension(".ec00"), Some(0));
|
||||
assert_eq!(parse_ec_shard_extension(".ec13"), Some(13));
|
||||
assert_eq!(parse_ec_shard_extension(".ec999"), None); // > 255
|
||||
assert_eq!(parse_ec_shard_extension(".ec255"), Some(255));
|
||||
assert_eq!(parse_ec_shard_extension(".ec0"), None); // need 2 digits
|
||||
assert_eq!(parse_ec_shard_extension(".ecx"), None);
|
||||
assert_eq!(parse_ec_shard_extension(".ecj"), None);
|
||||
assert_eq!(parse_ec_shard_extension(".dat"), None);
|
||||
assert_eq!(parse_ec_shard_extension("ec00"), None); // missing dot
|
||||
}
|
||||
|
||||
/// Auto-load: shards + .ecx on the same disk get mounted into
|
||||
/// `ec_volumes` after `load_existing_volumes` finishes. Without
|
||||
/// this, EC shards on disk are invisible until something issues
|
||||
/// `VolumeEcShardsMount`.
|
||||
#[test]
|
||||
fn test_load_all_ec_shards_mounts_pairs_with_ecx() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(1234);
|
||||
|
||||
// Two shards + .ecx + .ecj + .vif (the post-encode steady state).
|
||||
for &sid in &[0u8, 4u8] {
|
||||
let path = format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid);
|
||||
std::fs::write(&path, b"shard data nonempty").unwrap();
|
||||
}
|
||||
std::fs::write(format!("{}/{}_{}.ecx", dir, collection, vid.0), vec![0u8; 20])
|
||||
.unwrap();
|
||||
std::fs::write(format!("{}/{}_{}.ecj", dir, collection, vid.0), b"").unwrap();
|
||||
std::fs::write(
|
||||
format!("{}/{}_{}.vif", dir, collection, vid.0),
|
||||
br#"{"version":3}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
loc.load_existing_volumes(NeedleMapKind::InMemory).unwrap();
|
||||
|
||||
let ec_vol = loc.find_ec_volume(vid).expect("EcVolume should be mounted");
|
||||
assert_eq!(ec_vol.shard_count(), 2, "both shards should be loaded");
|
||||
}
|
||||
|
||||
/// Orphan shards with no `.ecx` and no `.dat` must stay on disk
|
||||
/// untouched (distributed-EC scenario, where the .ecx lives on
|
||||
/// another server). Mirrors Go's `checkOrphanedShards` no-op
|
||||
/// branch.
|
||||
#[test]
|
||||
fn test_load_all_ec_shards_keeps_orphan_shards_when_no_dat() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(2345);
|
||||
|
||||
// Shards on disk, but no .ecx and no .dat — distributed-EC
|
||||
// with the .ecx on a sibling/server.
|
||||
let shard_paths: Vec<String> = [0u8, 12u8]
|
||||
.iter()
|
||||
.map(|sid| format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid))
|
||||
.collect();
|
||||
for p in &shard_paths {
|
||||
std::fs::write(p, b"shard data nonempty").unwrap();
|
||||
}
|
||||
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
loc.load_existing_volumes(NeedleMapKind::InMemory).unwrap();
|
||||
|
||||
// No EcVolume should be mounted (no .ecx).
|
||||
assert!(loc.find_ec_volume(vid).is_none());
|
||||
// But the shard files must still be on disk for cross-disk
|
||||
// reconciliation / operator recovery.
|
||||
for p in &shard_paths {
|
||||
assert!(
|
||||
std::path::Path::new(p).exists(),
|
||||
"orphan shard {} was destroyed",
|
||||
p,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Orphan shards WITH a stale `.dat` are treated as an
|
||||
/// interrupted-encoding remnant and cleaned up. Mirrors Go's
|
||||
/// `checkOrphanedShards` cleanup branch. Guards a subtle case
|
||||
/// where a partial encoding wrote shards before .ecx and a crash
|
||||
/// left the .dat behind.
|
||||
#[test]
|
||||
fn test_load_all_ec_shards_cleans_orphan_shards_when_dat_exists() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(3456);
|
||||
|
||||
let shard_paths: Vec<String> = [0u8, 1u8]
|
||||
.iter()
|
||||
.map(|sid| format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid))
|
||||
.collect();
|
||||
for p in &shard_paths {
|
||||
std::fs::write(p, b"shard data nonempty").unwrap();
|
||||
}
|
||||
// Stale .dat from interrupted encoding.
|
||||
let dat_path = format!("{}/{}_{}.dat", dir, collection, vid.0);
|
||||
std::fs::write(&dat_path, b"unfinished").unwrap();
|
||||
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// load_existing_volumes processes .dat first; ec auto-load
|
||||
// runs after and finds the orphans. The .dat itself may or may
|
||||
// not be loadable as a regular volume — that's not what this
|
||||
// test is about. Just check that the orphan shard files were
|
||||
// removed.
|
||||
let _ = loc.load_existing_volumes(NeedleMapKind::InMemory);
|
||||
|
||||
for p in &shard_paths {
|
||||
assert!(
|
||||
!std::path::Path::new(p).exists(),
|
||||
"orphan shard {} should have been cleaned up because .dat exists",
|
||||
p,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,28 +16,11 @@ use crate::storage::volume::volume_file_name;
|
||||
/// Calculate .dat file size from the max offset entry in .ecx.
|
||||
/// Reads the volume version from the first EC shard (.ec00) superblock,
|
||||
/// then scans .ecx entries to find the largest (offset + needle_actual_size).
|
||||
///
|
||||
/// `dir` is used both for reading `.ec00` and `.ecx`. For split-disk
|
||||
/// reconciled volumes call [`find_dat_file_size_with_dirs`] instead.
|
||||
pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
|
||||
find_dat_file_size_with_dirs(dir, dir, collection, volume_id)
|
||||
}
|
||||
|
||||
/// Like [`find_dat_file_size`] but lets the caller pass separate dirs
|
||||
/// for `.ec00` (the data shard) and `.ecx` (the sealed index). This
|
||||
/// is the form needed when shards are split across data dirs and the
|
||||
/// `.ecx` lives on a sibling disk's idx dir (#9252).
|
||||
pub fn find_dat_file_size_with_dirs(
|
||||
ec00_dir: &str,
|
||||
ecx_dir: &str,
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<i64> {
|
||||
let ec00_base = volume_file_name(ec00_dir, collection, volume_id);
|
||||
let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
|
||||
let base = volume_file_name(dir, collection, volume_id);
|
||||
|
||||
// Read volume version from .ec00 superblock
|
||||
let ec00_path = format!("{}.ec00", ec00_base);
|
||||
let ec00_path = format!("{}.ec00", base);
|
||||
let mut ec00 = File::open(&ec00_path)?;
|
||||
let mut sb_buf = [0u8; SUPER_BLOCK_SIZE];
|
||||
ec00.read_exact(&mut sb_buf)?;
|
||||
@@ -47,7 +30,7 @@ pub fn find_dat_file_size_with_dirs(
|
||||
let mut dat_size: i64 = SUPER_BLOCK_SIZE as i64;
|
||||
|
||||
// Scan .ecx entries
|
||||
let ecx_path = format!("{}.ecx", ecx_base);
|
||||
let ecx_path = format!("{}.ecx", base);
|
||||
let ecx_data = std::fs::read(&ecx_path)?;
|
||||
let entry_count = ecx_data.len() / NEEDLE_MAP_ENTRY_SIZE;
|
||||
|
||||
@@ -69,10 +52,7 @@ pub fn find_dat_file_size_with_dirs(
|
||||
|
||||
/// Reconstruct a .dat file from EC data shards.
|
||||
///
|
||||
/// Reads from .ec00-.ec09 and writes a new .dat file. All data shards
|
||||
/// must live in `dir`. For the cross-disk reconciled layout where
|
||||
/// shards are split across multiple data dirs of the same node, use
|
||||
/// [`write_dat_file_from_shards_with_dirs`] instead.
|
||||
/// Reads from .ec00-.ec09 and writes a new .dat file.
|
||||
pub fn write_dat_file_from_shards(
|
||||
dir: &str,
|
||||
collection: &str,
|
||||
@@ -80,53 +60,12 @@ pub fn write_dat_file_from_shards(
|
||||
dat_file_size: i64,
|
||||
data_shards: usize,
|
||||
) -> io::Result<()> {
|
||||
let dirs: Vec<String> = (0..data_shards).map(|_| dir.to_string()).collect();
|
||||
write_dat_file_from_shards_with_dirs(
|
||||
dir,
|
||||
collection,
|
||||
volume_id,
|
||||
dat_file_size,
|
||||
data_shards,
|
||||
&dirs,
|
||||
)
|
||||
}
|
||||
|
||||
/// Reconstruct a .dat file from EC data shards, taking the source
|
||||
/// directory for each shard separately.
|
||||
///
|
||||
/// `dat_dir` is where the produced `.dat` is written. `shard_dirs[i]`
|
||||
/// is the directory holding shard `i`. For the simple "all shards in
|
||||
/// one dir" case both can be the same value.
|
||||
///
|
||||
/// Mirrors Go's `WriteDatFile(baseFileName, datFileSize,
|
||||
/// shardFileNames)` shape — Go passes per-shard paths so a
|
||||
/// reconciled volume with shards split across disks of the same
|
||||
/// volume server can still be decoded back to a regular .dat
|
||||
/// (seaweedfs/seaweedfs#9252).
|
||||
pub fn write_dat_file_from_shards_with_dirs(
|
||||
dat_dir: &str,
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
dat_file_size: i64,
|
||||
data_shards: usize,
|
||||
shard_dirs: &[String],
|
||||
) -> io::Result<()> {
|
||||
if shard_dirs.len() < data_shards {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"shard_dirs len {} < data_shards {}",
|
||||
shard_dirs.len(),
|
||||
data_shards
|
||||
),
|
||||
));
|
||||
}
|
||||
let base = volume_file_name(dat_dir, collection, volume_id);
|
||||
let base = volume_file_name(dir, collection, volume_id);
|
||||
let dat_path = format!("{}.dat", base);
|
||||
|
||||
// Open data shards from their individual home dirs.
|
||||
// Open data shards
|
||||
let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
|
||||
.map(|i| EcVolumeShard::new(&shard_dirs[i as usize], collection, volume_id, i))
|
||||
.map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
|
||||
.collect();
|
||||
|
||||
for shard in &mut shards {
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
#[cfg(not(unix))]
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::io::{Seek, SeekFrom};
|
||||
|
||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||
|
||||
|
||||
@@ -3,10 +3,9 @@
|
||||
//! Each EcVolume has a sorted index (.ecx) and a deletion journal (.ecj).
|
||||
//! Shards (.ec00-.ec13) may be distributed across multiple servers.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::sync::RwLock;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::pb::master_pb;
|
||||
@@ -29,72 +28,21 @@ pub struct EcVolume {
|
||||
ecx_file: Option<File>,
|
||||
ecx_file_size: i64,
|
||||
ecj_file: Option<File>,
|
||||
/// On-disk size of the .ecj deletion journal. Used only by IO helpers
|
||||
/// (seek / set_len on partial writes) — the authoritative runtime
|
||||
/// delete count comes from `deleted_needles.len()`.
|
||||
ecj_file_size: i64,
|
||||
/// In-memory set of needle ids that have been deleted since the volume
|
||||
/// was encoded. .ecx is immutable at runtime — it only stores the
|
||||
/// sorted (id, offset, size) index written at encode time — and runtime
|
||||
/// deletes are journaled to .ecj + tracked here. Reads consult this
|
||||
/// set to mask out deleted needles on top of the sealed .ecx lookup.
|
||||
/// Seeded from .ecj in `new()` and updated by `journal_delete`.
|
||||
deleted_needles: RwLock<HashSet<NeedleId>>,
|
||||
pub disk_type: DiskType,
|
||||
/// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
|
||||
ecx_actual_dir: String,
|
||||
/// Maps shard ID -> list of server addresses where that shard exists.
|
||||
/// Used for distributed EC reads across the cluster. Wrapped in
|
||||
/// `RwLock` so the read path can refresh the map (under master
|
||||
/// lookup) without holding the Store write lock — mirrors Go's
|
||||
/// `ShardLocationsLock sync.RWMutex` in `weed/storage/erasure_coding/ec_volume.go`.
|
||||
pub shard_locations: std::sync::RwLock<HashMap<ShardId, Vec<String>>>,
|
||||
/// Wall-clock timestamp of the most recent successful
|
||||
/// `LookupEcVolume` refresh of `shard_locations`. `None` until the
|
||||
/// first refresh. Drives the staleness heuristic in
|
||||
/// `cached_lookup_ec_shard_locations` (mirrors Go's
|
||||
/// `ShardLocationsRefreshTime`).
|
||||
pub shard_locations_refresh_time: std::sync::Mutex<Option<std::time::Instant>>,
|
||||
/// Used for distributed EC reads across the cluster.
|
||||
pub shard_locations: HashMap<ShardId, Vec<String>>,
|
||||
/// EC volume expiration time (unix epoch seconds), set during EC encode from TTL.
|
||||
pub expire_at_sec: u64,
|
||||
}
|
||||
|
||||
/// Locate the `.vif` for a (collection, vid) by preferring the data dir
|
||||
/// and falling back to the idx dir when it lives there instead. The
|
||||
/// fallback covers the cross-disk reconcile path: when a volume's
|
||||
/// shards live on one disk but its `.ecx` / `.ecj` / `.vif` live on a
|
||||
/// sibling disk (seaweedfs/seaweedfs#9212 / #9244), we want to read the
|
||||
/// real `.vif` from the sibling rather than write a stub on the shard
|
||||
/// disk and lose the EC config + dat file size.
|
||||
fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
|
||||
let data_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir, collection, volume_id),
|
||||
);
|
||||
if dir_idx != dir && !std::path::Path::new(&data_vif).exists() {
|
||||
let idx_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
|
||||
);
|
||||
if std::path::Path::new(&idx_vif).exists() {
|
||||
return idx_vif;
|
||||
}
|
||||
}
|
||||
data_vif
|
||||
}
|
||||
|
||||
/// Read EC data/parity shard counts from `.vif`, defaulting to the
|
||||
/// build's standard ratio when no `.vif` is present or is malformed.
|
||||
/// Looks at the data dir first, then the idx dir — see [`locate_vif_path`].
|
||||
pub fn read_ec_shard_config(
|
||||
dir: &str,
|
||||
dir_idx: &str,
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> (u32, u32) {
|
||||
pub fn read_ec_shard_config(dir: &str, collection: &str, volume_id: VolumeId) -> (u32, u32) {
|
||||
let mut data_shards = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
|
||||
let mut parity_shards = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
|
||||
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
|
||||
let vif_path = format!("{}.vif", base);
|
||||
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
|
||||
if let Ok(vif_info) =
|
||||
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
|
||||
@@ -121,7 +69,7 @@ impl EcVolume {
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Self> {
|
||||
let (data_shards, parity_shards) = read_ec_shard_config(dir, dir_idx, collection, volume_id);
|
||||
let (data_shards, parity_shards) = read_ec_shard_config(dir, collection, volume_id);
|
||||
|
||||
let total_shards = (data_shards + parity_shards) as usize;
|
||||
let mut shards = Vec::with_capacity(total_shards);
|
||||
@@ -129,16 +77,10 @@ impl EcVolume {
|
||||
shards.push(None);
|
||||
}
|
||||
|
||||
// Read expire_at_sec, version, and dat_file_size from .vif if
|
||||
// present (matches Go's MaybeLoadVolumeInfo). Prefer the data
|
||||
// dir; fall back to the idx dir for the cross-disk reconcile
|
||||
// case (#9212 / #9244). `dat_file_size` is the source .dat
|
||||
// size at encode time, used both by `locate_ec_shard_needle`
|
||||
// for shard-size math and by the Store-level prune in
|
||||
// `store_ec_reconcile.rs` to verify a sibling-disk .dat is
|
||||
// plausibly the encoding source (#9478).
|
||||
let (expire_at_sec, vif_version, vif_dat_file_size) = {
|
||||
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
// Read expire_at_sec and version from .vif if present (matches Go's MaybeLoadVolumeInfo)
|
||||
let (expire_at_sec, vif_version) = {
|
||||
let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
|
||||
let vif_path = format!("{}.vif", base);
|
||||
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
|
||||
if let Ok(vif_info) =
|
||||
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
|
||||
@@ -148,12 +90,12 @@ impl EcVolume {
|
||||
} else {
|
||||
Version::current()
|
||||
};
|
||||
(vif_info.expire_at_sec, ver, vif_info.dat_file_size)
|
||||
(vif_info.expire_at_sec, ver)
|
||||
} else {
|
||||
(0, Version::current(), 0)
|
||||
(0, Version::current())
|
||||
}
|
||||
} else {
|
||||
(0, Version::current(), 0)
|
||||
(0, Version::current())
|
||||
}
|
||||
};
|
||||
|
||||
@@ -164,18 +106,15 @@ impl EcVolume {
|
||||
dir_idx: dir_idx.to_string(),
|
||||
version: vif_version,
|
||||
shards,
|
||||
dat_file_size: vif_dat_file_size,
|
||||
dat_file_size: 0,
|
||||
data_shards,
|
||||
parity_shards,
|
||||
ecx_file: None,
|
||||
ecx_file_size: 0,
|
||||
ecj_file: None,
|
||||
ecj_file_size: 0,
|
||||
deleted_needles: RwLock::new(HashSet::new()),
|
||||
disk_type: DiskType::default(),
|
||||
ecx_actual_dir: dir_idx.to_string(),
|
||||
shard_locations: std::sync::RwLock::new(HashMap::new()),
|
||||
shard_locations_refresh_time: std::sync::Mutex::new(None),
|
||||
shard_locations: HashMap::new(),
|
||||
expire_at_sec,
|
||||
};
|
||||
|
||||
@@ -202,12 +141,10 @@ impl EcVolume {
|
||||
}
|
||||
}
|
||||
|
||||
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency.
|
||||
// Note: Go does NOT replay .ecj into .ecx at volume load (RebuildEcxFile
|
||||
// is only invoked from specific decode/rebuild gRPC handlers), so we
|
||||
// don't either. Tombstones from prior sessions were already written
|
||||
// in-place in .ecx, and the journal grows monotonically until a
|
||||
// decode/rebuild operation folds it in.
|
||||
// Replay .ecj journal into .ecx on startup (matches Go's RebuildEcxFile).
|
||||
vol.rebuild_ecx_from_journal()?;
|
||||
|
||||
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency
|
||||
let ecj_base =
|
||||
crate::storage::volume::volume_file_name(&vol.ecx_actual_dir, collection, volume_id);
|
||||
let ecj_path = format!("{}.ecj", ecj_base);
|
||||
@@ -217,75 +154,11 @@ impl EcVolume {
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&ecj_path)?;
|
||||
vol.ecj_file_size = ecj_file.metadata()?.len() as i64;
|
||||
vol.ecj_file = Some(ecj_file);
|
||||
|
||||
// Seed the in-memory deleted set from the journal.
|
||||
vol.load_deleted_needles_from_ecj()?;
|
||||
|
||||
Ok(vol)
|
||||
}
|
||||
|
||||
/// Walk the .ecj journal and populate `deleted_needles`. Called once
|
||||
/// from `new()` under exclusive ownership of the just-constructed
|
||||
/// EcVolume, so locking is not strictly required — but we take the
|
||||
/// write lock anyway for symmetry with later mutations.
|
||||
fn load_deleted_needles_from_ecj(&mut self) -> io::Result<()> {
|
||||
let ecj_file = match self.ecj_file.as_ref() {
|
||||
Some(f) => f,
|
||||
None => return Ok(()),
|
||||
};
|
||||
if self.ecj_file_size < NEEDLE_ID_SIZE as i64 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
let mut set = self
|
||||
.deleted_needles
|
||||
.write()
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::Other, "deleted_needles lock poisoned"))?;
|
||||
let mut off: i64 = 0;
|
||||
while off + NEEDLE_ID_SIZE as i64 <= self.ecj_file_size {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecj_file.read_exact_at(&mut buf, off as u64)?;
|
||||
}
|
||||
set.insert(NeedleId::from_bytes(&buf));
|
||||
off += NEEDLE_ID_SIZE as i64;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns (file_count, delete_count) for this EC volume. Mirrors Go's
|
||||
/// `EcVolume.FileAndDeleteCount`:
|
||||
///
|
||||
/// file_count = ecx_file_size / NEEDLE_MAP_ENTRY_SIZE — total
|
||||
/// entries in the sealed sorted .ecx index.
|
||||
/// delete_count = deleted_needles.len() — unique
|
||||
/// runtime deletes tracked in memory (seeded from
|
||||
/// .ecj on load and updated by `journal_delete`).
|
||||
///
|
||||
/// Because each needle delete is applied on exactly one shard holder,
|
||||
/// the admin aggregation sums delete_count across nodes while taking
|
||||
/// file_count from a single holder (they are identical per volume).
|
||||
pub fn file_and_delete_count(&self) -> (u64, u64) {
|
||||
let file_count = (self.ecx_file_size as u64) / (NEEDLE_MAP_ENTRY_SIZE as u64);
|
||||
let delete_count = self
|
||||
.deleted_needles
|
||||
.read()
|
||||
.map(|s| s.len() as u64)
|
||||
.unwrap_or(0);
|
||||
(file_count, delete_count)
|
||||
}
|
||||
|
||||
/// Reports whether the given needle id is in the in-memory deleted set.
|
||||
pub fn is_needle_deleted(&self, needle_id: NeedleId) -> bool {
|
||||
self.deleted_needles
|
||||
.read()
|
||||
.map(|s| s.contains(&needle_id))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
// ---- File names ----
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -293,16 +166,8 @@ impl EcVolume {
|
||||
crate::storage::volume::volume_file_name(&self.dir, &self.collection, self.volume_id)
|
||||
}
|
||||
|
||||
/// Base path for the .ecx / .ecj index pair. Resolved from
|
||||
/// `ecx_actual_dir` (initialized to `dir_idx` and only updated after a
|
||||
/// successful idx-dir → data-dir fallback in `new()`), so every call site
|
||||
/// agrees on the same file regardless of whether the fallback fired.
|
||||
fn idx_base_name(&self) -> String {
|
||||
crate::storage::volume::volume_file_name(
|
||||
&self.ecx_actual_dir,
|
||||
&self.collection,
|
||||
self.volume_id,
|
||||
)
|
||||
crate::storage::volume::volume_file_name(&self.dir_idx, &self.collection, self.volume_id)
|
||||
}
|
||||
|
||||
pub fn ecx_file_name(&self) -> String {
|
||||
@@ -342,20 +207,6 @@ impl EcVolume {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Override the disk type the EC volume (and its already-mounted
|
||||
/// shards) reports under. Used by the `VolumeEcShardsMount` handler
|
||||
/// so the source volume's disk type is preserved across encoding
|
||||
/// (#9423). Not persisted across restarts — disk-scan reload paths
|
||||
/// default to the physical location's disk type.
|
||||
pub fn set_disk_type(&mut self, d: DiskType) {
|
||||
self.disk_type = d.clone();
|
||||
for slot in self.shards.iter_mut() {
|
||||
if let Some(shard) = slot {
|
||||
shard.disk_type = d.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove and close a shard.
|
||||
pub fn remove_shard(&mut self, shard_id: ShardId) {
|
||||
if let Some(ref mut shard) = self.shards[shard_id as usize] {
|
||||
@@ -380,24 +231,6 @@ impl EcVolume {
|
||||
self.shards.iter().filter(|s| s.is_some()).count()
|
||||
}
|
||||
|
||||
/// Reports whether `shard_id` is currently registered to this
|
||||
/// EcVolume (used by the cross-disk reconcile to skip already-
|
||||
/// loaded shards).
|
||||
pub fn has_shard(&self, shard_id: u8) -> bool {
|
||||
self.shards
|
||||
.get(shard_id as usize)
|
||||
.map(|s| s.is_some())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Directory where this EcVolume's `.ecx` was actually opened
|
||||
/// (may differ from `dir_idx` when the legacy "written before
|
||||
/// -dir.idx was set" fallback or the cross-disk reconcile path
|
||||
/// pointed it elsewhere).
|
||||
pub fn ecx_actual_dir(&self) -> &str {
|
||||
&self.ecx_actual_dir
|
||||
}
|
||||
|
||||
pub fn is_time_to_destroy(&self) -> bool {
|
||||
self.expire_at_sec > 0
|
||||
&& SystemTime::now()
|
||||
@@ -422,8 +255,6 @@ impl EcVolume {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let (file_count, delete_count) = self.file_and_delete_count();
|
||||
|
||||
vec![master_pb::VolumeEcShardInformationMessage {
|
||||
id: self.volume_id.0,
|
||||
collection: self.collection.clone(),
|
||||
@@ -432,53 +263,23 @@ impl EcVolume {
|
||||
disk_type: self.disk_type.to_string(),
|
||||
expire_at_sec: self.expire_at_sec,
|
||||
disk_id,
|
||||
file_count,
|
||||
delete_count,
|
||||
..Default::default()
|
||||
}]
|
||||
}
|
||||
|
||||
// ---- Shard locations (distributed tracking) ----
|
||||
|
||||
/// Set the list of server addresses for a single shard ID. Does
|
||||
/// NOT touch `shard_locations_refresh_time` — a per-shard write
|
||||
/// from inside a multi-shard population (e.g. iterating the
|
||||
/// `LookupEcVolume` response shard-by-shard) would otherwise
|
||||
/// flip the staleness flag while the map is still incomplete,
|
||||
/// letting a concurrent reader observe `needs_refresh == false`
|
||||
/// against a half-populated cache and return NotFound for the
|
||||
/// not-yet-inserted shards.
|
||||
///
|
||||
/// Callers populating the whole cache atomically should use
|
||||
/// [`Self::replace_shard_locations`] instead — it swaps the
|
||||
/// entire map under the write lock and advances the refresh
|
||||
/// timestamp in one step.
|
||||
pub fn set_shard_locations(&self, shard_id: ShardId, locations: Vec<String>) {
|
||||
self.shard_locations
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(shard_id, locations);
|
||||
/// Set the list of server addresses for a given shard ID.
|
||||
pub fn set_shard_locations(&mut self, shard_id: ShardId, locations: Vec<String>) {
|
||||
self.shard_locations.insert(shard_id, locations);
|
||||
}
|
||||
|
||||
/// Atomically replace the entire shard-locations map and stamp
|
||||
/// the refresh time. Used by the distributed-read path's
|
||||
/// post-`LookupEcVolume` write-back so the cache transitions
|
||||
/// from old → fresh in a single observable step — concurrent
|
||||
/// readers either see the full prior map or the full new map,
|
||||
/// never an intermediate state with the freshness flag flipped.
|
||||
pub fn replace_shard_locations(&self, locations: HashMap<ShardId, Vec<String>>) {
|
||||
*self.shard_locations.write().unwrap() = locations;
|
||||
*self.shard_locations_refresh_time.lock().unwrap() = Some(std::time::Instant::now());
|
||||
}
|
||||
|
||||
/// Get a cloned list of server addresses for a given shard ID.
|
||||
pub fn get_shard_locations(&self, shard_id: ShardId) -> Vec<String> {
|
||||
/// Get the list of server addresses for a given shard ID.
|
||||
pub fn get_shard_locations(&self, shard_id: ShardId) -> &[String] {
|
||||
self.shard_locations
|
||||
.read()
|
||||
.unwrap()
|
||||
.get(&shard_id)
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.map(|v| v.as_slice())
|
||||
.unwrap_or(&[])
|
||||
}
|
||||
|
||||
// ---- Index operations ----
|
||||
@@ -512,13 +313,6 @@ impl EcVolume {
|
||||
|
||||
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
// Apply runtime deletion state on top of the sealed .ecx
|
||||
// lookup: a needle in the in-memory deleted set is
|
||||
// reported with TOMBSTONE_FILE_SIZE even though the .ecx
|
||||
// record itself is untouched.
|
||||
if self.is_needle_deleted(needle_id) {
|
||||
return Ok(Some((offset, TOMBSTONE_FILE_SIZE)));
|
||||
}
|
||||
return Ok(Some((offset, size)));
|
||||
} else if key < needle_id {
|
||||
lo = mid + 1;
|
||||
@@ -776,41 +570,38 @@ impl EcVolume {
|
||||
|
||||
// ---- Deletion ----
|
||||
|
||||
/// Write `TOMBSTONE_FILE_SIZE` over the Size field of an existing .ecx
|
||||
/// entry, matching Go's `MarkNeedleDeleted`. Only used by the offline
|
||||
/// `rebuild_ecx_from_journal` path — the runtime delete path does not
|
||||
/// touch .ecx because the index is treated as an immutable sorted
|
||||
/// (id, offset, size) table. Returns `false` if the needle is not in
|
||||
/// the index (ignored by callers) and an error on IO failure.
|
||||
fn tombstone_ecx_entry(&self, needle_id: NeedleId) -> io::Result<bool> {
|
||||
let ecx_file = self.ecx_file.as_ref().ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"ec volume {} has no open .ecx file (closed or corrupt)",
|
||||
self.volume_id.0
|
||||
),
|
||||
)
|
||||
})?;
|
||||
/// Mark a needle as deleted in the .ecx file in-place.
|
||||
/// Matches Go's MarkNeedleDeleted: binary search the .ecx, then overwrite
|
||||
/// the size field with TOMBSTONE_FILE_SIZE.
|
||||
fn mark_needle_deleted_in_ecx(&self, needle_id: NeedleId) -> io::Result<bool> {
|
||||
let ecx_file = match self.ecx_file.as_ref() {
|
||||
Some(f) => f,
|
||||
None => return Ok(false),
|
||||
};
|
||||
|
||||
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
|
||||
if entry_count == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Binary search for the needle
|
||||
let mut lo: usize = 0;
|
||||
let mut hi: usize = entry_count;
|
||||
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
|
||||
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
|
||||
}
|
||||
let (key, _offset, _old_size) = idx_entry_from_bytes(&entry_buf);
|
||||
|
||||
let (key, _offset, _size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
// Found — overwrite the size field with TOMBSTONE_FILE_SIZE
|
||||
let size_offset = file_offset + NEEDLE_ID_SIZE as u64 + OFFSET_SIZE as u64;
|
||||
let mut size_buf = [0u8; SIZE_SIZE];
|
||||
TOMBSTONE_FILE_SIZE.to_bytes(&mut size_buf);
|
||||
@@ -826,18 +617,13 @@ impl EcVolume {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
|
||||
Ok(false) // not found
|
||||
}
|
||||
|
||||
/// Replay .ecj journal entries into .ecx: for each needle id in .ecj,
|
||||
/// overwrite its .ecx size field with a tombstone, then remove the
|
||||
/// journal file. Mirrors Go's `RebuildEcxFile`, which is invoked from
|
||||
/// specific decode / rebuild gRPC handlers — it is intentionally
|
||||
/// **not** called on volume load (runtime reads consult
|
||||
/// `deleted_needles` instead). The rebuild is atomic with respect to
|
||||
/// the journal: if any individual write fails the .ecj file is left
|
||||
/// in place and the error is propagated so tombstones are not lost.
|
||||
#[allow(dead_code)]
|
||||
/// Replay .ecj journal entries into .ecx on startup.
|
||||
/// Matches Go's RebuildEcxFile: for each needle ID in .ecj, marks it
|
||||
/// deleted in .ecx, then removes the .ecj file.
|
||||
fn rebuild_ecx_from_journal(&mut self) -> io::Result<()> {
|
||||
let ecj_path = self.ecj_file_name();
|
||||
if !std::path::Path::new(&ecj_path).exists() {
|
||||
@@ -856,23 +642,14 @@ impl EcVolume {
|
||||
break;
|
||||
}
|
||||
let needle_id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
|
||||
// A needle that never made it into .ecx is fine (e.g. the
|
||||
// delete raced against encode). Any other IO error aborts the
|
||||
// rebuild so the journal survives to be retried later.
|
||||
self.tombstone_ecx_entry(needle_id)?;
|
||||
// Errors for individual entries are non-fatal (needle may not exist in .ecx)
|
||||
let _ = self.mark_needle_deleted_in_ecx(needle_id);
|
||||
}
|
||||
|
||||
// Durably flush the newly-written .ecx tombstones before dropping
|
||||
// the journal: the writes went through write_all_at and may still
|
||||
// be in page cache.
|
||||
if let Some(ref ecx_file) = self.ecx_file {
|
||||
ecx_file.sync_all()?;
|
||||
}
|
||||
// Remove the .ecj file after replay (matches Go)
|
||||
let _ = fs::remove_file(&ecj_path);
|
||||
|
||||
// Fold successful — drop and recreate the journal, clear the
|
||||
// in-memory deleted set (all of its contents are now materialized
|
||||
// in .ecx), and reset the cached size.
|
||||
fs::remove_file(&ecj_path)?;
|
||||
// Re-create .ecj for future deletions
|
||||
let ecj_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
@@ -880,126 +657,27 @@ impl EcVolume {
|
||||
.append(true)
|
||||
.open(&ecj_path)?;
|
||||
self.ecj_file = Some(ecj_file);
|
||||
self.ecj_file_size = 0;
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.clear();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Deletion journal ----
|
||||
|
||||
/// Record a needle delete: append the id to the .ecj deletion journal
|
||||
/// and insert it into the in-memory deleted set. `.ecx` is not touched
|
||||
/// at runtime — it is a sealed sorted (id, offset, size) index and
|
||||
/// runtime deletion state lives exclusively in .ecj + `deleted_needles`.
|
||||
/// A lookup via `find_needle_from_ecx` masks the id out by returning
|
||||
/// `TOMBSTONE_FILE_SIZE` on a subsequent read.
|
||||
///
|
||||
/// The .ecj append is the durable commit point. On any failure the
|
||||
/// file is truncated back to the pre-append length so the on-disk
|
||||
/// journal and in-memory state cannot drift. Only after the sync
|
||||
/// succeeds is the id published into the set, so a failure leaves
|
||||
/// the delete invisible to readers.
|
||||
/// Append a deleted needle ID to the .ecj journal and mark in .ecx.
|
||||
/// Matches Go's DeleteNeedleFromEcx: marks in .ecx first, then journals.
|
||||
pub fn journal_delete(&mut self, needle_id: NeedleId) -> io::Result<()> {
|
||||
// Look the needle up read-only. Missing is a silent no-op; a
|
||||
// pre-existing .ecx tombstone (from a prior decode/rebuild) is
|
||||
// mirrored into the in-memory set so delete_count stays accurate
|
||||
// without needing to walk .ecx on every heartbeat.
|
||||
match self.find_needle_from_ecx_raw(needle_id)? {
|
||||
None => return Ok(()),
|
||||
Some((_, size)) if size.is_deleted() => {
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.insert(needle_id);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
// Mark deleted in .ecx in-place (matches Go's MarkNeedleDeleted)
|
||||
let _ = self.mark_needle_deleted_in_ecx(needle_id);
|
||||
let ecj_file = self
|
||||
.ecj_file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
|
||||
|
||||
// Idempotent fast path for repeat deletes — avoids the journal
|
||||
// append entirely so the derived delete_count stays stable.
|
||||
if self.is_needle_deleted(needle_id) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let prev_ecj_size = self.ecj_file_size;
|
||||
let append_result: io::Result<()> = {
|
||||
let ecj_file = self
|
||||
.ecj_file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
needle_id.to_bytes(&mut buf);
|
||||
ecj_file
|
||||
.write_all(&buf)
|
||||
.and_then(|_| ecj_file.sync_all())
|
||||
};
|
||||
|
||||
match append_result {
|
||||
Ok(()) => {
|
||||
self.ecj_file_size += NEEDLE_ID_SIZE as i64;
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.insert(needle_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
// write_all may have extended the file on disk before
|
||||
// sync_all failed; truncate back to the known-good size so
|
||||
// the on-disk journal never drifts past `deleted_needles`.
|
||||
if let Some(ecj) = self.ecj_file.as_mut() {
|
||||
if let Err(trunc_err) = ecj.set_len(prev_ecj_size as u64) {
|
||||
tracing::error!(
|
||||
volume_id = self.volume_id.0,
|
||||
needle_id = needle_id.0,
|
||||
truncate_error = %trunc_err,
|
||||
"failed to truncate ecj after append failure"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal: binary search .ecx without masking by `deleted_needles`.
|
||||
/// Used by `journal_delete` so a repeat delete can still see the raw
|
||||
/// pre-existing .ecx tombstone from a prior rebuild.
|
||||
fn find_needle_from_ecx_raw(
|
||||
&self,
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<Option<(Offset, Size)>> {
|
||||
let ecx_file = self
|
||||
.ecx_file
|
||||
.as_ref()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecx file not open"))?;
|
||||
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
|
||||
if entry_count == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut lo: usize = 0;
|
||||
let mut hi: usize = entry_count;
|
||||
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
|
||||
}
|
||||
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
return Ok(Some((offset, size)));
|
||||
} else if key < needle_id {
|
||||
lo = mid + 1;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
needle_id.to_bytes(&mut buf);
|
||||
ecj_file.write_all(&buf)?;
|
||||
ecj_file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a deleted needle ID to the .ecj journal, validating the cookie first.
|
||||
@@ -1100,17 +778,15 @@ impl EcVolume {
|
||||
let _ = fs::remove_file(format!("{}.ecx", actual_base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", actual_base));
|
||||
let _ = fs::remove_file(format!("{}.vif", actual_base));
|
||||
// Also sweep the originally-configured idx dir in case stale files
|
||||
// exist there (ecx_file_name() / ecj_file_name() now resolve from
|
||||
// ecx_actual_dir, so we have to build the idx-dir paths explicitly).
|
||||
// Also try the configured idx dir and data dir in case files exist in either
|
||||
if self.ecx_actual_dir != self.dir_idx {
|
||||
let _ = fs::remove_file(self.ecx_file_name());
|
||||
let _ = fs::remove_file(self.ecj_file_name());
|
||||
let idx_base = crate::storage::volume::volume_file_name(
|
||||
&self.dir_idx,
|
||||
&self.collection,
|
||||
self.volume_id,
|
||||
);
|
||||
let _ = fs::remove_file(format!("{}.ecx", idx_base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", idx_base));
|
||||
let _ = fs::remove_file(format!("{}.vif", idx_base));
|
||||
}
|
||||
if self.ecx_actual_dir != self.dir && self.dir_idx != self.dir {
|
||||
@@ -1183,33 +859,16 @@ mod tests {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
// .ecj append is gated on a live->tombstone transition in .ecx, so
|
||||
// the fixture must contain the needles we are about to delete.
|
||||
let entries = vec![
|
||||
(NeedleId(10), Offset::from_actual_offset(8), Size(100)),
|
||||
(NeedleId(20), Offset::from_actual_offset(200), Size(200)),
|
||||
];
|
||||
write_ecx_file(dir, "", VolumeId(1), &entries);
|
||||
// Need ecx file for EcVolume::new to succeed
|
||||
write_ecx_file(dir, "", VolumeId(1), &[]);
|
||||
|
||||
let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
|
||||
let (fc0, dc0) = vol.file_and_delete_count();
|
||||
assert_eq!((fc0, dc0), (2, 0));
|
||||
|
||||
vol.journal_delete(NeedleId(10)).unwrap();
|
||||
vol.journal_delete(NeedleId(20)).unwrap();
|
||||
|
||||
let deleted = vol.read_deleted_needles().unwrap();
|
||||
assert_eq!(deleted, vec![NeedleId(10), NeedleId(20)]);
|
||||
|
||||
let (fc, dc) = vol.file_and_delete_count();
|
||||
assert_eq!((fc, dc), (2, 2));
|
||||
|
||||
// Idempotent re-delete must not bump delete_count.
|
||||
vol.journal_delete(NeedleId(10)).unwrap();
|
||||
// Deleting a missing needle must not bump delete_count either.
|
||||
vol.journal_delete(NeedleId(999)).unwrap();
|
||||
let (fc, dc) = vol.file_and_delete_count();
|
||||
assert_eq!((fc, dc), (2, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4,8 +4,6 @@ pub mod idx;
|
||||
pub mod needle;
|
||||
pub mod needle_map;
|
||||
pub mod store;
|
||||
pub mod store_ec_mirror;
|
||||
pub mod store_ec_reconcile;
|
||||
pub mod super_block;
|
||||
pub mod types;
|
||||
pub mod volume;
|
||||
|
||||
@@ -19,7 +19,6 @@ use compact_map::CompactMap;
|
||||
use redb::{Database, Durability, ReadableDatabase, ReadableTable, TableDefinition};
|
||||
|
||||
use crate::storage::idx;
|
||||
use crate::storage::needle::needle::get_actual_size;
|
||||
use crate::storage::types::*;
|
||||
|
||||
// ============================================================================
|
||||
@@ -64,11 +63,6 @@ pub struct NeedleMapMetric {
|
||||
pub deletion_count: AtomicI64,
|
||||
pub deletion_byte_count: AtomicU64,
|
||||
pub max_file_key: AtomicU64,
|
||||
/// Largest (offset.to_actual_offset() + get_actual_size(size, version))
|
||||
/// observed during the load walk. Used at volume load to verify that no
|
||||
/// .idx entry references bytes past the end of .dat (issue #8928)
|
||||
/// without paying for a second linear scan.
|
||||
pub max_needle_end: AtomicI64,
|
||||
}
|
||||
|
||||
impl NeedleMapMetric {
|
||||
@@ -114,29 +108,6 @@ impl NeedleMapMetric {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Update `max_needle_end` if this entry's (offset + actual size) exceeds
|
||||
/// the running maximum. Skips deleted/zero-offset entries because they
|
||||
/// don't reserve space in .dat.
|
||||
fn maybe_set_max_needle_end(&self, offset: Offset, size: Size, version: Version) {
|
||||
if offset.is_zero() || !size.is_valid() {
|
||||
return;
|
||||
}
|
||||
let end = offset.to_actual_offset() + get_actual_size(size, version);
|
||||
loop {
|
||||
let current = self.max_needle_end.load(Ordering::Relaxed);
|
||||
if end <= current {
|
||||
break;
|
||||
}
|
||||
if self
|
||||
.max_needle_end
|
||||
.compare_exchange(current, end, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -146,9 +117,9 @@ impl NeedleMapMetric {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NeedleMapKind {
|
||||
InMemory,
|
||||
Redb,
|
||||
RedbMedium,
|
||||
RedbLarge,
|
||||
LevelDb,
|
||||
LevelDbMedium,
|
||||
LevelDbLarge,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -192,10 +163,9 @@ impl CompactNeedleMap {
|
||||
}
|
||||
|
||||
/// Load from an .idx file, building the in-memory map.
|
||||
pub fn load_from_idx<R: Read + Seek>(reader: &mut R, version: Version) -> io::Result<Self> {
|
||||
pub fn load_from_idx<R: Read + Seek>(reader: &mut R) -> io::Result<Self> {
|
||||
let mut nm = CompactNeedleMap::new();
|
||||
idx::walk_index_file(reader, 0, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
nm.delete_from_map(key);
|
||||
} else {
|
||||
@@ -314,12 +284,6 @@ impl CompactNeedleMap {
|
||||
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
self.metric.max_needle_end.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
self.idx_file_offset
|
||||
}
|
||||
@@ -467,7 +431,7 @@ impl RedbNeedleMap {
|
||||
|
||||
/// Rebuild metrics by scanning all entries in the redb table.
|
||||
/// Called when reusing an existing .rdb without a full rebuild.
|
||||
fn rebuild_metrics_from_db(&self, version: Version) -> io::Result<()> {
|
||||
fn rebuild_metrics_from_db(&self) -> io::Result<()> {
|
||||
let txn = self
|
||||
.db
|
||||
.begin_read()
|
||||
@@ -489,8 +453,6 @@ impl RedbNeedleMap {
|
||||
arr.copy_from_slice(bytes);
|
||||
let nv = unpack_needle_value(&arr);
|
||||
self.metric.maybe_set_max_file_key(key);
|
||||
self.metric
|
||||
.maybe_set_max_needle_end(nv.offset, nv.size, version);
|
||||
if nv.size.is_valid() {
|
||||
self.metric.file_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.metric
|
||||
@@ -515,24 +477,20 @@ impl RedbNeedleMap {
|
||||
/// 2. If .idx size matches → reuse .rdb, rebuild metrics from scan
|
||||
/// 3. If .idx is larger → replay new entries incrementally
|
||||
/// 4. Otherwise (missing, corrupted, .idx smaller) → full rebuild
|
||||
pub fn load_from_idx<R: Read + Seek>(
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
pub fn load_from_idx<R: Read + Seek>(db_path: &str, reader: &mut R) -> io::Result<Self> {
|
||||
let idx_size = reader.seek(io::SeekFrom::End(0))?;
|
||||
reader.seek(io::SeekFrom::Start(0))?;
|
||||
|
||||
// Try to reuse existing .rdb
|
||||
if Path::new(db_path).exists() {
|
||||
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size, version) {
|
||||
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size) {
|
||||
return Ok(nm);
|
||||
}
|
||||
// Reuse failed — fall through to full rebuild
|
||||
reader.seek(io::SeekFrom::Start(0))?;
|
||||
}
|
||||
|
||||
Self::full_rebuild(db_path, reader, idx_size, version)
|
||||
Self::full_rebuild(db_path, reader, idx_size)
|
||||
}
|
||||
|
||||
/// Try to reuse an existing .rdb file. Returns Ok if successful,
|
||||
@@ -541,7 +499,6 @@ impl RedbNeedleMap {
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
idx_size: u64,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
let db = Database::open(db_path)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("redb open: {}", e)))?;
|
||||
@@ -566,7 +523,7 @@ impl RedbNeedleMap {
|
||||
}
|
||||
|
||||
// Rebuild metrics from existing data
|
||||
nm.rebuild_metrics_from_db(version)?;
|
||||
nm.rebuild_metrics_from_db()?;
|
||||
|
||||
if stored_idx_size < idx_size {
|
||||
// .idx grew — replay new entries incrementally
|
||||
@@ -577,7 +534,6 @@ impl RedbNeedleMap {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb open_table: {}", e))
|
||||
})?;
|
||||
idx::walk_index_file(reader, start_entry, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
let key_u64: u64 = key.into();
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
// Delete: look up old value for metric update, then
|
||||
@@ -651,7 +607,6 @@ impl RedbNeedleMap {
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
idx_size: u64,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
let _ = std::fs::remove_file(db_path);
|
||||
let nm = RedbNeedleMap::new(db_path)?;
|
||||
@@ -659,7 +614,6 @@ impl RedbNeedleMap {
|
||||
// Collect entries from idx file, resolving duplicates/deletions
|
||||
let mut entries: HashMap<NeedleId, Option<NeedleValue>> = HashMap::new();
|
||||
idx::walk_index_file(reader, 0, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
entries.insert(key, None);
|
||||
} else {
|
||||
@@ -834,12 +788,6 @@ impl RedbNeedleMap {
|
||||
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
self.metric.max_needle_end.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
self.idx_file_offset
|
||||
}
|
||||
@@ -1040,16 +988,6 @@ impl NeedleMap {
|
||||
}
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. Used at volume load to detect .idx entries that
|
||||
/// reference past the end of .dat (issue #8928) without a second scan.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
match self {
|
||||
NeedleMap::InMemory(nm) => nm.max_needle_end(),
|
||||
NeedleMap::Redb(nm) => nm.max_needle_end(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Index file size in bytes.
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
match self {
|
||||
@@ -1219,7 +1157,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let mut cursor = Cursor::new(idx_data);
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut cursor, Version::current()).unwrap();
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut cursor).unwrap();
|
||||
|
||||
assert!(nm.get(NeedleId(1)).is_some());
|
||||
assert!(nm.get(NeedleId(2)).is_none()); // deleted
|
||||
@@ -1362,7 +1300,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let mut cursor = Cursor::new(idx_data);
|
||||
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor, Version::current()).unwrap();
|
||||
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor).unwrap();
|
||||
|
||||
assert!(nm.get(NeedleId(1)).is_some());
|
||||
assert!(nm.get(NeedleId(2)).is_none()); // deleted and removed
|
||||
@@ -1439,7 +1377,7 @@ mod tests {
|
||||
|
||||
// Load back with CompactNeedleMap to verify
|
||||
let mut idx_file = std::fs::File::open(&idx_path).unwrap();
|
||||
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file, Version::current()).unwrap();
|
||||
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file).unwrap();
|
||||
assert_eq!(loaded.file_count(), 2); // only live entries
|
||||
assert!(loaded.get(NeedleId(1)).is_some());
|
||||
assert!(loaded.get(NeedleId(2)).is_none()); // deleted, not saved
|
||||
|
||||
@@ -84,30 +84,6 @@ impl Store {
|
||||
}
|
||||
|
||||
self.locations.push(loc);
|
||||
|
||||
// First scrub partial EC artefacts left on one disk by an
|
||||
// interrupted encode while the source .dat still lives on a
|
||||
// sibling disk of the same store. The per-disk loader cannot
|
||||
// see the sibling .dat and so loads the partial shards as if
|
||||
// they were a distributed-EC layout, which makes the volume
|
||||
// server heartbeat both a regular replica and an EC shard set
|
||||
// for the same vid (seaweedfs/seaweedfs#9478). Running before
|
||||
// the cross-disk reconcile keeps that pass from later
|
||||
// re-loading shards we just cleaned up.
|
||||
self.prune_incomplete_ec_with_sibling_dat();
|
||||
|
||||
// Physically mirror EC sidecars onto every shard-bearing disk
|
||||
// so each disk mounts self-contained. Must run before the
|
||||
// cross-disk reconciler so the orphan pass can prefer the
|
||||
// local idx_directory.
|
||||
self.mirror_ec_metadata_to_shard_disks();
|
||||
|
||||
// Cross-disk fallback for orphan shards — ec.balance can land
|
||||
// shards on one disk while leaving the index on another. Still
|
||||
// needed after the mirror pass for volumes whose mirror failed
|
||||
// (read-only target, partial copy).
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -119,9 +95,6 @@ impl Store {
|
||||
tracing::error!("load_new_volumes error in {}: {}", loc.directory, e);
|
||||
}
|
||||
}
|
||||
self.prune_incomplete_ec_with_sibling_dat();
|
||||
self.mirror_ec_metadata_to_shard_disks();
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
}
|
||||
|
||||
// ---- Volume lookup ----
|
||||
@@ -224,72 +197,6 @@ impl Store {
|
||||
best.map(|(i, _)| i)
|
||||
}
|
||||
|
||||
/// Returns the index of the disk that should receive a new EC
|
||||
/// shard / index file for `(collection, vid)`. Selection order:
|
||||
///
|
||||
/// 1. a disk that already has the EC volume mounted (in-memory state),
|
||||
/// 2. a disk that owns the `.ecx` file on disk (volume not yet mounted),
|
||||
/// 3. any HDD with free space,
|
||||
/// 4. any disk with free space.
|
||||
///
|
||||
/// Step 2 is the missing primitive that pinned subsequent shards to
|
||||
/// the first-shard disk during `ec.rebuild`: rebuild only sets
|
||||
/// `CopyEcxFile=true` on the first shard, then relies on auto-select
|
||||
/// to land later shards on the same disk. Without an on-disk check,
|
||||
/// `has_ec_volume` returns false (no mount yet) and the fallback
|
||||
/// picks "any HDD with free space" — which can split shards from
|
||||
/// their index files across disks of the same node and lose them at
|
||||
/// startup. See seaweedfs/seaweedfs#9212.
|
||||
///
|
||||
/// `data_shard_count` is taken as a parameter rather than read from
|
||||
/// `DATA_SHARDS_COUNT` so custom-ratio builds can swap the default
|
||||
/// without touching this helper.
|
||||
///
|
||||
/// Single pass over `self.locations` with tier scoring; the
|
||||
/// highest-tier disk wins, ties broken by free shard-slot count.
|
||||
/// Mirrors `Store.FindEcShardTargetLocation` in
|
||||
/// `weed/storage/store_ec.go`.
|
||||
pub fn find_ec_shard_target_location(
|
||||
&self,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
data_shard_count: u32,
|
||||
) -> Option<usize> {
|
||||
const TIER_ANY_DISK: u8 = 1;
|
||||
const TIER_HDD: u8 = 2;
|
||||
const TIER_ECX_ON_DISK: u8 = 3;
|
||||
const TIER_MOUNTED: u8 = 4;
|
||||
|
||||
let mut best: Option<(usize, u8, i64)> = None;
|
||||
for (i, loc) in self.locations.iter().enumerate() {
|
||||
if loc.is_disk_space_low.load(Ordering::Relaxed) {
|
||||
continue;
|
||||
}
|
||||
let free = ec_free_shard_count(loc, data_shard_count);
|
||||
if free <= 0 {
|
||||
continue;
|
||||
}
|
||||
let mut tier = TIER_ANY_DISK;
|
||||
if loc.disk_type == DiskType::HardDrive {
|
||||
tier = TIER_HDD;
|
||||
}
|
||||
if loc.has_ecx_file_on_disk(collection, vid) {
|
||||
tier = TIER_ECX_ON_DISK;
|
||||
}
|
||||
if loc.has_ec_volume(vid) {
|
||||
tier = TIER_MOUNTED;
|
||||
}
|
||||
let better = match best {
|
||||
None => true,
|
||||
Some((_, b_tier, b_free)) => tier > b_tier || (tier == b_tier && free > b_free),
|
||||
};
|
||||
if better {
|
||||
best = Some((i, tier, free));
|
||||
}
|
||||
}
|
||||
best.map(|(i, _, _)| i)
|
||||
}
|
||||
|
||||
/// Create a new volume, placing it on the location with the most free space.
|
||||
pub fn add_volume(
|
||||
&mut self,
|
||||
@@ -322,18 +229,11 @@ impl Store {
|
||||
)
|
||||
}
|
||||
|
||||
/// Delete a volume from any location. When keep_remote_data is true the
|
||||
/// cloud-tier object backing the volume is left intact — used by moves
|
||||
/// where another server is taking over the same .vif.
|
||||
pub fn delete_volume(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
only_empty: bool,
|
||||
keep_remote_data: bool,
|
||||
) -> Result<(), VolumeError> {
|
||||
/// Delete a volume from any location.
|
||||
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
|
||||
for loc in &mut self.locations {
|
||||
if loc.find_volume(vid).is_some() {
|
||||
return loc.delete_volume(vid, only_empty, keep_remote_data);
|
||||
return loc.delete_volume(vid, only_empty);
|
||||
}
|
||||
}
|
||||
Err(VolumeError::NotFound)
|
||||
@@ -592,8 +492,7 @@ impl Store {
|
||||
let vol_count = loc.volumes_len() as i32;
|
||||
let loc_ec_shards = loc.ec_shard_count();
|
||||
let ec_equivalent = ((loc_ec_shards
|
||||
+ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT
|
||||
- 1)
|
||||
+ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT)
|
||||
/ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT)
|
||||
as i32;
|
||||
let mut max_count = vol_count + ec_equivalent;
|
||||
@@ -651,25 +550,22 @@ impl Store {
|
||||
))
|
||||
})?;
|
||||
|
||||
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids, "")
|
||||
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids)
|
||||
}
|
||||
|
||||
/// Mount a single EC shard, searching all locations for the shard file.
|
||||
/// Matches Go's Store.MountEcShards which mounts one shard at a time.
|
||||
/// `source_disk_type` is the source volume's disk type from the
|
||||
/// `VolumeEcShardsMount` RPC (#9423); pass `""` for non-RPC paths.
|
||||
pub fn mount_ec_shard(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_id: u32,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
for loc in &mut self.locations {
|
||||
// Check if the shard file exists on this location
|
||||
let shard = EcVolumeShard::new(&loc.directory, collection, vid, shard_id as u8);
|
||||
if std::path::Path::new(&shard.file_name()).exists() {
|
||||
loc.mount_ec_shards(vid, collection, &[shard_id], source_disk_type)?;
|
||||
loc.mount_ec_shards(vid, collection, &[shard_id])?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -680,18 +576,11 @@ impl Store {
|
||||
}
|
||||
|
||||
/// Unmount EC shards for a volume (batch).
|
||||
///
|
||||
/// Iterates every location that has an EcVolume for `vid` and
|
||||
/// asks it to unmount whatever subset of `shard_ids` it actually
|
||||
/// has — required for cross-disk reconciled volumes where the
|
||||
/// requested shards may legitimately live on different disks of
|
||||
/// the same store (#9252). DiskLocation::unmount_ec_shards
|
||||
/// already skips shards that aren't mounted, so this is safe to
|
||||
/// fan out blindly.
|
||||
pub fn unmount_ec_shards(&mut self, vid: VolumeId, shard_ids: &[u32]) {
|
||||
for loc in &mut self.locations {
|
||||
if loc.has_ec_volume(vid) {
|
||||
loc.unmount_ec_shards(vid, shard_ids);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -699,23 +588,11 @@ impl Store {
|
||||
/// Unmount a single EC shard, searching all locations.
|
||||
/// Matches Go's Store.UnmountEcShards which unmounts one shard at a time.
|
||||
pub fn unmount_ec_shard(&mut self, vid: VolumeId, shard_id: u32) -> Result<(), VolumeError> {
|
||||
// Walk all locations rather than stopping at the first with the
|
||||
// vid — split-disk reconciled volumes can have the same vid on
|
||||
// multiple disks, with the target shard on any of them.
|
||||
for disk_id in 0..self.locations.len() {
|
||||
let has_shard = self.locations[disk_id]
|
||||
.find_ec_volume(vid)
|
||||
.is_some_and(|ec_vol| ec_vol.has_shard(shard_id as u8));
|
||||
if !has_shard {
|
||||
continue;
|
||||
for loc in &mut self.locations {
|
||||
if loc.has_ec_volume(vid) {
|
||||
loc.unmount_ec_shards(vid, &[shard_id]);
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!(
|
||||
volume_id = vid.0,
|
||||
shard_id,
|
||||
disk_id,
|
||||
"UnmountEcShards"
|
||||
);
|
||||
self.locations[disk_id].unmount_ec_shards(vid, &[shard_id]);
|
||||
}
|
||||
// Go returns nil if shard not found (no error)
|
||||
Ok(())
|
||||
@@ -746,90 +623,6 @@ impl Store {
|
||||
self.locations.iter().any(|loc| loc.has_ec_volume(vid))
|
||||
}
|
||||
|
||||
/// Returns every disk_id on this store that has an EcVolume entry
|
||||
/// for `vid`. Useful for diagnostic logging when a single
|
||||
/// `has_ec_volume` hit hides which disk is actually holding the
|
||||
/// mount (e.g., the ReceiveFile mounted-volume guard).
|
||||
/// Mirrors Go's `Store.FindEcVolumeDiskIds`.
|
||||
pub fn find_ec_volume_disk_ids(&self, vid: VolumeId) -> Vec<u32> {
|
||||
let mut ids = Vec::new();
|
||||
for (idx, loc) in self.locations.iter().enumerate() {
|
||||
if loc.has_ec_volume(vid) {
|
||||
ids.push(idx as u32);
|
||||
}
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
/// Returns the index of the disk location that has `(vid, shard_id)`
|
||||
/// mounted, if any. Mirrors Go's `Store.findEcShard` and is the
|
||||
/// right primitive for read/unmount/delete operations on a single
|
||||
/// shard, since reconciliation can mount the same `vid` on multiple
|
||||
/// disks (each holding a disjoint subset of the shards). Without
|
||||
/// this, callers using `find_ec_volume(vid)` would only see the
|
||||
/// first disk and miss shards that live on a sibling.
|
||||
pub fn find_ec_shard_location(&self, vid: VolumeId, shard_id: u32) -> Option<usize> {
|
||||
for (i, loc) in self.locations.iter().enumerate() {
|
||||
if let Some(ecv) = loc.find_ec_volume(vid) {
|
||||
if ecv.has_shard(shard_id as u8) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Like [`Self::find_ec_shard_location`] but returns the EcVolume
|
||||
/// reference directly. Borrows the store immutably for the
|
||||
/// EcVolume's lifetime.
|
||||
pub fn find_ec_volume_with_shard(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
shard_id: u32,
|
||||
) -> Option<&EcVolume> {
|
||||
for loc in &self.locations {
|
||||
if let Some(ecv) = loc.find_ec_volume(vid) {
|
||||
if ecv.has_shard(shard_id as u8) {
|
||||
return Some(ecv);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Aggregate the data dir of each mounted shard for `vid` across
|
||||
/// all locations. Returns the EcVolume to use for metadata
|
||||
/// (`.ecx`, `dir_idx`, etc — any per-disk EcVolume for this vid
|
||||
/// will do, since they all open the same `.ecx`) and a vector of
|
||||
/// per-shard data dirs (`None` when the shard isn't mounted on
|
||||
/// any disk).
|
||||
///
|
||||
/// Mirrors Go's `Store.CollectEcShards` in
|
||||
/// `weed/storage/store_ec.go`. The decoder needs per-shard paths
|
||||
/// to support cross-disk reconciled volumes whose shards are
|
||||
/// split across data dirs.
|
||||
pub fn collect_ec_shard_dirs(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
max_shard_count: usize,
|
||||
) -> Option<(&EcVolume, Vec<Option<String>>)> {
|
||||
let mut found_vol: Option<&EcVolume> = None;
|
||||
let mut dirs: Vec<Option<String>> = vec![None; max_shard_count];
|
||||
for loc in &self.locations {
|
||||
if let Some(ecv) = loc.find_ec_volume(vid) {
|
||||
if found_vol.is_none() {
|
||||
found_vol = Some(ecv);
|
||||
}
|
||||
for shard_id in 0..max_shard_count {
|
||||
if dirs[shard_id].is_none() && ecv.has_shard(shard_id as u8) {
|
||||
dirs[shard_id] = Some(loc.directory.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
found_vol.map(|v| (v, dirs))
|
||||
}
|
||||
|
||||
pub fn delete_expired_ec_volumes(
|
||||
&mut self,
|
||||
) -> (
|
||||
@@ -1126,48 +919,6 @@ fn save_vif_volume_info(path: &str, info: &VifVolumeInfo) -> Result<(), VolumeEr
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Free EC shard capacity of `loc`, expressed in shard slots (not
|
||||
/// volume-equivalent slots). `find_free_location_predicate` does similar
|
||||
/// math but divides by `data_shard_count` at the end. That truncation can
|
||||
/// exclude a disk that still has room for several individual shards
|
||||
/// (e.g. `MaxVolumeCount=1`, `EcShardCount=1`, `data_shard_count=10` →
|
||||
/// reports 0 despite 9 free shard slots), which would re-route subsequent
|
||||
/// shards off the `.ecx`-owning disk and re-introduce the orphan-shard
|
||||
/// layout this helper is meant to prevent (seaweedfs/seaweedfs#9212).
|
||||
///
|
||||
/// `MaxVolumeCount == 0` is the "unlimited" sentinel honoured elsewhere
|
||||
/// in the store; report a synthetic large free count decremented by
|
||||
/// current usage so unlimited disks are eligible and tie-breaks still
|
||||
/// prefer the less-loaded one.
|
||||
///
|
||||
/// Mirrors `ecFreeShardCount` in `weed/storage/store_ec.go`.
|
||||
fn ec_free_shard_count(loc: &DiskLocation, data_shard_count: u32) -> i64 {
|
||||
if data_shard_count == 0 {
|
||||
return 0;
|
||||
}
|
||||
let dsc = data_shard_count as i64;
|
||||
let max = loc.max_volume_count.load(Ordering::Relaxed) as i64;
|
||||
if max <= 0 {
|
||||
// Synthetic "unlimited" capacity. Use a large but
|
||||
// well-below-overflow base (1 << 60 ≈ 1.15e18) and saturating
|
||||
// arithmetic so even pathological usage never wraps and
|
||||
// tie-breaks among unlimited disks still meaningfully prefer
|
||||
// the less-loaded one. Clamp to ≥ 1 so the disk stays eligible
|
||||
// for placement no matter how loaded it is.
|
||||
const UNLIMITED_FREE: i64 = 1 << 60;
|
||||
let used = (loc.volumes_len() as i64)
|
||||
.saturating_mul(dsc)
|
||||
.saturating_add(loc.ec_shard_count() as i64);
|
||||
return UNLIMITED_FREE.saturating_sub(used).max(1);
|
||||
}
|
||||
let mut free = (max - loc.volumes_len() as i64) * dsc;
|
||||
free -= loc.ec_shard_count() as i64;
|
||||
if free < 0 {
|
||||
return 0;
|
||||
}
|
||||
free
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
@@ -1176,7 +927,6 @@ fn ec_free_shard_count(loc: &DiskLocation, data_shard_count: u32) -> i64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::storage::needle::needle::Needle;
|
||||
use crate::storage::volume::volume_file_name;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_test_store(dirs: &[&str]) -> Store {
|
||||
@@ -1408,13 +1158,7 @@ mod tests {
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// Use a large volume_size_limit so the unused-space difference between
|
||||
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
|
||||
// that integer-division rounding and disk-free fluctuations between the
|
||||
// two get_disk_stats calls cannot make the quotients equal.
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(100 * 1024 * 1024, Ordering::Relaxed);
|
||||
store.volume_size_limit.store(1024, Ordering::Relaxed);
|
||||
store
|
||||
.add_volume(
|
||||
VolumeId(61),
|
||||
@@ -1524,7 +1268,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/expired_ec_case_9.ec00", dir), b"expired").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0])
|
||||
.unwrap();
|
||||
store.find_ec_volume_mut(VolumeId(9)).unwrap().expire_at_sec = 1;
|
||||
|
||||
@@ -1550,151 +1294,4 @@ mod tests {
|
||||
let err = store.read_volume_needle(VolumeId(99), &mut n);
|
||||
assert!(matches!(err, Err(VolumeError::NotFound)));
|
||||
}
|
||||
|
||||
/// Build a Store with N HDD disk locations under a single TempDir.
|
||||
/// Returns the store and the TempDir guard so callers keep the dirs
|
||||
/// alive for the test's lifetime.
|
||||
fn make_ec_target_test_store(numdirs: usize) -> (Store, TempDir) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
for i in 0..numdirs {
|
||||
let path = tmp.path().join(format!("data{}", i));
|
||||
std::fs::create_dir_all(&path).unwrap();
|
||||
store
|
||||
.add_location(
|
||||
path.to_str().unwrap(),
|
||||
path.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
(store, tmp)
|
||||
}
|
||||
|
||||
/// Reproduces the placement half of seaweedfs/seaweedfs#9212. After
|
||||
/// `ec.rebuild`'s first VolumeEcShardsCopy lands `.ecx` on disk N,
|
||||
/// subsequent shards arrive with `CopyEcxFile=false` and rely on
|
||||
/// auto-select. Without an on-disk check, `has_ec_volume` returns
|
||||
/// false (no mount yet) and the fallback picks "any HDD with free
|
||||
/// space" — splitting shards from their index files across disks.
|
||||
/// `find_ec_shard_target_location` must pin to the `.ecx`-owning
|
||||
/// disk via the on-disk check.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_pins_to_ecx_on_disk() {
|
||||
let (store, _tmp) = make_ec_target_test_store(3);
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(1093);
|
||||
|
||||
// Drop a sealed .ecx onto disk 2. Nothing is mounted yet — this
|
||||
// is the state right after the first VolumeEcShardsCopy with
|
||||
// CopyEcxFile=true and before any VolumeEcShardsMount has run.
|
||||
let base = volume_file_name(&store.locations[2].idx_directory, collection, vid);
|
||||
std::fs::write(format!("{}.ecx", base), vec![0u8; 20]).unwrap();
|
||||
|
||||
let got = store.find_ec_shard_target_location(collection, vid, 10);
|
||||
assert_eq!(
|
||||
got,
|
||||
Some(2),
|
||||
"placement leaked off the .ecx-owning disk; got {:?}",
|
||||
got,
|
||||
);
|
||||
}
|
||||
|
||||
/// An already-mounted EC volume on disk 1 must win over a stray
|
||||
/// `.ecx` on disk 2. Protects the post-startup steady state from
|
||||
/// being perturbed by leftover index files from a prior failed move.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_prefers_mounted_over_ecx() {
|
||||
let (mut store, _tmp) = make_ec_target_test_store(3);
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(2222);
|
||||
|
||||
// Mount an EC shard on disk 1 so has_ec_volume returns true.
|
||||
std::fs::write(
|
||||
format!("{}/{}_{}.ec00", store.locations[1].directory, collection, vid.0),
|
||||
b"shard data",
|
||||
)
|
||||
.unwrap();
|
||||
store.locations[1]
|
||||
.mount_ec_shards(vid, collection, &[0], "")
|
||||
.unwrap();
|
||||
|
||||
// Stray .ecx on disk 2 must not win.
|
||||
let base = volume_file_name(&store.locations[2].idx_directory, collection, vid);
|
||||
std::fs::write(format!("{}.ecx", base), vec![0u8; 20]).unwrap();
|
||||
|
||||
let got = store.find_ec_shard_target_location(collection, vid, 10);
|
||||
assert_eq!(got, Some(1), "expected the mounted disk to win; got {:?}", got);
|
||||
}
|
||||
|
||||
/// Cold-volume case: no mount, no `.ecx` anywhere on this server.
|
||||
/// Selection should still fall through to an HDD fallback.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_falls_through_to_hdd_when_nothing_matches() {
|
||||
let (store, _tmp) = make_ec_target_test_store(2);
|
||||
let got = store.find_ec_shard_target_location("grafana-loki", VolumeId(3333), 10);
|
||||
assert!(got.is_some(), "expected an HDD fallback");
|
||||
assert_eq!(store.locations[got.unwrap()].disk_type, DiskType::HardDrive);
|
||||
}
|
||||
|
||||
/// `MaxVolumeCount=0` is the "unlimited disk" sentinel. The previous
|
||||
/// formula returned a negative free count for unlimited disks,
|
||||
/// making placement skip them entirely. The unlimited branch in
|
||||
/// `ec_free_shard_count` must report a synthetic large free count.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_honours_unlimited_disk() {
|
||||
let (store, _tmp) = make_ec_target_test_store(1);
|
||||
store.locations[0]
|
||||
.max_volume_count
|
||||
.store(0, Ordering::Relaxed);
|
||||
|
||||
let got = store.find_ec_shard_target_location("grafana-loki", VolumeId(4444), 10);
|
||||
assert_eq!(
|
||||
got,
|
||||
Some(0),
|
||||
"expected the only (unlimited) disk to be picked",
|
||||
);
|
||||
}
|
||||
|
||||
/// Truncation hazard: with `MaxVolumeCount=1, EcShardCount=1,
|
||||
/// data_shard_count=10`, the old formula `(1*10 - 1) / 10 = 0`
|
||||
/// would have rounded the disk to "full" and routed subsequent
|
||||
/// shards elsewhere — the orphan-shard layout this PR exists to
|
||||
/// prevent. Accounting in shard slots fixes it.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_tight_provisioning_keeps_ecx_disk() {
|
||||
let (mut store, _tmp) = make_ec_target_test_store(2);
|
||||
store.locations[0]
|
||||
.max_volume_count
|
||||
.store(1, Ordering::Relaxed);
|
||||
store.locations[1]
|
||||
.max_volume_count
|
||||
.store(1, Ordering::Relaxed);
|
||||
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(5555);
|
||||
|
||||
// Seed disk 1 with one EC shard so it owns .ecx and has 9
|
||||
// free shard slots remaining; the old formula would have
|
||||
// rounded that to 0.
|
||||
std::fs::write(
|
||||
format!("{}/{}_{}.ec00", store.locations[1].directory, collection, vid.0),
|
||||
b"shard data",
|
||||
)
|
||||
.unwrap();
|
||||
store.locations[1]
|
||||
.mount_ec_shards(vid, collection, &[0], "")
|
||||
.unwrap();
|
||||
|
||||
let got = store.find_ec_shard_target_location(collection, vid, 10);
|
||||
assert_eq!(
|
||||
got,
|
||||
Some(1),
|
||||
"expected the .ecx-owning disk (1 shard placed, 9 free shard slots) to be picked; got {:?}",
|
||||
got,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,421 +0,0 @@
|
||||
//! Physical EC sidecar mirroring across disks of the same volume
|
||||
//! server. Mirrors `weed/storage/store_ec_mirror.go`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage::disk_location::{parse_collection_volume_id_pub, DiskLocation};
|
||||
use crate::storage::store::Store;
|
||||
use crate::storage::types::VolumeId;
|
||||
|
||||
// Listed in `EcVolume::new`'s open order.
|
||||
const EC_MIRRORED_SIDECARS: &[&str] = &[".ecx", ".ecj", ".vif"];
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
struct EcKey {
|
||||
collection: String,
|
||||
vid: VolumeId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct EcxOwner {
|
||||
location: usize,
|
||||
idx_dir: String,
|
||||
data_dir: String,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Mirror EC sidecars onto every shard-bearing disk that lacks
|
||||
/// them, so each disk mounts self-contained. Runs before
|
||||
/// `reconcile_ec_shards_across_disks` so the orphan pass can
|
||||
/// prefer the local idx_directory.
|
||||
pub fn mirror_ec_metadata_to_shard_disks(&mut self) {
|
||||
if self.locations.len() < 2 {
|
||||
return;
|
||||
}
|
||||
let owners = self.index_ecx_owners_for_mirror();
|
||||
if owners.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Two-pass: gather work under an immutable borrow, then apply
|
||||
// copies under independent mutable borrows.
|
||||
struct Mirror<'a> {
|
||||
target_idx: usize,
|
||||
owner: &'a EcxOwner,
|
||||
collection: String,
|
||||
vid: VolumeId,
|
||||
}
|
||||
let mut mirrors: Vec<Mirror> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let orphans = collect_shard_disk_volumes(loc);
|
||||
for (key, _shards) in orphans {
|
||||
let Some(owner) = owners.get(&key) else {
|
||||
continue;
|
||||
};
|
||||
if owner.location == loc_idx {
|
||||
continue;
|
||||
}
|
||||
if disk_has_all_sidecars(loc, &key.collection, key.vid) {
|
||||
continue;
|
||||
}
|
||||
mirrors.push(Mirror {
|
||||
target_idx: loc_idx,
|
||||
owner,
|
||||
collection: key.collection,
|
||||
vid: key.vid,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for m in mirrors {
|
||||
let loc_dir = self.locations[m.target_idx].directory.clone();
|
||||
let loc_idx_dir = self.locations[m.target_idx].idx_directory.clone();
|
||||
match mirror_sidecars_for_volume(
|
||||
&m.owner.idx_dir,
|
||||
&m.owner.data_dir,
|
||||
&loc_dir,
|
||||
&loc_idx_dir,
|
||||
&m.collection,
|
||||
m.vid,
|
||||
) {
|
||||
Ok(0) => {}
|
||||
Ok(copied) => {
|
||||
info!(
|
||||
volume_id = m.vid.0,
|
||||
collection = %m.collection,
|
||||
from = %m.owner.data_dir,
|
||||
to = %loc_dir,
|
||||
copied,
|
||||
"mirrored EC sidecar(s) for same-disk invariant",
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
volume_id = m.vid.0,
|
||||
collection = %m.collection,
|
||||
from = %m.owner.data_dir,
|
||||
to = %loc_dir,
|
||||
error = %e,
|
||||
"mirror EC sidecars failed; cross-disk fallback will handle this volume",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Records both idx_dir (where .ecx was found) and data_dir, so
|
||||
// the mirror can resolve .vif from data_dir even when .ecx lives
|
||||
// in idx_directory.
|
||||
fn index_ecx_owners_for_mirror(&self) -> HashMap<EcKey, EcxOwner> {
|
||||
let mut owners: HashMap<EcKey, EcxOwner> = HashMap::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let mut seen: Vec<&str> = Vec::with_capacity(2);
|
||||
for scan in [loc.idx_directory.as_str(), loc.directory.as_str()] {
|
||||
if scan.is_empty() || seen.contains(&scan) {
|
||||
continue;
|
||||
}
|
||||
seen.push(scan);
|
||||
let Ok(read) = fs::read_dir(scan) else {
|
||||
continue;
|
||||
};
|
||||
for ent in read.flatten() {
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
let Some(base) = name.strip_suffix(".ecx") else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
owners
|
||||
.entry(EcKey { collection, vid })
|
||||
.or_insert_with(|| EcxOwner {
|
||||
location: loc_idx,
|
||||
idx_dir: scan.to_string(),
|
||||
data_dir: loc.directory.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
owners
|
||||
}
|
||||
}
|
||||
|
||||
// Checks the modern routing and the opposite directory — without
|
||||
// that fallback, a destination with a legacy pre-`-dir.idx` .ecx in
|
||||
// its data dir would be re-mirrored into idx_directory.
|
||||
fn disk_has_all_sidecars(loc: &DiskLocation, collection: &str, vid: VolumeId) -> bool {
|
||||
for ext in EC_MIRRORED_SIDECARS {
|
||||
let primary = sidecar_dest_path(&loc.directory, &loc.idx_directory, collection, vid, ext);
|
||||
if path_is_regular_file(&primary) {
|
||||
continue;
|
||||
}
|
||||
if loc.idx_directory != loc.directory {
|
||||
let (fallback_data, fallback_idx) = if *ext == ".vif" {
|
||||
(loc.idx_directory.as_str(), loc.directory.as_str())
|
||||
} else {
|
||||
(loc.directory.as_str(), loc.directory.as_str())
|
||||
};
|
||||
let fallback = sidecar_dest_path(fallback_data, fallback_idx, collection, vid, ext);
|
||||
if path_is_regular_file(&fallback) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn path_is_regular_file(path: &str) -> bool {
|
||||
fs::metadata(path).map(|m| !m.is_dir()).unwrap_or(false)
|
||||
}
|
||||
|
||||
// `.ecx`/`.ecj` route to idx_directory, `.vif` to directory.
|
||||
fn sidecar_dest_path(
|
||||
data_dir: &str,
|
||||
idx_dir: &str,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
ext: &str,
|
||||
) -> String {
|
||||
let dir = if ext == ".vif" { data_dir } else { idx_dir };
|
||||
if collection.is_empty() {
|
||||
format!("{}/{}{}", dir, vid.0, ext)
|
||||
} else {
|
||||
format!("{}/{}_{}{}", dir, collection, vid.0, ext)
|
||||
}
|
||||
}
|
||||
|
||||
fn mirror_sidecars_for_volume(
|
||||
src_idx_dir: &str,
|
||||
src_data_dir: &str,
|
||||
dst_data_dir: &str,
|
||||
dst_idx_dir: &str,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<usize> {
|
||||
let mut copied = 0usize;
|
||||
for ext in EC_MIRRORED_SIDECARS {
|
||||
let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext);
|
||||
// An existing local copy is authoritative — it may be newer
|
||||
// than the owner's after a delete journal append.
|
||||
if fs::metadata(&dst).is_ok() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let candidates = [
|
||||
sidecar_dest_path(src_data_dir, src_idx_dir, collection, vid, ext),
|
||||
sidecar_dest_path(src_idx_dir, src_data_dir, collection, vid, ext),
|
||||
];
|
||||
let mut src_path: Option<String> = None;
|
||||
for c in candidates.iter() {
|
||||
if fs::metadata(c).map(|m| !m.is_dir()).unwrap_or(false) {
|
||||
src_path = Some(c.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
let Some(src) = src_path else {
|
||||
continue;
|
||||
};
|
||||
copy_sidecar_atomic(Path::new(&src), Path::new(&dst))?;
|
||||
copied += 1;
|
||||
}
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
if let Some(parent) = dst.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut src_file = fs::File::open(src)?;
|
||||
let tmp = {
|
||||
let mut s = dst.as_os_str().to_owned();
|
||||
s.push(".mirror.tmp");
|
||||
std::path::PathBuf::from(s)
|
||||
};
|
||||
let _ = fs::remove_file(&tmp);
|
||||
let mut dst_file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&tmp)?;
|
||||
let mut buf = [0u8; 64 * 1024];
|
||||
loop {
|
||||
let n = src_file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
dst_file.write_all(&buf[..n])?;
|
||||
}
|
||||
dst_file.sync_all()?;
|
||||
drop(dst_file);
|
||||
if let Err(e) = fs::rename(&tmp, dst) {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(e);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>> {
|
||||
let mut out: HashMap<EcKey, Vec<String>> = HashMap::new();
|
||||
let Ok(read) = fs::read_dir(&loc.directory) else {
|
||||
return out;
|
||||
};
|
||||
for ent in read.flatten() {
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
let Some(dot) = name.rfind('.') else {
|
||||
continue;
|
||||
};
|
||||
let (base, ext) = name.split_at(dot);
|
||||
if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() {
|
||||
continue;
|
||||
}
|
||||
match ent.metadata() {
|
||||
Ok(meta) if meta.len() > 0 => {}
|
||||
_ => continue,
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
out.entry(EcKey { collection, vid })
|
||||
.or_default()
|
||||
.push(name);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::types::DiskType;
|
||||
use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo};
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn plant_shard(dir: &Path, collection: &str, vid: u32, shard_id: u8) {
|
||||
let path = if collection.is_empty() {
|
||||
dir.join(format!("{}.ec{:02}", vid, shard_id))
|
||||
} else {
|
||||
dir.join(format!("{}_{}.ec{:02}", collection, vid, shard_id))
|
||||
};
|
||||
fs::write(&path, b"shard data nonempty").unwrap();
|
||||
}
|
||||
|
||||
fn plant_ecx(dir: &Path, collection: &str, vid: u32, bytes: &[u8]) {
|
||||
let path = dir.join(format!("{}_{}.ecx", collection, vid));
|
||||
fs::write(&path, bytes).unwrap();
|
||||
}
|
||||
|
||||
fn plant_ecj(dir: &Path, collection: &str, vid: u32, bytes: &[u8]) {
|
||||
let path = dir.join(format!("{}_{}.ecj", collection, vid));
|
||||
fs::write(&path, bytes).unwrap();
|
||||
}
|
||||
|
||||
fn plant_vif(dir: &Path, collection: &str, vid: u32, data_shards: u32, parity_shards: u32) {
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards,
|
||||
parity_shards,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let path = dir.join(format!("{}_{}.vif", collection, vid));
|
||||
fs::write(&path, serde_json::to_string(&vif).unwrap()).unwrap();
|
||||
}
|
||||
|
||||
fn add_loc(store: &mut Store, dir: &Path) {
|
||||
store
|
||||
.add_location(
|
||||
dir.to_str().unwrap(),
|
||||
dir.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_copies_sidecars_to_shard_only_disk() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
fs::create_dir_all(&dir0).unwrap();
|
||||
fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "video-recordings";
|
||||
let vid = 4121u32;
|
||||
|
||||
plant_shard(&dir0, collection, vid, 0);
|
||||
plant_shard(&dir0, collection, vid, 12);
|
||||
plant_shard(&dir1, collection, vid, 1);
|
||||
|
||||
let ecx = vec![0xA1u8; 20];
|
||||
let ecj = vec![0xB2u8; 16];
|
||||
plant_ecx(&dir1, collection, vid, &ecx);
|
||||
plant_ecj(&dir1, collection, vid, &ecj);
|
||||
plant_vif(&dir1, collection, vid, 10, 4);
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
for ext in [".ecx", ".ecj", ".vif"] {
|
||||
let dst = dir0.join(format!("{}_{}{}", collection, vid, ext));
|
||||
assert!(
|
||||
dst.exists(),
|
||||
"mirror did not install sidecar {} on dir0",
|
||||
ext
|
||||
);
|
||||
}
|
||||
let ecx_dst = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
|
||||
assert_eq!(ecx_dst, ecx, ".ecx mirrored bytes differ from source");
|
||||
let ecj_dst = fs::read(dir0.join(format!("{}_{}.ecj", collection, vid))).unwrap();
|
||||
assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_preserves_existing_destination_sidecars() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
fs::create_dir_all(&dir0).unwrap();
|
||||
fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "video-recordings";
|
||||
let vid = 7777u32;
|
||||
|
||||
plant_shard(&dir0, collection, vid, 0);
|
||||
plant_shard(&dir0, collection, vid, 12);
|
||||
plant_shard(&dir1, collection, vid, 1);
|
||||
|
||||
let ecx_owner = vec![0xC3u8; 20];
|
||||
let ecx_local = vec![0x5Au8; 20];
|
||||
let ecj_bytes = vec![0xD4u8; 16];
|
||||
plant_ecx(&dir1, collection, vid, &ecx_owner);
|
||||
plant_ecj(&dir1, collection, vid, &ecj_bytes);
|
||||
plant_vif(&dir1, collection, vid, 10, 4);
|
||||
plant_ecx(&dir0, collection, vid, &ecx_local);
|
||||
plant_ecj(&dir0, collection, vid, &ecj_bytes);
|
||||
plant_vif(&dir0, collection, vid, 10, 4);
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
let post = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
|
||||
assert_eq!(post, ecx_local, "mirror overwrote dir0's existing .ecx");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,8 +19,6 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tracing::warn;
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::storage::idx;
|
||||
use crate::storage::needle::needle::{self, get_actual_size, Needle, NeedleError};
|
||||
use crate::storage::needle_map::{CompactNeedleMap, NeedleMap, NeedleMapKind, RedbNeedleMap};
|
||||
use crate::storage::super_block::{ReplicaPlacement, SuperBlock, SUPER_BLOCK_SIZE};
|
||||
@@ -75,24 +73,6 @@ pub enum VolumeError {
|
||||
StreamingUnsupported,
|
||||
}
|
||||
|
||||
/// Returns true when a needle read failed because the on-disk bytes are
|
||||
/// unreadable in a permanent way (offset past EOF, header corruption, CRC
|
||||
/// mismatch, malformed v2/v3/v4 fields). Vacuum can safely drop such entries
|
||||
/// during compaction. Anything else (real disk EIO, ERROR_CRC on Windows,
|
||||
/// network timeouts, EROFS, etc.) is transient or environmental and must
|
||||
/// abort the compaction so an operator notices.
|
||||
fn is_skippable_needle_read_error(e: &VolumeError) -> bool {
|
||||
match e {
|
||||
VolumeError::Io(io_err) => io_err.kind() == io::ErrorKind::UnexpectedEof,
|
||||
VolumeError::Needle(NeedleError::SizeMismatch { .. }) => true,
|
||||
VolumeError::Needle(NeedleError::CrcMismatch { .. }) => true,
|
||||
VolumeError::Needle(NeedleError::IndexOutOfRange(_)) => true,
|
||||
VolumeError::Needle(NeedleError::TailTooShort) => true,
|
||||
VolumeError::SizeMismatch => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// VolumeInfo (.vif persistence)
|
||||
// ============================================================================
|
||||
@@ -732,28 +712,6 @@ impl Volume {
|
||||
"volumeDataIntegrityChecking failed"
|
||||
);
|
||||
}
|
||||
|
||||
// Structural check: no .idx entry may reference bytes past the
|
||||
// end of .dat. The needle map's load walk above already
|
||||
// populated max_needle_end, so this is a numeric comparison
|
||||
// — no extra disk I/O. A violation marks the volume read-only
|
||||
// so vacuum doesn't silently drop reachable data based on a
|
||||
// corrupt .idx left over from a crashed batched write.
|
||||
// See issue #8928.
|
||||
if let Some(ref nm) = self.nm {
|
||||
if let Ok(dat_size) = self.current_dat_file_size() {
|
||||
let max_end = nm.max_needle_end();
|
||||
if dat_size > 0 && max_end > dat_size as i64 {
|
||||
self.no_write_or_delete = true;
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
max_needle_end = max_end,
|
||||
dat_size,
|
||||
"idx references bytes past end of .dat; marking volume read-only"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -773,17 +731,10 @@ impl Volume {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn nm_or_not_found(&self) -> Result<&NeedleMap, VolumeError> {
|
||||
self.nm.as_ref().ok_or_else(|| {
|
||||
tracing::warn!(volume_id = self.id.0, "needle map not loaded");
|
||||
VolumeError::NotFound
|
||||
})
|
||||
}
|
||||
|
||||
fn load_index(&mut self) -> Result<(), VolumeError> {
|
||||
let use_redb = matches!(
|
||||
self.needle_map_kind,
|
||||
NeedleMapKind::Redb | NeedleMapKind::RedbMedium | NeedleMapKind::RedbLarge
|
||||
NeedleMapKind::LevelDb | NeedleMapKind::LevelDbMedium | NeedleMapKind::LevelDbLarge
|
||||
);
|
||||
|
||||
let idx_path = self.file_name(".idx");
|
||||
@@ -808,7 +759,7 @@ impl Volume {
|
||||
// Open read-only
|
||||
if Path::new(&idx_path).exists() {
|
||||
let mut idx_file = File::open(&idx_path)?;
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut idx_file, self.version())?;
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut idx_file)?;
|
||||
self.nm = Some(NeedleMap::InMemory(nm));
|
||||
} else {
|
||||
// Missing .idx with existing .dat could orphan needles
|
||||
@@ -834,7 +785,7 @@ impl Volume {
|
||||
|
||||
let idx_size = idx_file.metadata()?.len();
|
||||
let mut idx_reader = io::BufReader::new(&idx_file);
|
||||
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader, self.version())?;
|
||||
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader)?;
|
||||
|
||||
// Re-open for append-only writes
|
||||
let write_file = OpenOptions::new()
|
||||
@@ -857,7 +808,7 @@ impl Volume {
|
||||
// Open read-only
|
||||
if Path::new(&idx_path).exists() {
|
||||
let mut idx_file = File::open(&idx_path)?;
|
||||
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file, self.version())?;
|
||||
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file)?;
|
||||
self.nm = Some(NeedleMap::Redb(nm));
|
||||
} else {
|
||||
// Missing .idx with existing .dat could orphan needles
|
||||
@@ -883,7 +834,7 @@ impl Volume {
|
||||
|
||||
let idx_size = idx_file.metadata()?.len();
|
||||
let mut idx_reader = io::BufReader::new(&idx_file);
|
||||
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader, self.version())?;
|
||||
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader)?;
|
||||
|
||||
// Re-open for append-only writes
|
||||
let write_file = OpenOptions::new()
|
||||
@@ -1065,7 +1016,7 @@ impl Volume {
|
||||
read_option: &mut ReadOption,
|
||||
) -> Result<i32, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
|
||||
|
||||
if nv.offset.is_zero() {
|
||||
@@ -1307,7 +1258,7 @@ impl Volume {
|
||||
read_deleted: bool,
|
||||
) -> Result<NeedleStreamInfo, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
|
||||
|
||||
if nv.offset.is_zero() {
|
||||
@@ -1409,7 +1360,7 @@ impl Volume {
|
||||
&self,
|
||||
needle_id: NeedleId,
|
||||
) -> Result<(u64, u16), VolumeError> {
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(needle_id).ok_or(VolumeError::NotFound)?;
|
||||
if nv.offset.is_zero() {
|
||||
return Err(VolumeError::NotFound);
|
||||
@@ -1726,7 +1677,7 @@ impl Volume {
|
||||
/// Read all live needles from the volume (for ReadAllNeedles streaming RPC).
|
||||
pub fn read_all_needles(&self) -> Result<Vec<Needle>, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let version = self.version();
|
||||
let dat_size = self.current_dat_file_size()? as i64;
|
||||
let mut needles = Vec::new();
|
||||
@@ -1801,11 +1752,6 @@ impl Volume {
|
||||
|
||||
let version = self.version();
|
||||
|
||||
// The deeper-than-tail structural check (every (offset + actual size)
|
||||
// fits inside .dat — issue #8928) is now handled in load() via the
|
||||
// needle map's max_needle_end accumulator, so we don't pay for a
|
||||
// second linear scan of the .idx here.
|
||||
|
||||
// Check last 10 index entries (matching Go's CheckVolumeDataIntegrity).
|
||||
// Go starts healthyIndexSize = indexSize and reduces on EOF.
|
||||
// On success: break (err != ErrorSizeMismatch when err == nil).
|
||||
@@ -1915,7 +1861,7 @@ impl Volume {
|
||||
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let dat_size = self.dat_file_size().map_err(VolumeError::Io)?;
|
||||
|
||||
let mut files_checked: u64 = 0;
|
||||
@@ -1975,7 +1921,7 @@ impl Volume {
|
||||
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
|
||||
let dat_size = self.dat_file_size().map_err(|e| VolumeError::Io(e))?;
|
||||
let version = self.version();
|
||||
@@ -2737,8 +2683,6 @@ impl Volume {
|
||||
}
|
||||
entries.sort_by_key(|(_, offset, _)| *offset);
|
||||
|
||||
let mut skipped_needles: u64 = 0;
|
||||
let mut skipped_data_bytes: u64 = 0;
|
||||
for (id, offset, size) in entries {
|
||||
// Progress callback
|
||||
if !progress_fn(offset.to_actual_offset()) {
|
||||
@@ -2755,41 +2699,7 @@ impl Volume {
|
||||
id,
|
||||
..Needle::default()
|
||||
};
|
||||
match self.read_needle_data_at(&mut n, offset.to_actual_offset(), size) {
|
||||
Ok(()) => {}
|
||||
Err(e) => {
|
||||
// Record EIO for health monitoring (parity with Go's checkReadWriteError).
|
||||
if let VolumeError::Io(ref io_err) = e {
|
||||
self.check_read_write_error(Some(io_err));
|
||||
}
|
||||
// Only drop the entry when the failure is one of the well-
|
||||
// known permanent-corruption shapes. A transient disk fault,
|
||||
// a tiered-read timeout, or a Windows hardware error (which
|
||||
// surfaces as a generic Io rather than UnexpectedEof) must
|
||||
// abort so an operator notices, rather than silently
|
||||
// compacting away data that might come back on retry.
|
||||
// See issue #8928.
|
||||
if !is_skippable_needle_read_error(&e) {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("cannot hydrate needle from file: {}", e),
|
||||
)));
|
||||
}
|
||||
skipped_needles += 1;
|
||||
if size.is_valid() {
|
||||
skipped_data_bytes += size.0 as u64;
|
||||
}
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
key = id.0,
|
||||
offset = offset.to_actual_offset(),
|
||||
size = size.0,
|
||||
error = %e,
|
||||
"vacuum: dropping unreadable needle"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
self.read_needle_data_at(&mut n, offset.to_actual_offset(), size)?;
|
||||
|
||||
// Skip TTL-expired needles using the volume's TTL (matches Go's volume_vacuum.go)
|
||||
if n.has_ttl() {
|
||||
@@ -2811,15 +2721,6 @@ impl Volume {
|
||||
new_offset += bytes.len() as i64;
|
||||
}
|
||||
|
||||
if skipped_needles > 0 {
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
skipped_needles,
|
||||
skipped_data_bytes,
|
||||
"vacuum: dropped unreadable index entries during compaction"
|
||||
);
|
||||
}
|
||||
|
||||
dst.sync_all()?;
|
||||
|
||||
// Save new index
|
||||
@@ -3088,10 +2989,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
/// Remove all volume files from disk.
|
||||
/// Destroy removes everything related to this volume. When keep_remote_data
|
||||
/// is true the cloud-tier object backing the volume is left intact — used
|
||||
/// by moves where another server is taking over the same .vif.
|
||||
pub fn destroy(&mut self, only_empty: bool, keep_remote_data: bool) -> Result<(), VolumeError> {
|
||||
pub fn destroy(&mut self, only_empty: bool) -> Result<(), VolumeError> {
|
||||
if only_empty && self.file_count() > 0 {
|
||||
return Err(VolumeError::NotEmpty);
|
||||
}
|
||||
@@ -3103,11 +3001,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
let (storage_name, storage_key) = self.remote_storage_name_key();
|
||||
if !keep_remote_data
|
||||
&& self.has_remote_file
|
||||
&& !storage_name.is_empty()
|
||||
&& !storage_key.is_empty()
|
||||
{
|
||||
if self.has_remote_file && !storage_name.is_empty() && !storage_key.is_empty() {
|
||||
let backend = crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.read()
|
||||
.unwrap()
|
||||
@@ -3203,17 +3097,11 @@ fn get_append_at_ns(last: u64) -> u64 {
|
||||
}
|
||||
|
||||
/// Remove all files associated with a volume.
|
||||
/// .dat/.idx removals log at info level so destructive calls are traceable.
|
||||
pub(crate) fn remove_volume_files(base: &str) {
|
||||
for ext in &[
|
||||
".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
|
||||
] {
|
||||
let path = format!("{}{}", base, ext);
|
||||
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
|
||||
let existed = fs::remove_file(&path).is_ok();
|
||||
if existed && (*ext == ".dat" || *ext == ".idx") {
|
||||
tracing::info!("removed volume file {} (size={})", path, size);
|
||||
}
|
||||
let _ = fs::remove_file(format!("{}{}", base, ext));
|
||||
}
|
||||
// leveldb uses a directory
|
||||
let _ = fs::remove_dir_all(format!("{}.ldb", base));
|
||||
@@ -3337,7 +3225,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -3392,7 +3279,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -3402,8 +3288,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -3679,7 +3563,7 @@ mod tests {
|
||||
dat_path = v.file_name(".dat");
|
||||
idx_path = v.file_name(".idx");
|
||||
assert!(Path::new(&dat_path).exists());
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
}
|
||||
|
||||
assert!(!Path::new(&dat_path).exists());
|
||||
@@ -3822,138 +3706,6 @@ mod tests {
|
||||
v.cleanup_compact().unwrap();
|
||||
}
|
||||
|
||||
/// Vacuum compaction must tolerate an .idx entry whose offset points past
|
||||
/// the end of the .dat file (the failure mode in issue #8928). The bad
|
||||
/// entry is silently dropped from the resulting .cpx; healthy needles
|
||||
/// survive untouched.
|
||||
#[test]
|
||||
fn test_compact_by_index_drops_dangling_needle() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
// Write a handful of healthy needles to establish a baseline .dat/.idx.
|
||||
for i in 1..=5u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: format!("payload-{}", i).into_bytes(),
|
||||
data_size: format!("payload-{}", i).len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
let dat_size = v.dat_file_size().unwrap();
|
||||
let bad_key = NeedleId(9999);
|
||||
let bad_offset = Offset::from_actual_offset((dat_size + 1024 * 1024) as i64);
|
||||
let bad_size = Size(2048);
|
||||
v.nm
|
||||
.as_mut()
|
||||
.expect("needle map present")
|
||||
.put(bad_key, bad_offset, bad_size)
|
||||
.unwrap();
|
||||
v.nm.as_ref().unwrap().sync().unwrap();
|
||||
|
||||
// Vacuum must succeed in spite of the dangling entry.
|
||||
v.compact_by_index(0, 0, |_| true)
|
||||
.expect("compact_by_index should tolerate dangling entries");
|
||||
|
||||
// Walk the resulting .cpx and confirm the bad key was dropped while
|
||||
// every healthy key made it through.
|
||||
let cpx_path = v.file_name(".cpx");
|
||||
let mut cpx = File::open(&cpx_path).unwrap();
|
||||
let mut kept: Vec<u64> = Vec::new();
|
||||
idx::walk_index_file(&mut cpx, 0, |key, _, size| {
|
||||
if !size.is_deleted() {
|
||||
kept.push(key.0);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
!kept.contains(&bad_key.0),
|
||||
"dangling key {} should have been dropped from .cpx, got {:?}",
|
||||
bad_key.0,
|
||||
kept
|
||||
);
|
||||
for i in 1..=5u64 {
|
||||
assert!(
|
||||
kept.contains(&i),
|
||||
"healthy key {} missing from compacted .cpx, got {:?}",
|
||||
i,
|
||||
kept
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The needle map's max_needle_end accumulator must let volume.load
|
||||
/// detect an .idx whose entries point past the end of the .dat — the
|
||||
/// deeper-than-tail corruption shape from issue #8928 that the existing
|
||||
/// last-10-entries scan cannot see. The check is populated by the load
|
||||
/// walk and read in volume.load() to flip the volume read-only.
|
||||
#[test]
|
||||
fn test_max_needle_end_detects_dangling_entry() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
for i in 1..=4u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: format!("data-{}", i).into_bytes(),
|
||||
data_size: format!("data-{}", i).len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
let dat_size = v.dat_file_size().unwrap() as i64;
|
||||
let idx_path = v.file_name(".idx");
|
||||
let version = v.version();
|
||||
|
||||
// Sanity: a fresh load walk over the healthy .idx puts max_needle_end
|
||||
// somewhere inside the .dat.
|
||||
let mut idx_reader = File::open(&idx_path).unwrap();
|
||||
let healthy_nm =
|
||||
CompactNeedleMap::load_from_idx(&mut idx_reader, version).unwrap();
|
||||
let healthy_end = healthy_nm.max_needle_end();
|
||||
assert!(
|
||||
healthy_end > 0 && healthy_end <= dat_size,
|
||||
"healthy volume should have max_needle_end ({}) in [1, dat_size={}]",
|
||||
healthy_end,
|
||||
dat_size
|
||||
);
|
||||
|
||||
// Inject a dangling entry by appending a bogus 16/17-byte record
|
||||
// directly to the .idx, then reload. The load walk should observe
|
||||
// max_needle_end past dat_size — which is exactly the signal
|
||||
// volume.load uses to mark the volume read-only.
|
||||
let bad_offset = Offset::from_actual_offset(dat_size + 4 * 1024 * 1024);
|
||||
let mut idx_append = OpenOptions::new()
|
||||
.write(true)
|
||||
.append(true)
|
||||
.open(&idx_path)
|
||||
.unwrap();
|
||||
idx::write_index_entry(&mut idx_append, NeedleId(9999), bad_offset, Size(1024))
|
||||
.unwrap();
|
||||
idx_append.sync_all().unwrap();
|
||||
|
||||
let mut idx_reread = File::open(&idx_path).unwrap();
|
||||
let bad_nm = CompactNeedleMap::load_from_idx(&mut idx_reread, version).unwrap();
|
||||
let bad_end = bad_nm.max_needle_end();
|
||||
assert!(
|
||||
bad_end > dat_size,
|
||||
"after dangling-entry inject max_needle_end ({}) should exceed dat_size ({})",
|
||||
bad_end,
|
||||
dat_size
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compaction_revision_relookup() {
|
||||
// Verifies that re_lookup_needle_data_offset returns the correct data offset
|
||||
@@ -4081,7 +3833,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "vif_rw_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: v.dat_file_size().unwrap(),
|
||||
@@ -4097,27 +3849,6 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Register a dummy S3 backend so load_remote_dat_file can look it
|
||||
// up. This test never reads remote data, so a dead endpoint is fine.
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry.
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket-a".to_string(),
|
||||
endpoint: "http://127.0.0.1:1".to_string(),
|
||||
storage_class: "STANDARD".to_string(),
|
||||
force_path_style: true,
|
||||
};
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.register(
|
||||
"s3.vif_rw_test".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
|
||||
v.dat_file_size().unwrap()
|
||||
};
|
||||
|
||||
@@ -4161,11 +3892,6 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(deleted_size.0 > 0);
|
||||
assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
|
||||
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.vif_rw_test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4328,9 +4054,10 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4345,7 +4072,11 @@ mod tests {
|
||||
.write()
|
||||
.unwrap();
|
||||
registry.register(
|
||||
"s3.remote_only_rw".to_string(),
|
||||
"s3.default".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register(
|
||||
"s3".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
}
|
||||
@@ -4353,7 +4084,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "remote_only_rw".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4410,10 +4141,6 @@ mod tests {
|
||||
assert_eq!(meta.data_size, 11);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.remote_only_rw");
|
||||
}
|
||||
|
||||
/// Volume destroy removes .vif alongside the primary data files.
|
||||
@@ -4444,7 +4171,7 @@ mod tests {
|
||||
assert!(std::path::Path::new(&idx_path).exists());
|
||||
|
||||
// Destroy the volume
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
|
||||
// .dat and .idx should be gone
|
||||
assert!(
|
||||
@@ -4501,7 +4228,7 @@ mod tests {
|
||||
assert!(std::path::Path::new(&dat_path).exists());
|
||||
assert!(std::path::Path::new(&idx_path).exists());
|
||||
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
|
||||
assert!(
|
||||
!std::path::Path::new(&dat_path).exists(),
|
||||
|
||||
@@ -101,8 +101,6 @@ fn test_state_with_guard(
|
||||
read_mode: seaweed_volume::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
|
||||
@@ -33,11 +33,11 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/image v0.36.0 // indirect
|
||||
golang.org/x/net v0.50.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
||||
@@ -27,8 +27,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -84,35 +84,35 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
|
||||
+2
-2
@@ -999,9 +999,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.4"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user