mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-14 10:35:51 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fafd8493fb | ||
|
|
b270f1698d | ||
|
|
105257e9b5 | ||
|
|
4bd8e5dd59 |
@@ -4,10 +4,6 @@ on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -22,7 +18,6 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
|
||||
@@ -9,96 +9,17 @@ permissions:
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build-dev-containers:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-normal-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -109,40 +30,40 @@ jobs:
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Build
|
||||
uses: docker/build-push-action@v7
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -23,22 +23,15 @@ on:
|
||||
- all
|
||||
- standard
|
||||
- large_disk
|
||||
publish:
|
||||
description: 'Publish images and manifests'
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
variants: ${{ steps.set-variants.outputs.variants }}
|
||||
publish: ${{ steps.set-publish.outputs.publish }}
|
||||
steps:
|
||||
- name: Select variants for this run
|
||||
id: set-variants
|
||||
@@ -49,89 +42,9 @@ jobs:
|
||||
variants='["standard","large_disk"]'
|
||||
fi
|
||||
echo "variants=$variants" >> "$GITHUB_OUTPUT"
|
||||
- name: Select publish mode
|
||||
id: set-publish
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "publish=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
build:
|
||||
needs: [setup, build-rust-binaries]
|
||||
needs: [setup]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -166,32 +79,11 @@ jobs:
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=large-disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
echo "rust_variant=normal" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v6
|
||||
@@ -219,14 +111,14 @@ jobs:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -237,7 +129,7 @@ jobs:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ needs.setup.outputs.publish == 'true' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
@@ -257,151 +149,10 @@ jobs:
|
||||
# Remove Go build cache
|
||||
sudo rm -rf /tmp/go-build*
|
||||
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, build-rust-binaries]
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
steps:
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Checkout for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
|
||||
- name: Download pre-built Rust binaries for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
- name: Place Rust binaries in Docker context for local scan
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
rust_variant="normal"
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
rust_variant="large-disk"
|
||||
fi
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${rust_variant}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
- name: Create BuildKit config for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
[registry."docker.io"]
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx for local scan build
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Build local scan image tarball
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
context: ./docker
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/amd64
|
||||
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
|
||||
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
# since they all use the same alpine base, so a single-arch scan
|
||||
# provides sufficient coverage without multiplying CI time.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
- name: Trivy gate (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
format: table
|
||||
exit-code: '1'
|
||||
skip-setup-trivy: true
|
||||
|
||||
create-manifest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [setup, build, trivy-scan]
|
||||
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
|
||||
needs: [setup, build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
|
||||
@@ -429,12 +180,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -4,19 +4,11 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_foundationdb:
|
||||
@@ -34,7 +26,7 @@ jobs:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
|
||||
type=ref,event=tag,suffix=_large_disk_foundationdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
@@ -50,7 +42,7 @@ jobs:
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -39,82 +39,7 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
||||
# ── Pre-build Rust volume server binaries natively ──────────────────
|
||||
# Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour
|
||||
# emulated cargo build into ~15 minutes of native compilation.
|
||||
build-rust-binaries:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-musl
|
||||
arch: amd64
|
||||
- target: aarch64-unknown-linux-musl
|
||||
arch: arm64
|
||||
cross: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install musl tools (amd64)
|
||||
if: ${{ !matrix.cross }}
|
||||
run: sudo apt-get install -y musl-tools
|
||||
|
||||
- name: Install cross-compilation tools (arm64)
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
# Disable glibc fortify source — its __memcpy_chk etc. symbols don't exist in musl
|
||||
echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-docker-${{ matrix.target }}-
|
||||
|
||||
- name: Build large-disk variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }}
|
||||
|
||||
- name: Build normal variant
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.arch }}
|
||||
path: |
|
||||
weed-volume-large-disk-${{ matrix.arch }}
|
||||
weed-volume-normal-${{ matrix.arch }}
|
||||
|
||||
# ── Build Docker containers ─────────────────────────────────────────
|
||||
build:
|
||||
needs: [build-rust-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# Build sequentially to avoid rate limits
|
||||
@@ -127,23 +52,20 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: ""
|
||||
tag_suffix: ""
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk - multi-arch
|
||||
|
||||
# Large disk - multi-arch
|
||||
- variant: large_disk
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# Full tags - multi-arch
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
rust_variant: normal
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- variant: large_disk_full
|
||||
@@ -151,42 +73,19 @@ jobs:
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
rust_variant: large-disk
|
||||
|
||||
|
||||
# RocksDB large disk - amd64 only
|
||||
- variant: rocksdb
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.rocksdb_large
|
||||
build_args: ""
|
||||
tag_suffix: _large_disk_rocksdb
|
||||
rust_variant: large-disk
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download pre-built Rust binaries
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: rust-volume-*
|
||||
merge-multiple: true
|
||||
path: ./rust-bins
|
||||
|
||||
- name: Place Rust binaries in Docker context
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
mkdir -p docker/weed-volume-prebuilt
|
||||
for arch in amd64 arm64; do
|
||||
src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}"
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}"
|
||||
echo "Placed pre-built Rust binary for ${arch}"
|
||||
fi
|
||||
done
|
||||
ls -la docker/weed-volume-prebuilt/
|
||||
|
||||
|
||||
- name: Free Disk Space
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
run: |
|
||||
@@ -236,14 +135,14 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -299,14 +198,14 @@ jobs:
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.1.0
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -88,13 +88,13 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
@@ -1,69 +0,0 @@
|
||||
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run P2P integration tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-p2p-test-logs
|
||||
path: /tmp/seaweedfs-fuse-p2p-logs/
|
||||
retention-days: 3
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v5
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
@@ -217,106 +217,6 @@ jobs:
|
||||
print("✓ No blank lines in security+S3 command blocks")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
|
||||
# Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
|
||||
# the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
|
||||
# them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
|
||||
# makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
|
||||
# HTTP request to an HTTPS server" into the pod log.
|
||||
#
|
||||
# When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
|
||||
# together so the opt-in HTTPS listener actually has credentials.
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import subprocess, sys, yaml
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def script_of(manifest, kind_name):
|
||||
for doc in yaml.safe_load_all(manifest):
|
||||
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
if doc["metadata"]["name"] != kind_name:
|
||||
continue
|
||||
for c in doc["spec"]["template"]["spec"]["containers"]:
|
||||
cmd = c.get("command", [])
|
||||
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
|
||||
return cmd[2]
|
||||
raise AssertionError(f"no container script for {kind_name}")
|
||||
|
||||
cases = [
|
||||
# (values, workload-name, httpsPort-set?, arg-prefix)
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true"},
|
||||
"test-seaweedfs-s3", False, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true",
|
||||
"s3.httpsPort": "8443"},
|
||||
"test-seaweedfs-s3", True, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true"},
|
||||
"test-seaweedfs-filer", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true",
|
||||
"filer.s3.httpsPort": "8444"},
|
||||
"test-seaweedfs-filer", True, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true"},
|
||||
"test-seaweedfs-all-in-one", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true",
|
||||
"allInOne.s3.httpsPort": "8445"},
|
||||
"test-seaweedfs-all-in-one", True, "s3."),
|
||||
]
|
||||
|
||||
failed = False
|
||||
for values, name, https_on, prefix in cases:
|
||||
script = script_of(render(values), name)
|
||||
cert_flag = f"-{prefix}cert.file="
|
||||
key_flag = f"-{prefix}key.file="
|
||||
https_flag = f"-{prefix}port.https="
|
||||
has_cert = cert_flag in script
|
||||
has_key = key_flag in script
|
||||
has_https = https_flag in script
|
||||
label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
|
||||
if https_on:
|
||||
if not (has_cert and has_key and has_https):
|
||||
print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: cert/key/https args emitted together")
|
||||
else:
|
||||
if has_cert or has_key or has_https:
|
||||
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
|
||||
f"main S3 -port would silently become HTTPS and break HTTP probes "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
|
||||
|
||||
# bash -n: pin down that the rendered script parses. Guards against
|
||||
# a future helper change that leaves a dangling `\` with nothing
|
||||
# after it (every current caller already exits cleanly because
|
||||
# bash treats trailing `\<newline><EOF>` as line-continuation to
|
||||
# an empty line — but keep the contract explicit).
|
||||
parse = subprocess.run(["bash", "-n"], input=script, text=True,
|
||||
capture_output=True)
|
||||
if parse.returncode != 0:
|
||||
print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
|
||||
file=sys.stderr)
|
||||
failed = True
|
||||
|
||||
sys.exit(1 if failed else 0)
|
||||
PYEOF
|
||||
|
||||
echo "✅ All template rendering tests passed!"
|
||||
|
||||
- name: Create kind cluster
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
name: "helm: release"
|
||||
name: "helm: manual release"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -445,7 +445,7 @@ jobs:
|
||||
# Test consumer group functionality with explicit timeout
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
@@ -112,10 +112,11 @@ jobs:
|
||||
working-directory: test/kms
|
||||
run: |
|
||||
echo "=== OpenBao Container Logs ==="
|
||||
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
|
||||
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
|
||||
echo "=== SeaweedFS Server Logs ==="
|
||||
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
|
||||
echo "=== Setup Logs ==="
|
||||
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No OpenBao setup log"
|
||||
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
@@ -127,7 +128,7 @@ jobs:
|
||||
name: s3-kms-e2e-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-kms-mini.log
|
||||
/tmp/openbao-ci-container.log
|
||||
/tmp/openbao-ci-setup.log
|
||||
/tmp/weed-kms-ci-setup.log
|
||||
retention-days: 3
|
||||
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
name: "Multi-Master Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/multi-master-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run multi-master failover tests
|
||||
# The tests in test/multi_master spin up their own 3-node master raft
|
||||
# cluster (using the freshly-installed `weed` binary) and exercise
|
||||
# leader-election, failover and recovery scenarios. The shared
|
||||
# test/testutil port-allocator regression test runs alongside since it
|
||||
# is a prerequisite for the cluster fixtures.
|
||||
run: |
|
||||
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/multi_master/cluster.go creates per-test dirs via
|
||||
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
|
||||
# node's log into <baseDir>/logs/master*.log.
|
||||
echo "Collecting per-node master logs from temp directories..."
|
||||
mkdir -p /tmp/multi-master-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: multi-master-test-logs
|
||||
path: /tmp/multi-master-logs/
|
||||
retention-days: 7
|
||||
@@ -1,137 +0,0 @@
|
||||
name: "NFS Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/nfs-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
nfs-integration:
|
||||
name: NFS Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run NFS Integration Tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running NFS integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run the protocol-layer tests. The kernel-mount tests require root
|
||||
# for mount(2) and are exercised in their own privileged step below;
|
||||
# skip them here so a "skipped because not root" line doesn't show
|
||||
# up as noise on every CI run.
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "NFS integration tests completed"
|
||||
|
||||
- name: Install kernel NFS client
|
||||
run: |
|
||||
# nfs-common provides mount.nfs; netbase provides /etc/protocols
|
||||
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nfs-common netbase
|
||||
|
||||
- name: Run kernel-mount E2E tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running kernel-mount end-to-end tests..."
|
||||
echo "These mount the running 'weed nfs' subprocess via the actual"
|
||||
echo "Linux NFS client to catch protocol regressions invisible to"
|
||||
echo "the go-nfs-client-based tests above."
|
||||
echo "============================================"
|
||||
|
||||
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
|
||||
# binary that test/nfs/framework.go locates via $PATH) resolve
|
||||
# correctly under sudo, and pass through the Go module/cache dirs
|
||||
# so we don't redownload modules under root.
|
||||
sudo env "PATH=$PATH" \
|
||||
GOMODCACHE="$(go env GOMODCACHE)" \
|
||||
GOCACHE="$(go env GOCACHE)" \
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Kernel-mount E2E tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
|
||||
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
|
||||
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
|
||||
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
|
||||
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1,118 +0,0 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-e2e
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build pjdfstest image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/pjdfstest
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-harness
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-harness
|
||||
|
||||
- name: Tag pjdfstest image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -1,242 +0,0 @@
|
||||
name: "Rust Volume Server Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- 'test/volume_server/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- '.github/workflows/rust-volume-server-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
jobs:
|
||||
rust-unit-tests:
|
||||
name: Rust Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build Rust volume server
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run Rust unit tests
|
||||
run: cd seaweed-volume && cargo test
|
||||
|
||||
rust-integration-tests:
|
||||
name: Rust Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
run: |
|
||||
echo "Running Rust volume server integration tests..."
|
||||
go test -v -count=1 -timeout=15m ./test/volume_server/rust/...
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-server-it-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-server-it-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-server-integration-test-logs
|
||||
path: /tmp/rust-volume-server-it-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Rust Volume Server Integration Test Summary" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/rust" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=15m ./test/volume_server/rust/..." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
rust-volume-go-tests:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test-type: [grpc, http]
|
||||
shard: [1, 2, 3]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags 5BytesOffset -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Run volume server integration tests with Rust volume
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/rust-volume-go-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/rust-volume-go-test-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-go-test-logs-${{ matrix.test-type }}-shard${{ matrix.shard }}
|
||||
path: /tmp/rust-volume-go-test-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,166 +0,0 @@
|
||||
name: "rust: build dev volume server binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'seaweed-volume/**'
|
||||
- '.github/workflows/rust_binaries_dev.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
cleanup:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
fail-if-no-assets: false
|
||||
assets: |
|
||||
weed-volume-*
|
||||
|
||||
build-rust-volume-dev-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: cleanup
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
build-rust-volume-dev-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
needs: build-rust-volume-dev-linux
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin-arm64
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin-amd64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-dev-${{ matrix.target }}-
|
||||
|
||||
- name: Set BUILD_TIME
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Package large disk binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf "weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: cd seaweed-volume && cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package normal binary
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf "weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz" weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
files: |
|
||||
weed-volume-large-disk-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume-${{ env.BUILD_TIME }}-${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -1,253 +0,0 @@
|
||||
name: "rust: build versioned volume server binaries"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
build-rust-volume-linux:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
asset_suffix: linux_amd64
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
asset_suffix: linux_arm64
|
||||
cross: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cross-compilation tools
|
||||
if: matrix.cross
|
||||
run: |
|
||||
sudo dpkg --add-architecture arm64
|
||||
sudo sed -i 's/^deb /deb [arch=amd64] /' /etc/apt/sources.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy main restricted universe multiverse" | sudo tee /etc/apt/sources.list.d/arm64.list
|
||||
echo "deb [arch=arm64] http://ports.ubuntu.com/ jammy-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list.d/arm64.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-aarch64-linux-gnu libssl-dev:arm64
|
||||
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_DIR=/usr" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_INCLUDE_DIR=/usr/include" >> "$GITHUB_ENV"
|
||||
echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
# Large disk (default, 5bytes feature)
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
# Normal volume size
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-darwin:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: macos-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-apple-darwin
|
||||
asset_suffix: darwin_amd64
|
||||
- target: aarch64-apple-darwin
|
||||
asset_suffix: darwin_arm64
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: brew install protobuf
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-${{ matrix.target }}-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --target ${{ matrix.target }} --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
run: |
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-large-disk
|
||||
tar czf weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz weed-volume-large-disk
|
||||
rm weed-volume-large-disk
|
||||
|
||||
cp seaweed-volume/target/${{ matrix.target }}/release/weed-volume weed-volume-normal
|
||||
tar czf weed-volume_${{ matrix.asset_suffix }}.tar.gz weed-volume-normal
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-${{ matrix.asset_suffix }}
|
||||
path: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
weed-volume_${{ matrix.asset_suffix }}.tar.gz
|
||||
|
||||
build-rust-volume-windows:
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: windows-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Install protobuf compiler
|
||||
run: choco install protoc -y
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-release-windows-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-release-windows-
|
||||
|
||||
- name: Build Rust volume server (large disk)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release
|
||||
|
||||
- name: Build Rust volume server (normal)
|
||||
env:
|
||||
SEAWEEDFS_COMMIT: ${{ github.sha }}
|
||||
run: |
|
||||
cd seaweed-volume
|
||||
cargo build --release --no-default-features
|
||||
|
||||
- name: Package binaries
|
||||
shell: bash
|
||||
run: |
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-large-disk.exe
|
||||
7z a weed-volume_large_disk_windows_amd64.zip weed-volume-large-disk.exe
|
||||
rm weed-volume-large-disk.exe
|
||||
|
||||
cp seaweed-volume/target/release/weed-volume.exe weed-volume-normal.exe
|
||||
7z a weed-volume_windows_amd64.zip weed-volume-normal.exe
|
||||
rm weed-volume-normal.exe
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
if: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: rust-volume-windows_amd64
|
||||
path: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
weed-volume_windows_amd64.zip
|
||||
@@ -1,129 +0,0 @@
|
||||
name: "S3 ETag and ACL Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-etag-acl
|
||||
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
|
||||
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-etag-acl \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
# Wait for the S3 endpoint to come up (returns 403 unauth before any
|
||||
# request is signed; that's fine — it means the server is listening).
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run ETag tests
|
||||
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
|
||||
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
|
||||
# SDK for Java v2 rejects the latter on the PutObject path.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/etag/...
|
||||
|
||||
- name: Run ACL versioning tests
|
||||
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
|
||||
# PutObjectAcl with and without versionId, modifying ACLs on different
|
||||
# versions independently.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/acl/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-etag-acl-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -245,61 +245,6 @@ jobs:
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Checksum Tests
|
||||
timeout-minutes: 16
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-checksum-test-logs
|
||||
path: test/s3/checksum/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -138,23 +138,6 @@ jobs:
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
"sts")
|
||||
echo "Running STS and service account tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
# SigV4-signed STS calls need admin credentials matching test_config.json.
|
||||
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
|
||||
export STS_TEST_ACCESS_KEY=test-access-key
|
||||
export STS_TEST_SECRET_KEY=test-secret-key
|
||||
# The use_service_account_credentials subtest is excluded because
|
||||
# newly-created service-account access keys are not currently
|
||||
# persisted to the filer after CreateServiceAccount — a
|
||||
# pre-existing sync issue tracked separately from the
|
||||
# GetFederationToken routing fix this PR addresses.
|
||||
go test -v -timeout 15m \
|
||||
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
|
||||
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
|
||||
./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
name: "S3 Mutation Regression Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/delete/**'
|
||||
- 'test/s3/distributed_lock/**'
|
||||
- 'test/s3/versioning/**'
|
||||
- 'test/volume_server/framework/**'
|
||||
- 'docker/compose/s3.json'
|
||||
- '.github/workflows/s3-mutation-regression-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-mutation-regression-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-versioning-regressions:
|
||||
name: S3 Versioning Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 versioning regression tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload versioning logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-regression-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-delete-regressions:
|
||||
name: S3 Delete Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Run S3 delete regression tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/delete
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
|
||||
- name: Upload delete logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-delete-regression-logs
|
||||
path: test/s3/delete/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-distributed-lock-regressions:
|
||||
name: S3 Distributed Lock Regression Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run distributed lock regressions
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
TMPDIR: ${{ github.workspace }}/test/s3/distributed_lock/tmp
|
||||
S3_DISTRIBUTED_LOCK_KEEP_LOGS: "1"
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
set -x
|
||||
mkdir -p "$TMPDIR"
|
||||
go test -v -count=1 -timeout=20m ./test/s3/distributed_lock
|
||||
|
||||
- name: Upload distributed lock logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-distributed-lock-regression-logs
|
||||
path: test/s3/distributed_lock/tmp/seaweedfs_s3_distributed_lock_*
|
||||
retention-days: 3
|
||||
@@ -73,12 +73,8 @@ jobs:
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
else
|
||||
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
|
||||
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
|
||||
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
|
||||
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
|
||||
# same change that fixed them.
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
|
||||
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
fi
|
||||
|
||||
- name: Show server logs on failure
|
||||
|
||||
@@ -2,6 +2,10 @@ name: "S3 Tables Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-tables-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -190,77 +194,6 @@ jobs:
|
||||
path: test/s3tables/catalog_trino/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
dremio-iceberg-catalog-tests:
|
||||
name: Dremio Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Dremio image
|
||||
run: docker pull dremio/dremio-oss:25.2.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Dremio Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Dremio Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Dremio Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|dremio)" || true
|
||||
echo "=== Dremio containers ==="
|
||||
docker ps -a --filter "name=seaweed-dremio" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dremio-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_dremio/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -37,8 +37,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -341,8 +339,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -511,8 +507,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -736,8 +730,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "TLS Rotation Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run TLS Rotation Integration Tests
|
||||
working-directory: test/tls_rotation
|
||||
run: |
|
||||
go test -v -count=1 -timeout 5m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting master logs from temp directories..."
|
||||
mkdir -p /tmp/tls-rotation-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tls-rotation-test-logs
|
||||
path: /tmp/tls-rotation-test-logs/
|
||||
retention-days: 14
|
||||
@@ -2,6 +2,17 @@ name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "Vacuum Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run Vacuum Integration Tests
|
||||
working-directory: test/vacuum
|
||||
run: |
|
||||
go test -v -timeout 10m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/vacuum-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vacuum-integration-test-logs
|
||||
path: /tmp/vacuum-test-logs/
|
||||
retention-days: 14
|
||||
@@ -2,7 +2,6 @@
|
||||
vendor
|
||||
tags
|
||||
*.swp
|
||||
.claude/
|
||||
### OSX template
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
@@ -144,4 +143,3 @@ test/s3/iam/.test_env
|
||||
weed_bin
|
||||
telemetry/server/telemetry-server
|
||||
.aider*
|
||||
/seaweed-volume/docs
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"setup": [],
|
||||
"teardown": [],
|
||||
"run": []
|
||||
}
|
||||
@@ -42,7 +42,8 @@ test: admin-generate
|
||||
|
||||
# Admin component targets
|
||||
admin-generate:
|
||||
@cd $(ADMIN_DIR) && $(MAKE) generate
|
||||
@echo "Generating admin component templates..."
|
||||
@cd $(ADMIN_DIR) && templ generate ./view
|
||||
|
||||
admin-build: admin-generate
|
||||
@echo "Building admin component..."
|
||||
|
||||
@@ -80,29 +80,28 @@ Table of Contents
|
||||
|
||||
|
||||
## Quick Start with weed mini ##
|
||||
The easiest way to get started with SeaweedFS for development and testing:
|
||||
|
||||
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
AWS_ACCESS_KEY_ID=admin \
|
||||
AWS_SECRET_ACCESS_KEY=secret \
|
||||
S3_BUCKET=my-bucket \
|
||||
# remove quarantine on macOS
|
||||
# xattr -d com.apple.quarantine ./weed
|
||||
|
||||
./weed mini -dir=/data
|
||||
```
|
||||
|
||||
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
|
||||
|
||||
The same command starts everything else too:
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
This single command starts a complete SeaweedFS setup with:
|
||||
- **Master UI**: http://localhost:9333
|
||||
- **Volume Server**: http://localhost:9340
|
||||
- **Filer UI**: http://localhost:8888
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
- **WebDAV**: http://localhost:7333
|
||||
- **Admin UI**: http://localhost:23646
|
||||
|
||||
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
|
||||
|
||||
Perfect for development, testing, learning SeaweedFS, and single-node deployments.
|
||||
Perfect for development, testing, learning SeaweedFS, and single node deployments!
|
||||
|
||||
## Quick Start for S3 API on Docker ##
|
||||
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you find a security issue in SeaweedFS, please report it privately:
|
||||
|
||||
- Email: support@seaweedfs.com
|
||||
- Do not open a public GitHub issue
|
||||
|
||||
Please include:
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce (if possible)
|
||||
- Affected versions
|
||||
|
||||
## Response
|
||||
|
||||
- We will respond as soon as possible (usually within 1 business day)
|
||||
- We will investigate and work on a fix
|
||||
- We may coordinate disclosure with you
|
||||
|
||||
## Notes
|
||||
|
||||
- Please allow time for a fix before public disclosure
|
||||
- If you’re unsure whether something is a security issue, feel free to reach out
|
||||
@@ -1,790 +0,0 @@
|
||||
# Execution Plan: SeaweedFS Volume Server — Go to Rust Port
|
||||
|
||||
## Scope Summary
|
||||
|
||||
| Component | Go Source | Lines (non-test) | Description |
|
||||
|---|---|---|---|
|
||||
| CLI & startup | `weed/command/volume.go` | 476 | ~40 CLI flags, server bootstrap |
|
||||
| HTTP server + handlers | `weed/server/volume_server*.go` | 1,517 | Struct, routes, read/write/delete handlers |
|
||||
| gRPC handlers | `weed/server/volume_grpc_*.go` | 3,073 | 40 RPC method implementations |
|
||||
| Storage engine | `weed/storage/` | 15,271 | Volumes, needles, index, compaction, EC, backend |
|
||||
| Protobuf definitions | `weed/pb/volume_server.proto` | 759 | Service + message definitions |
|
||||
| Shared utilities | `weed/security/`, `weed/stats/`, `weed/util/` | ~2,000+ | JWT, TLS, metrics, helpers |
|
||||
| **Total** | | **~23,000+** | |
|
||||
|
||||
## Rust Crate & Dependency Strategy
|
||||
|
||||
```
|
||||
seaweed-volume/
|
||||
├── Cargo.toml
|
||||
├── build.rs # protobuf codegen
|
||||
├── proto/
|
||||
│ ├── volume_server.proto # copied from Go, adapted
|
||||
│ └── remote.proto
|
||||
├── src/
|
||||
│ ├── main.rs # CLI entry point
|
||||
│ ├── config.rs # CLI flags + config
|
||||
│ ├── server/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── volume_server.rs # VolumeServer struct + lifecycle
|
||||
│ │ ├── http_handlers.rs # HTTP route dispatch
|
||||
│ │ ├── http_read.rs # GET/HEAD handlers
|
||||
│ │ ├── http_write.rs # POST/PUT handlers
|
||||
│ │ ├── http_delete.rs # DELETE handler
|
||||
│ │ ├── http_admin.rs # /status, /healthz, /ui
|
||||
│ │ ├── grpc_service.rs # gRPC trait impl dispatch
|
||||
│ │ ├── grpc_vacuum.rs
|
||||
│ │ ├── grpc_copy.rs
|
||||
│ │ ├── grpc_erasure_coding.rs
|
||||
│ │ ├── grpc_tail.rs
|
||||
│ │ ├── grpc_admin.rs
|
||||
│ │ ├── grpc_read_write.rs
|
||||
│ │ ├── grpc_batch_delete.rs
|
||||
│ │ ├── grpc_scrub.rs
|
||||
│ │ ├── grpc_tier.rs
|
||||
│ │ ├── grpc_remote.rs
|
||||
│ │ ├── grpc_query.rs
|
||||
│ │ ├── grpc_state.rs
|
||||
│ │ └── grpc_client_to_master.rs # heartbeat
|
||||
│ ├── storage/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── store.rs # Store (multi-disk manager)
|
||||
│ │ ├── volume.rs # Volume struct + lifecycle
|
||||
│ │ ├── volume_read.rs
|
||||
│ │ ├── volume_write.rs
|
||||
│ │ ├── volume_compact.rs
|
||||
│ │ ├── volume_info.rs
|
||||
│ │ ├── needle/
|
||||
│ │ │ ├── mod.rs
|
||||
│ │ │ ├── needle.rs # Needle struct + serialization
|
||||
│ │ │ ├── needle_read.rs
|
||||
│ │ │ ├── needle_write.rs
|
||||
│ │ │ ├── needle_map.rs # in-memory NeedleMap
|
||||
│ │ │ ├── needle_value.rs
|
||||
│ │ │ └── crc.rs
|
||||
│ │ ├── super_block.rs
|
||||
│ │ ├── idx/
|
||||
│ │ │ ├── mod.rs
|
||||
│ │ │ └── idx.rs # .idx file format read/write
|
||||
│ │ ├── needle_map_leveldb.rs
|
||||
│ │ ├── types.rs # NeedleId, Offset, Size, DiskType
|
||||
│ │ ├── disk_location.rs # DiskLocation per-directory
|
||||
│ │ ├── erasure_coding/
|
||||
│ │ │ ├── mod.rs
|
||||
│ │ │ ├── ec_volume.rs
|
||||
│ │ │ ├── ec_shard.rs
|
||||
│ │ │ ├── ec_encoder.rs # Reed-Solomon encoding
|
||||
│ │ │ └── ec_decoder.rs
|
||||
│ │ └── backend/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── disk.rs
|
||||
│ │ └── s3_backend.rs # tiered storage to S3
|
||||
│ ├── topology/
|
||||
│ │ └── volume_layout.rs # replication placement
|
||||
│ ├── security/
|
||||
│ │ ├── mod.rs
|
||||
│ │ ├── guard.rs # whitelist + JWT gate
|
||||
│ │ ├── jwt.rs
|
||||
│ │ └── tls.rs
|
||||
│ ├── stats/
|
||||
│ │ ├── mod.rs
|
||||
│ │ └── metrics.rs # Prometheus counters/gauges
|
||||
│ └── util/
|
||||
│ ├── mod.rs
|
||||
│ ├── grpc.rs
|
||||
│ ├── http.rs
|
||||
│ └── file.rs
|
||||
└── tests/
|
||||
├── integration/
|
||||
│ ├── http_read_test.rs
|
||||
│ ├── http_write_test.rs
|
||||
│ ├── grpc_test.rs
|
||||
│ └── storage_test.rs
|
||||
└── unit/
|
||||
├── needle_test.rs
|
||||
├── idx_test.rs
|
||||
├── super_block_test.rs
|
||||
└── ec_test.rs
|
||||
```
|
||||
|
||||
### Key Rust dependencies
|
||||
|
||||
| Purpose | Crate |
|
||||
|---|---|
|
||||
| Async runtime | `tokio` |
|
||||
| gRPC | `tonic` + `prost` |
|
||||
| HTTP server | `hyper` + `axum` |
|
||||
| CLI parsing | `clap` (derive) |
|
||||
| Prometheus metrics | `prometheus` |
|
||||
| JWT | `jsonwebtoken` |
|
||||
| TLS | `rustls` + `tokio-rustls` |
|
||||
| LevelDB | `rusty-leveldb` or `rocksdb` |
|
||||
| Reed-Solomon EC | `reed-solomon-erasure` |
|
||||
| Logging | `tracing` + `tracing-subscriber` |
|
||||
| Config (security.toml) | `toml` + `serde` |
|
||||
| CRC32 | `crc32fast` |
|
||||
| Memory-mapped files | `memmap2` |
|
||||
|
||||
---
|
||||
|
||||
## Phased Execution Plan
|
||||
|
||||
### Phase 1: Project Skeleton & Protobuf Codegen
|
||||
**Goal:** Cargo project compiles, proto codegen works, CLI parses all flags.
|
||||
|
||||
**Steps:**
|
||||
|
||||
1.1. Create `seaweed-volume/Cargo.toml` with all dependencies listed above.
|
||||
|
||||
1.2. Copy `volume_server.proto` and `remote.proto` into `proto/`. Adjust package paths for Rust codegen.
|
||||
|
||||
1.3. Create `build.rs` using `tonic-build` to compile `.proto` files into Rust types.
|
||||
|
||||
1.4. Create `src/main.rs` with `clap` derive structs mirroring all 40 CLI flags from `weed/command/volume.go`:
|
||||
- `--port` (default 8080)
|
||||
- `--port.grpc` (default 0 → 10000+port)
|
||||
- `--port.public` (default 0 → same as port)
|
||||
- `--ip` (auto-detect)
|
||||
- `--id` (default empty → ip:port)
|
||||
- `--publicUrl`
|
||||
- `--ip.bind`
|
||||
- `--master` (default "localhost:9333")
|
||||
- `--mserver` (deprecated compat)
|
||||
- `--preStopSeconds` (default 10)
|
||||
- `--idleTimeout` (default 30)
|
||||
- `--dataCenter`
|
||||
- `--rack`
|
||||
- `--index` [memory|leveldb|leveldbMedium|leveldbLarge]
|
||||
- `--disk` [hdd|ssd|<tag>]
|
||||
- `--tags`
|
||||
- `--dir` (default temp dir)
|
||||
- `--dir.idx`
|
||||
- `--max` (default "8")
|
||||
- `--whiteList`
|
||||
- `--minFreeSpacePercent` (default "1")
|
||||
- `--minFreeSpace`
|
||||
- `--images.fix.orientation` (default false)
|
||||
- `--readMode` [local|proxy|redirect] (default "proxy")
|
||||
- `--cpuprofile`
|
||||
- `--memprofile`
|
||||
- `--compactionMBps` (default 0)
|
||||
- `--maintenanceMBps` (default 0)
|
||||
- `--fileSizeLimitMB` (default 256)
|
||||
- `--concurrentUploadLimitMB` (default 0)
|
||||
- `--concurrentDownloadLimitMB` (default 0)
|
||||
- `--pprof` (default false)
|
||||
- `--metricsPort` (default 0)
|
||||
- `--metricsIp`
|
||||
- `--inflightUploadDataTimeout` (default 60s)
|
||||
- `--inflightDownloadDataTimeout` (default 60s)
|
||||
- `--hasSlowRead` (default true)
|
||||
- `--readBufferSizeMB` (default 4)
|
||||
- `--index.leveldbTimeout` (default 0)
|
||||
- `--debug` (default false)
|
||||
- `--debug.port` (default 6060)
|
||||
|
||||
1.5. Implement the same flag validation logic from `startVolumeServer()`:
|
||||
- Parse comma-separated `--dir`, `--max`, `--minFreeSpace`, `--disk`, `--tags`
|
||||
- Replicate single-value-to-all-dirs expansion
|
||||
- Validate count matches between dirs and limits
|
||||
- `--mserver` backward compat
|
||||
|
||||
1.6. **Test:** `cargo build` succeeds. `cargo run -- --help` shows all flags. Proto types generated.
|
||||
|
||||
**Verification:** Run with `--port 8080 --dir /tmp --master localhost:9333` — should parse without error and print config.
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Core Storage Types & On-Disk Format
|
||||
**Goal:** Read and write the SeaweedFS needle/volume binary format bit-for-bit compatible with Go.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/types/needle_types.go` → `src/storage/types.rs`
|
||||
- `weed/storage/needle/needle.go` → `src/storage/needle/needle.rs`
|
||||
- `weed/storage/needle/needle_read.go` → `src/storage/needle/needle_read.rs`
|
||||
- `weed/storage/needle/needle_write.go` (partial) → `src/storage/needle/needle_write.rs`
|
||||
- `weed/storage/needle/crc.go` → `src/storage/needle/crc.rs`
|
||||
- `weed/storage/needle/needle_value_map.go` → `src/storage/needle/needle_value.rs`
|
||||
- `weed/storage/super_block/super_block.go` → `src/storage/super_block.rs`
|
||||
- `weed/storage/idx/` → `src/storage/idx/`
|
||||
|
||||
**Steps:**
|
||||
|
||||
2.1. **Fundamental types** (`types.rs`):
|
||||
- `NeedleId` (u64), `Offset` (u32 or u64 depending on version), `Size` (i32, negative = deleted)
|
||||
- `Cookie` (u32)
|
||||
- `DiskType` enum (HDD, SSD, Custom)
|
||||
- Version constants (Version1=1, Version2=2, Version3=3, CurrentVersion=3)
|
||||
- Byte serialization matching Go's `binary.BigEndian` encoding
|
||||
|
||||
2.2. **SuperBlock** (`super_block.rs`):
|
||||
- 8-byte header: Version(1) + ReplicaPlacement(1) + TTL(2) + CompactRevision(2) + Reserved(2)
|
||||
- `ReplicaPlacement` struct with same/diff rack/dc counts
|
||||
- `TTL` struct with count + unit
|
||||
- Read/write from first 8 bytes of `.dat` file
|
||||
- Match exact byte layout from `super_block.go`
|
||||
|
||||
2.3. **Needle binary format** (`needle.rs`, `needle_read.rs`):
|
||||
- Version 2/3 header: Cookie(4) + NeedleId(8) + Size(4)
|
||||
- Body: Data, Flags, Name, Mime, PairsSize, Pairs, LastModified, TTL, Checksum, AppendAtNs, Padding
|
||||
- CRC32 checksum (matching Go's `crc32.ChecksumIEEE`)
|
||||
- Padding to 8-byte alignment
|
||||
- Read path: read header → compute body length → read body → verify CRC
|
||||
|
||||
2.4. **Idx file format** (`idx/`):
|
||||
- Fixed 16-byte records: NeedleId(8) + Offset(4) + Size(4)
|
||||
- Sequential append-only file
|
||||
- Walk/iterate all entries
|
||||
- Binary search not used (loaded into memory map)
|
||||
|
||||
2.5. **NeedleMap (in-memory)** (`needle_map.rs`):
|
||||
- HashMap<NeedleId, NeedleValue> where NeedleValue = {Offset, Size}
|
||||
- Load from `.idx` file on volume mount
|
||||
- Support Get, Set, Delete operations
|
||||
- Track file count, deleted count, deleted byte count
|
||||
|
||||
2.6. **Tests:**
|
||||
- Unit test: write a needle to bytes → read it back → verify fields match
|
||||
- Unit test: write/read SuperBlock round-trip
|
||||
- Unit test: write/read idx entries round-trip
|
||||
- **Cross-compat test:** Use Go volume server to create a small volume with known data. Read it from Rust and verify all needles decoded correctly. (Keep test fixture `.dat`/`.idx` files in `tests/fixtures/`)
|
||||
|
||||
---
|
||||
|
||||
### Phase 3: Volume Struct & Lifecycle
|
||||
**Goal:** Mount, read from, write to, and unmount a volume.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/volume.go` → `src/storage/volume.rs`
|
||||
- `weed/storage/volume_read.go` → `src/storage/volume_read.rs`
|
||||
- `weed/storage/volume_write.go` → `src/storage/volume_write.rs`
|
||||
- `weed/storage/volume_loading.go`
|
||||
- `weed/storage/volume_vacuum.go` → `src/storage/volume_compact.rs`
|
||||
- `weed/storage/volume_info/volume_info.go` → `src/storage/volume_info.rs`
|
||||
- `weed/storage/volume_super_block.go`
|
||||
|
||||
**Steps:**
|
||||
|
||||
3.1. **Volume struct** (`volume.rs`):
|
||||
- Fields: Id, dir, dataFile, nm (NeedleMap), SuperBlock, readOnly, lastModifiedTs, lastCompactIndexOffset, lastCompactRevision
|
||||
- `noWriteOrDelete` / `noWriteCanDelete` / `readOnly` state flags
|
||||
- File handles for `.dat` file (read + append)
|
||||
- Lock strategy: `RwLock` for concurrent reads, exclusive writes
|
||||
|
||||
3.2. **Volume loading** — exact logic from `volume_loading.go`:
|
||||
- Open `.dat` file, read SuperBlock from first 8 bytes
|
||||
- Load `.idx` file into NeedleMap
|
||||
- Handle `.vif` (VolumeInfo) JSON sidecar file
|
||||
- Set volume state based on SuperBlock + VolumeInfo
|
||||
|
||||
3.3. **Volume read** (`volume_read.rs`) — from `volume_read.go`:
|
||||
- `ReadNeedle(needleId, cookie)`: lookup in NeedleMap → seek in .dat → read needle bytes → verify cookie + CRC → return data
|
||||
- Handle deleted needles (Size < 0)
|
||||
- `ReadNeedleBlob(offset, size)`: raw blob read
|
||||
- `ReadNeedleMeta(needleId, offset, size)`: read metadata only
|
||||
|
||||
3.4. **Volume write** (`volume_write.rs`) — from `volume_write.go`:
|
||||
- `WriteNeedle(needle)`: serialize needle → append to .dat → update .idx → update NeedleMap
|
||||
- `DeleteNeedle(needleId)`: mark as deleted in NeedleMap + append tombstone to .idx
|
||||
- File size limit check
|
||||
- Concurrent write serialization (mutex on write path)
|
||||
|
||||
3.5. **Volume compaction** (`volume_compact.rs`) — from `volume_vacuum.go`:
|
||||
- `CheckCompact()`: compute garbage ratio
|
||||
- `Compact()`: create new .dat/.idx, copy only live needles, update compact revision
|
||||
- `CommitCompact()`: rename compacted files over originals
|
||||
- `CleanupCompact()`: remove temp files
|
||||
- Throttle by `compactionBytePerSecond`
|
||||
|
||||
3.6. **Volume info** (`volume_info.rs`):
|
||||
- Read/write `.vif` JSON sidecar
|
||||
- VolumeInfo protobuf struct mapping
|
||||
- Remote file references for tiered storage
|
||||
|
||||
3.7. **Tests:**
|
||||
- Mount a volume, write 100 needles, read them all back, verify content
|
||||
- Delete 50 needles, verify they return "deleted"
|
||||
- Compact, verify only 50 remain, verify content
|
||||
- Read Go-created volume fixtures
|
||||
|
||||
---
|
||||
|
||||
### Phase 4: Store (Multi-Volume, Multi-Disk Manager)
|
||||
**Goal:** Manage multiple volumes across multiple disk directories.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/store.go` → `src/storage/store.rs`
|
||||
- `weed/storage/disk_location.go` → `src/storage/disk_location.rs`
|
||||
- `weed/storage/store_ec.go`
|
||||
- `weed/storage/store_state.go`
|
||||
|
||||
**Steps:**
|
||||
|
||||
4.1. **DiskLocation** (`disk_location.rs`):
|
||||
- Directory path, max volume count, min free space, disk type, tags
|
||||
- Load all volumes from directory on startup
|
||||
- Track free space, check writable
|
||||
|
||||
4.2. **Store** (`store.rs`):
|
||||
- Vector of `DiskLocation`s
|
||||
- `GetVolume(volumeId)` → lookup across all locations
|
||||
- `HasVolume(volumeId)` check
|
||||
- `AllocateVolume(...)` — create new volume in appropriate location
|
||||
- `DeleteVolume(...)`, `MountVolume(...)`, `UnmountVolume(...)`
|
||||
- `DeleteCollection(collection)` — delete all volumes of a collection
|
||||
- Collect volume status for heartbeat
|
||||
- `SetStopping()`, `Close()`
|
||||
- Persistent state (maintenance mode) via `store_state.go`
|
||||
|
||||
4.3. **Store state** — `VolumeServerState` protobuf with maintenance flag, persisted to disk.
|
||||
|
||||
4.4. **Tests:**
|
||||
- Create store with 2 dirs, allocate volumes in each, verify load balancing
|
||||
- Mount/unmount/delete lifecycle
|
||||
- State persistence across restart
|
||||
|
||||
---
|
||||
|
||||
### Phase 5: Erasure Coding
|
||||
**Goal:** Full EC shard encode/decode/read/write/rebuild.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/erasure_coding/` (3,599 lines)
|
||||
|
||||
**Steps:**
|
||||
|
||||
5.1. **EC volume + shard structs** — `EcVolume`, `EcShard` with file handles for `.ec00`–`.ec13` shard files + `.ecx` index + `.ecj` journal.
|
||||
|
||||
5.2. **EC encoder** — Reed-Solomon 10+4 (configurable) encoding using `reed-solomon-erasure` crate:
|
||||
- `VolumeEcShardsGenerate`: read .dat → split into data shards → compute parity → write .ec00-.ec13 + .ecx
|
||||
|
||||
5.3. **EC decoder/reader** — reconstruct data from any 10 of 14 shards:
|
||||
- `EcShardRead`: read range from a specific shard
|
||||
- Locate needle in EC volume via .ecx index
|
||||
- Handle cross-shard needle reads
|
||||
|
||||
5.4. **EC shard operations:**
|
||||
- Copy, delete, mount, unmount shards
|
||||
- `VolumeEcShardsRebuild`: rebuild missing shards from remaining
|
||||
- `VolumeEcShardsToVolume`: reconstruct .dat from EC shards
|
||||
- `VolumeEcBlobDelete`: mark deleted in EC journal
|
||||
- `VolumeEcShardsInfo`: report shard metadata
|
||||
|
||||
5.5. **Tests:**
|
||||
- Encode a volume → verify 14 shards created
|
||||
- Delete 4 shards → rebuild → verify data intact
|
||||
- Read individual needles from EC volume
|
||||
- Cross-compat with Go-generated EC shards
|
||||
|
||||
---
|
||||
|
||||
### Phase 6: Backend / Tiered Storage
|
||||
**Goal:** Support tiered storage to remote backends (S3, etc).
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/storage/backend/` (1,850 lines)
|
||||
|
||||
**Steps:**
|
||||
|
||||
6.1. **Backend trait** — abstract `BackendStorage` trait with `ReadAt`, `WriteAt`, `Truncate`, `Close`, `Name`.
|
||||
|
||||
6.2. **Disk backend** — default local disk implementation.
|
||||
|
||||
6.3. **S3 backend** — upload .dat to S3, read ranges via S3 range requests.
|
||||
|
||||
6.4. **Tier move operations:**
|
||||
- `VolumeTierMoveDatToRemote`: upload .dat to remote, optionally delete local
|
||||
- `VolumeTierMoveDatFromRemote`: download .dat from remote
|
||||
|
||||
6.5. **Tests:**
|
||||
- Disk backend read/write round-trip
|
||||
- S3 backend with mock/localstack
|
||||
|
||||
---
|
||||
|
||||
### Phase 7: Security Layer
|
||||
**Goal:** JWT authentication, whitelist guard, TLS configuration.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/security/guard.go` → `src/security/guard.rs`
|
||||
- `weed/security/jwt.go` → `src/security/jwt.rs`
|
||||
- `weed/security/tls.go` → `src/security/tls.rs`
|
||||
|
||||
**Steps:**
|
||||
|
||||
7.1. **Guard** (`guard.rs`):
|
||||
- Whitelist IP check (exact match on `r.RemoteAddr`)
|
||||
- Wrap handlers with whitelist enforcement
|
||||
- `UpdateWhiteList()` for live reload
|
||||
|
||||
7.2. **JWT** (`jwt.rs`):
|
||||
- `SeaweedFileIdClaims` with `fid` field
|
||||
- Sign with HMAC-SHA256
|
||||
- Verify + decode with expiry check
|
||||
- Separate signing keys for read vs write
|
||||
- `GetJwt(request)` — extract from `Authorization: Bearer` header or `jwt` query param
|
||||
|
||||
7.3. **TLS** (`tls.rs`):
|
||||
- Load server TLS cert/key for gRPC and HTTPS
|
||||
- Load client TLS for mutual TLS
|
||||
- Read from `security.toml` config (same format as Go's viper config)
|
||||
|
||||
7.4. **Tests:**
|
||||
- JWT sign → verify round-trip
|
||||
- JWT with wrong key → reject
|
||||
- JWT with expired token → reject
|
||||
- JWT fid mismatch → reject
|
||||
- Whitelist allow/deny
|
||||
|
||||
---
|
||||
|
||||
### Phase 8: Prometheus Metrics
|
||||
**Goal:** Export same metric names as Go for dashboard compatibility.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/stats/metrics.go` (volume server counters/gauges/histograms)
|
||||
|
||||
**Steps:**
|
||||
|
||||
8.1. Define all Prometheus metrics matching Go names:
|
||||
- `VolumeServerRequestCounter` (labels: method, status)
|
||||
- `VolumeServerRequestHistogram` (labels: method)
|
||||
- `VolumeServerInFlightRequestsGauge` (labels: method)
|
||||
- `VolumeServerInFlightUploadSize`
|
||||
- `VolumeServerInFlightDownloadSize`
|
||||
- `VolumeServerConcurrentUploadLimit`
|
||||
- `VolumeServerConcurrentDownloadLimit`
|
||||
- `VolumeServerHandlerCounter` (labels: type — UploadLimitCond, DownloadLimitCond)
|
||||
- Read/Write/Delete request counters
|
||||
|
||||
8.2. Metrics HTTP endpoint on `--metricsPort`.
|
||||
|
||||
8.3. Optional push-based metrics loop (`LoopPushingMetric`).
|
||||
|
||||
8.4. **Test:** Verify metric names and labels match Go output.
|
||||
|
||||
---
|
||||
|
||||
### Phase 9: HTTP Server & Handlers
|
||||
**Goal:** All HTTP endpoints with exact same behavior as Go.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/server/volume_server.go` → `src/server/volume_server.rs`
|
||||
- `weed/server/volume_server_handlers.go` → `src/server/http_handlers.rs`
|
||||
- `weed/server/volume_server_handlers_read.go` → `src/server/http_read.rs`
|
||||
- `weed/server/volume_server_handlers_write.go` → `src/server/http_write.rs`
|
||||
- `weed/server/volume_server_handlers_admin.go` → `src/server/http_admin.rs`
|
||||
- `weed/server/volume_server_handlers_helper.go` (URL parsing, proxy, JSON responses)
|
||||
- `weed/server/volume_server_handlers_ui.go` → `src/server/http_admin.rs`
|
||||
|
||||
**Steps:**
|
||||
|
||||
9.1. **URL path parsing** — from `handlers_helper.go`:
|
||||
- Parse `/<vid>,<fid>` and `/<vid>/<fid>` patterns
|
||||
- Extract volume ID, file ID, filename, ext
|
||||
|
||||
9.2. **Route dispatch** — from `privateStoreHandler` and `publicReadOnlyHandler`:
|
||||
- `GET /` → `GetOrHeadHandler`
|
||||
- `HEAD /` → `GetOrHeadHandler`
|
||||
- `POST /` → `PostHandler` (whitelist gated)
|
||||
- `PUT /` → `PostHandler` (whitelist gated)
|
||||
- `DELETE /` → `DeleteHandler` (whitelist gated)
|
||||
- `OPTIONS /` → CORS preflight
|
||||
- `GET /status` → JSON status
|
||||
- `GET /healthz` → health check
|
||||
- `GET /ui/index.html` → HTML UI page
|
||||
- Static resources (CSS/JS for UI)
|
||||
|
||||
9.3. **GET/HEAD handler** (`http_read.rs`) — from `handlers_read.go` (468 lines):
|
||||
- JWT read authorization check
|
||||
- Lookup needle by volume ID + needle ID + cookie
|
||||
- ETag / If-None-Match / If-Modified-Since conditional responses
|
||||
- Content-Type from stored MIME or filename extension
|
||||
- Content-Disposition header
|
||||
- Content-Encoding (gzip/zstd stored data)
|
||||
- Range request support (HTTP 206 Partial Content)
|
||||
- JPEG orientation fix (if configured)
|
||||
- Proxy to replica on local miss (readMode=proxy)
|
||||
- Redirect to replica (readMode=redirect)
|
||||
- Download tracking (in-flight size accounting)
|
||||
|
||||
9.4. **POST/PUT handler** (`http_write.rs`) — from `handlers_write.go` (170 lines):
|
||||
- JWT write authorization check
|
||||
- Multipart form parsing
|
||||
- Extract file data, filename, content type, TTL, last-modified
|
||||
- Optional gzip/zstd compression
|
||||
- Write needle to volume
|
||||
- Replicate to peers (same logic as Go's `DistributedOperation`)
|
||||
- Return JSON: {name, size, eTag, error}
|
||||
|
||||
9.5. **DELETE handler** — already in handlers.go:
|
||||
- JWT authorization
|
||||
- Delete from local volume
|
||||
- Replicate delete to peers
|
||||
- Return JSON result
|
||||
|
||||
9.6. **Admin handlers** (`http_admin.rs`):
|
||||
- `/status` → JSON with volumes, version, disk status
|
||||
- `/healthz` → 200 OK if serving
|
||||
- `/ui/index.html` → HTML dashboard
|
||||
|
||||
9.7. **Concurrency limiting** — from `handlers.go`:
|
||||
- Upload concurrency limit with `sync::Condvar` + timeout
|
||||
- Download concurrency limit with proxy fallback to replicas
|
||||
- HTTP 429 on timeout, 499 on client cancel
|
||||
- Replication traffic bypasses upload limits
|
||||
|
||||
9.8. **Public port** — if configured, separate listener with read-only routes (GET/HEAD/OPTIONS only).
|
||||
|
||||
9.9. **Request ID middleware** — generate unique request ID per request.
|
||||
|
||||
9.10. **Tests:**
|
||||
- Integration: start server → upload file via POST → GET it back → verify content
|
||||
- Integration: upload → DELETE → GET returns 404
|
||||
- Integration: conditional GET with ETag → 304
|
||||
- Integration: range request → 206 with correct bytes
|
||||
- Integration: exceed upload limit → 429
|
||||
- Integration: whitelist enforcement
|
||||
- Integration: JWT enforcement
|
||||
|
||||
---
|
||||
|
||||
### Phase 10: gRPC Service Implementation
|
||||
**Goal:** All 40 gRPC methods with exact logic.
|
||||
|
||||
**Source files to port:**
|
||||
- `weed/server/volume_grpc_admin.go` (380 lines)
|
||||
- `weed/server/volume_grpc_vacuum.go` (124 lines)
|
||||
- `weed/server/volume_grpc_copy.go` (636 lines)
|
||||
- `weed/server/volume_grpc_copy_incremental.go` (66 lines)
|
||||
- `weed/server/volume_grpc_read_write.go` (74 lines)
|
||||
- `weed/server/volume_grpc_batch_delete.go` (124 lines)
|
||||
- `weed/server/volume_grpc_tail.go` (140 lines)
|
||||
- `weed/server/volume_grpc_erasure_coding.go` (619 lines)
|
||||
- `weed/server/volume_grpc_scrub.go` (121 lines)
|
||||
- `weed/server/volume_grpc_tier_upload.go` (98 lines)
|
||||
- `weed/server/volume_grpc_tier_download.go` (85 lines)
|
||||
- `weed/server/volume_grpc_remote.go` (95 lines)
|
||||
- `weed/server/volume_grpc_query.go` (69 lines)
|
||||
- `weed/server/volume_grpc_state.go` (26 lines)
|
||||
- `weed/server/volume_grpc_read_all.go` (35 lines)
|
||||
- `weed/server/volume_grpc_client_to_master.go` (325 lines)
|
||||
|
||||
**Steps (grouped by functional area):**
|
||||
|
||||
10.1. **Implement `tonic::Service` for `VolumeServer`** — the generated trait from proto.
|
||||
|
||||
10.2. **Admin RPCs** (`grpc_admin.rs`):
|
||||
- `AllocateVolume` — create volume on appropriate disk location
|
||||
- `VolumeMount` / `VolumeUnmount` / `VolumeDelete`
|
||||
- `VolumeMarkReadonly` / `VolumeMarkWritable`
|
||||
- `VolumeConfigure` — change replication
|
||||
- `VolumeStatus` — return read-only, size, file counts
|
||||
- `VolumeServerStatus` — disk statuses, memory, version, DC, rack
|
||||
- `VolumeServerLeave` — deregister from master
|
||||
- `DeleteCollection`
|
||||
- `VolumeNeedleStatus` — get needle metadata by ID
|
||||
- `Ping` — latency measurement
|
||||
- `GetState` / `SetState` — maintenance mode
|
||||
|
||||
10.3. **Vacuum RPCs** (`grpc_vacuum.rs`):
|
||||
- `VacuumVolumeCheck` — return garbage ratio
|
||||
- `VacuumVolumeCompact` — stream progress (streaming response)
|
||||
- `VacuumVolumeCommit` — finalize compaction
|
||||
- `VacuumVolumeCleanup` — remove temp files
|
||||
|
||||
10.4. **Copy RPCs** (`grpc_copy.rs`):
|
||||
- `VolumeCopy` — stream .dat/.idx from source to create local copy
|
||||
- `VolumeSyncStatus` — return sync metadata
|
||||
- `VolumeIncrementalCopy` — stream .dat delta since timestamp (streaming)
|
||||
- `CopyFile` — generic file copy by extension (streaming)
|
||||
- `ReceiveFile` — receive streamed file (client streaming)
|
||||
- `ReadVolumeFileStatus` — return file timestamps and sizes
|
||||
|
||||
10.5. **Read/Write RPCs** (`grpc_read_write.rs`):
|
||||
- `ReadNeedleBlob` — raw needle blob read
|
||||
- `ReadNeedleMeta` — needle metadata
|
||||
- `WriteNeedleBlob` — raw needle blob write
|
||||
- `ReadAllNeedles` — stream all needles from volume(s) (streaming)
|
||||
|
||||
10.6. **Batch delete** (`grpc_batch_delete.rs`):
|
||||
- `BatchDelete` — delete multiple file IDs, return per-ID results
|
||||
|
||||
10.7. **Tail RPCs** (`grpc_tail.rs`):
|
||||
- `VolumeTailSender` — stream new needles since timestamp (streaming)
|
||||
- `VolumeTailReceiver` — connect to another volume server and tail its changes
|
||||
|
||||
10.8. **Erasure coding RPCs** (`grpc_erasure_coding.rs`):
|
||||
- `VolumeEcShardsGenerate` — generate EC shards from volume
|
||||
- `VolumeEcShardsRebuild` — rebuild missing shards
|
||||
- `VolumeEcShardsCopy` — copy shards from another server
|
||||
- `VolumeEcShardsDelete` — delete EC shards
|
||||
- `VolumeEcShardsMount` / `VolumeEcShardsUnmount`
|
||||
- `VolumeEcShardRead` — read from EC shard (streaming)
|
||||
- `VolumeEcBlobDelete` — mark blob deleted in EC volume
|
||||
- `VolumeEcShardsToVolume` — reconstruct volume from EC shards
|
||||
- `VolumeEcShardsInfo` — return shard metadata
|
||||
|
||||
10.9. **Scrub RPCs** (`grpc_scrub.rs`):
|
||||
- `ScrubVolume` — integrity check volumes (INDEX / FULL / LOCAL modes)
|
||||
- `ScrubEcVolume` — integrity check EC volumes
|
||||
|
||||
10.10. **Tier RPCs** (`grpc_tier.rs`):
|
||||
- `VolumeTierMoveDatToRemote` — upload to remote backend (streaming progress)
|
||||
- `VolumeTierMoveDatFromRemote` — download from remote (streaming progress)
|
||||
|
||||
10.11. **Remote storage** (`grpc_remote.rs`):
|
||||
- `FetchAndWriteNeedle` — fetch from remote storage, write locally, replicate
|
||||
|
||||
10.12. **Query** (`grpc_query.rs`):
|
||||
- `Query` — experimental CSV/JSON/Parquet select on stored data (streaming)
|
||||
|
||||
10.13. **Master heartbeat** (`grpc_client_to_master.rs`):
|
||||
- `heartbeat()` background task — periodic gRPC stream to master
|
||||
- Send: volume info, EC shard info, disk stats, has-no-space flags, deleted volumes
|
||||
- Receive: volume size limit, leader address, metrics config
|
||||
- Reconnect on failure with backoff
|
||||
- `StopHeartbeat()` for graceful shutdown
|
||||
|
||||
10.14. **Tests:**
|
||||
- Integration test per RPC: call via tonic client → verify response
|
||||
- Streaming RPCs: verify all chunks received
|
||||
- Error cases: invalid volume ID, non-existent volume, etc.
|
||||
- Heartbeat: mock master gRPC server, verify registration
|
||||
|
||||
---
|
||||
|
||||
### Phase 11: Startup, Lifecycle & Graceful Shutdown
|
||||
**Goal:** Full server startup matching Go's `runVolume()` and `startVolumeServer()`.
|
||||
|
||||
**Steps:**
|
||||
|
||||
11.1. **Startup sequence** (match `volume.go` exactly):
|
||||
1. Load security configuration from `security.toml`
|
||||
2. Start metrics server on metrics port
|
||||
3. Parse folder/max/minFreeSpace/diskType/tags
|
||||
4. Validate all directory writable
|
||||
5. Resolve IP, bind IP, public URL, gRPC port
|
||||
6. Create `VolumeServer` struct
|
||||
7. Check with master (initial handshake)
|
||||
8. Create `Store` (loads all existing volumes from disk)
|
||||
9. Create security `Guard`
|
||||
10. Register HTTP routes on admin mux
|
||||
11. Optionally register public mux
|
||||
12. Start gRPC server on gRPC port
|
||||
13. Start public HTTP server (if separated)
|
||||
14. Start cluster HTTP server (with optional TLS)
|
||||
15. Start heartbeat background task
|
||||
16. Start metrics push loop
|
||||
17. Register SIGHUP handler for config reload + new volume loading
|
||||
|
||||
11.2. **Graceful shutdown** (match Go exactly):
|
||||
1. On SIGINT/SIGTERM:
|
||||
2. Stop heartbeat (notify master we're leaving)
|
||||
3. Wait `preStopSeconds`
|
||||
4. Stop public HTTP server
|
||||
5. Stop cluster HTTP server
|
||||
6. Graceful stop gRPC server
|
||||
7. `volumeServer.Shutdown()` → `store.Close()` (flush all volumes)
|
||||
|
||||
11.3. **Reload** (SIGHUP):
|
||||
- Reload security config
|
||||
- Update whitelist
|
||||
- Load newly appeared volumes from disk
|
||||
|
||||
11.4. **Tests:**
|
||||
- Start server → send SIGTERM → verify clean shutdown
|
||||
- Start server → SIGHUP → verify config reloaded
|
||||
|
||||
---
|
||||
|
||||
### Phase 12: Integration & Cross-Compatibility Testing
|
||||
**Goal:** Rust volume server is a drop-in replacement for Go volume server.
|
||||
|
||||
**Steps:**
|
||||
|
||||
12.1. **Binary compatibility tests:**
|
||||
- Create volumes with Go volume server
|
||||
- Start Rust volume server on same data directory
|
||||
- Read all data → verify identical
|
||||
- Write new data with Rust → read with Go → verify
|
||||
|
||||
12.2. **API compatibility tests:**
|
||||
- Run same HTTP requests against both Go and Rust servers
|
||||
- Compare response bodies, headers, status codes
|
||||
- Test all gRPC RPCs against both
|
||||
|
||||
12.3. **Master interop test:**
|
||||
- Start Go master server
|
||||
- Register Rust volume server
|
||||
- Verify heartbeat works
|
||||
- Verify volume assignment works
|
||||
- Upload via filer → stored on Rust volume server → read back
|
||||
|
||||
12.4. **Performance benchmarks:**
|
||||
- Throughput: sequential writes, sequential reads
|
||||
- Latency: p50/p99 for read/write
|
||||
- Concurrency: parallel reads/writes
|
||||
- Compare Rust vs Go numbers
|
||||
|
||||
12.5. **Edge cases:**
|
||||
- Volume at max size
|
||||
- Disk full handling
|
||||
- Corrupt .dat file recovery
|
||||
- Network partition during replication
|
||||
- EC shard loss + rebuild
|
||||
|
||||
---
|
||||
|
||||
## Execution Order & Dependencies
|
||||
|
||||
```
|
||||
Phase 1 (Skeleton + CLI) ← no deps, start here
|
||||
↓
|
||||
Phase 2 (Storage types) ← needs Phase 1 (types used everywhere)
|
||||
↓
|
||||
Phase 3 (Volume struct) ← needs Phase 2
|
||||
↓
|
||||
Phase 4 (Store manager) ← needs Phase 3
|
||||
↓
|
||||
Phase 7 (Security) ← independent, can parallel with 3-4
|
||||
Phase 8 (Metrics) ← independent, can parallel with 3-4
|
||||
↓
|
||||
Phase 9 (HTTP server) ← needs Phase 4 + 7 + 8
|
||||
Phase 10 (gRPC server) ← needs Phase 4 + 7 + 8
|
||||
↓
|
||||
Phase 5 (Erasure coding) ← needs Phase 4, wire into Phase 10
|
||||
Phase 6 (Tiered storage) ← needs Phase 4, wire into Phase 10
|
||||
↓
|
||||
Phase 11 (Startup + shutdown) ← needs Phase 9 + 10
|
||||
↓
|
||||
Phase 12 (Integration tests) ← needs all above
|
||||
```
|
||||
|
||||
## Estimated Scope
|
||||
|
||||
| Phase | Estimated Rust Lines | Complexity |
|
||||
|---|---|---|
|
||||
| 1. Skeleton + CLI | ~400 | Low |
|
||||
| 2. Storage types | ~2,000 | High (binary compat critical) |
|
||||
| 3. Volume struct | ~2,500 | High |
|
||||
| 4. Store manager | ~1,000 | Medium |
|
||||
| 5. Erasure coding | ~3,000 | High |
|
||||
| 6. Tiered storage | ~1,500 | Medium |
|
||||
| 7. Security | ~500 | Medium |
|
||||
| 8. Metrics | ~300 | Low |
|
||||
| 9. HTTP server | ~2,000 | High |
|
||||
| 10. gRPC server | ~3,500 | High |
|
||||
| 11. Startup/shutdown | ~500 | Medium |
|
||||
| 12. Integration tests | ~2,000 | Medium |
|
||||
| **Total** | **~19,000** | |
|
||||
|
||||
## Critical Invariants to Preserve
|
||||
|
||||
1. **Binary format compatibility** — Rust must read/write `.dat`, `.idx`, `.vif`, `.ecX` files identically to Go. A single byte off = data loss.
|
||||
2. **gRPC wire compatibility** — Same proto, same field semantics. Go master must talk to Rust volume server seamlessly.
|
||||
3. **HTTP API compatibility** — Same URL patterns, same JSON response shapes, same headers, same status codes.
|
||||
4. **Replication protocol** — Write replication between Go and Rust volume servers must work bidirectionally.
|
||||
5. **Heartbeat protocol** — Rust volume server must register with Go master and maintain heartbeat.
|
||||
6. **CRC32 algorithm** — Must use IEEE polynomial (same as Go's `crc32.ChecksumIEEE`).
|
||||
7. **JWT compatibility** — Tokens signed by Go filer/master must be verifiable by Rust volume server and vice versa.
|
||||
@@ -7,8 +7,8 @@ LABEL author="Chris Lu"
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
RUN apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y \
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
|
||||
@@ -16,38 +16,9 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
|
||||
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
|
||||
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
|
||||
|
||||
# Rust volume server: use pre-built binary from CI when available (placed in
|
||||
# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile
|
||||
# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build
|
||||
# for non-native architectures.
|
||||
FROM alpine:3.23 as rust_builder
|
||||
ARG TARGETARCH
|
||||
ARG TAGS
|
||||
COPY weed-volume-prebuilt/ /prebuilt/
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
|
||||
WORKDIR /build/seaweed-volume
|
||||
RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
|
||||
echo "Using pre-built Rust binary for ${TARGETARCH}" && \
|
||||
cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \
|
||||
elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
|
||||
if [ "$TAGS" = "5BytesOffset" ]; then \
|
||||
cargo build --release; \
|
||||
else \
|
||||
cargo build --release --no-default-features; \
|
||||
fi && \
|
||||
cp target/release/weed-volume /weed-volume; \
|
||||
else \
|
||||
echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
|
||||
touch /weed-volume; \
|
||||
fi
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
COPY --from=builder /go/bin/weed /usr/bin/
|
||||
# Copy Rust volume server binary (real binary on amd64/arm64, empty placeholder on other platforms)
|
||||
COPY --from=rust_builder /weed-volume /usr/bin/weed-volume
|
||||
RUN mkdir -p /etc/seaweedfs
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
@@ -56,8 +27,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@@ -89,8 +59,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -7,8 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec && \
|
||||
RUN apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@@ -40,8 +39,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -34,8 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@@ -68,8 +67,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -17,8 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "steward",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "steward-key",
|
||||
"secretKey": "steward-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List",
|
||||
"Write"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "le001",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "le001-key",
|
||||
"secretKey": "le001-secret"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+2
-29
@@ -37,21 +37,17 @@ if [ "$(id -u)" = "0" ]; then
|
||||
fi
|
||||
|
||||
isArgPassed() {
|
||||
# Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9
|
||||
# library accepts both, and users may pick either form on the CLI.
|
||||
arg="$1"
|
||||
argWithEqualSign="$1="
|
||||
argDouble="-$1"
|
||||
argDoubleWithEqualSign="-$1="
|
||||
shift
|
||||
while [ $# -gt 0 ]; do
|
||||
passedArg="$1"
|
||||
shift
|
||||
case $passedArg in
|
||||
"$arg"|"$argDouble")
|
||||
"$arg")
|
||||
return 0
|
||||
;;
|
||||
"$argWithEqualSign"*|"$argDoubleWithEqualSign"*)
|
||||
"$argWithEqualSign"*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
@@ -76,20 +72,6 @@ case "$1" in
|
||||
exec /usr/bin/weed -logtostderr=true volume $ARGS $@
|
||||
;;
|
||||
|
||||
'volume-rust')
|
||||
ARGS="-dir /data -max 0"
|
||||
if isArgPassed "-max" "$@"; then
|
||||
ARGS="-dir /data"
|
||||
fi
|
||||
shift
|
||||
if [ ! -s /usr/bin/weed-volume ]; then
|
||||
echo "Error: Rust volume server is not available on this platform ($(uname -m))." >&2
|
||||
echo "Use 'volume' for the Go volume server instead." >&2
|
||||
exit 1
|
||||
fi
|
||||
exec /usr/bin/weed-volume $ARGS $@
|
||||
;;
|
||||
|
||||
'server')
|
||||
ARGS="-dir=/data -volume.max=0 -master.volumeSizeLimitMB=1024"
|
||||
if isArgPassed "-volume.max" "$@"; then
|
||||
@@ -99,15 +81,6 @@ case "$1" in
|
||||
exec /usr/bin/weed -logtostderr=true server $ARGS $@
|
||||
;;
|
||||
|
||||
'mini')
|
||||
ARGS="-dir=/data"
|
||||
if isArgPassed "-dir" "$@"; then
|
||||
ARGS=""
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true mini $ARGS $@
|
||||
;;
|
||||
|
||||
'filer')
|
||||
ARGS=""
|
||||
shift
|
||||
|
||||
@@ -4,7 +4,7 @@ go 1.25.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.2
|
||||
cloud.google.com/go/pubsub v1.50.1
|
||||
cloud.google.com/go/storage v1.60.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
@@ -34,12 +34,12 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/jackc/pgx/v5 v5.8.0
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
@@ -67,7 +67,7 @@ require (
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.8
|
||||
github.com/seaweedfs/raft v1.1.7
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
@@ -83,82 +83,81 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0
|
||||
github.com/viant/ptrie v1.0.1
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.2.0
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.10
|
||||
go.etcd.io/etcd/client/v3 v3.6.7
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.50.0
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
|
||||
golang.org/x/image v0.38.0
|
||||
golang.org/x/net v0.53.0
|
||||
golang.org/x/image v0.36.0
|
||||
golang.org/x/net v0.51.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.43.0
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.274.0
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/api v0.267.0
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.49.1
|
||||
modernc.org/sqlite v1.46.1
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.26.0
|
||||
cloud.google.com/go/kms v1.25.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.977
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
|
||||
github.com/apache/iceberg-go v0.5.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.6
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.8.0
|
||||
github.com/getsentry/sentry-go v0.43.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.28.0
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.73.5
|
||||
github.com/rclone/rclone v1.73.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2
|
||||
github.com/shirou/gopsutil/v4 v4.26.2
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.3
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
@@ -172,17 +171,17 @@ require (
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.8.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38 // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -212,6 +211,7 @@ require (
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
@@ -224,7 +224,7 @@ require (
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -258,7 +258,6 @@ require (
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
@@ -285,14 +284,14 @@ require (
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth v0.18.1 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
@@ -303,7 +302,7 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
@@ -324,23 +323,23 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/smithy-go v1.25.0
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -372,7 +371,7 @@ require (
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
@@ -388,7 +387,7 @@ require (
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
@@ -479,7 +478,7 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@@ -487,27 +486,27 @@ require (
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.42.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/libc v1.68.0 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
|
||||
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
|
||||
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
|
||||
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
|
||||
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
|
||||
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
|
||||
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
|
||||
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
|
||||
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
|
||||
cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
|
||||
cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
|
||||
cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
|
||||
cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
|
||||
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
|
||||
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
|
||||
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
|
||||
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA=
|
||||
cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
@@ -391,10 +391,10 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp
|
||||
cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI=
|
||||
cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0=
|
||||
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
|
||||
cloud.google.com/go/pubsub v1.50.2 h1:54Up97HnThdP4H8jjWJSSQ/mnYG2EKon7ZSNETRq0tM=
|
||||
cloud.google.com/go/pubsub v1.50.2/go.mod h1:jyCWeZdGFqd4mitSsBERnJcpqaHBsxQoPkNvjj4sp0w=
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 h1:oMKNiBQpXImRWnHYla9uSU66ZzByZwBSCJOEs/pTKVg=
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0/go.mod h1:2lS/XQKq5qtOMs6kHBK+WX1ytUC36kLl2ig3zqsGUx8=
|
||||
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
|
||||
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
|
||||
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
|
||||
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
|
||||
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
|
||||
@@ -577,8 +577,8 @@ github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M=
|
||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A=
|
||||
github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
|
||||
@@ -593,8 +593,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38 h1:YCyHs3wUXEe2BEWn40vcoyaQ2ruHNmNwkasfo3Th16A=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 h1:W8S9TrAyZ4//3PXsU6+Bi+fe/6uIL986GyS7PVzIDL4=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 h1:lMHTplAYI9FtmCo/QOcpRxmPA5REVAct1r2riQmDQKw=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264/go.mod h1:wGqkOzRu/ClJibvDgcfuJNAqI2nLhe8g91tPlDKRCdE=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||
@@ -692,8 +692,8 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 h1:r/gg2gzUXiXoy72VU3jnODh8l/5rL0aslnTAbtmai/U=
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87/go.mod h1:IJTMBTlHe7cDOhRh0ioGuEKBl5iTR6xPfl5BN4AgirU=
|
||||
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq/RrkjCcxCnqUxhwf8=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0 h1:Omnzb1Z/P90Dr2TbVNu54ICQL7TKVIIsJO231w484HU=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
|
||||
github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
@@ -714,50 +714,50 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1 h1:IbWiN670htmBioc+Zj32vSpJgQ2+OYSlvTvfQ1nCORQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1/go.mod h1:tw/B596EUhBWDFGdDGuLC21fVU4A3s4/5Efy8S39W18=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U=
|
||||
github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
|
||||
github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
@@ -1071,13 +1071,13 @@ github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3
|
||||
github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
|
||||
github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0=
|
||||
github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY=
|
||||
github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM=
|
||||
github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
|
||||
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
|
||||
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
|
||||
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
|
||||
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
|
||||
@@ -1383,8 +1383,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKc
|
||||
github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
|
||||
github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A=
|
||||
github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko=
|
||||
github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0=
|
||||
github.com/hashicorp/vault/api v1.22.0/go.mod h1:IUZA2cDvr4Ok3+NtK2Oq/r+lJeXkeCrHRmqdyWfpmGM=
|
||||
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
|
||||
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
@@ -1397,14 +1397,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd h1:dSIuz2mpJAPQfhHYtG57D0qwSkgC/vQ69gHfeyQ4kxA=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c h1:r+KtxPyrhsYeNbsfeqTfEM8xRdwgV6LuNhLZxpXecb4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
|
||||
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jaegertracing/jaeger v1.47.0 h1:XXxTMO+GxX930gxKWsg90rFr6RswkCRIW0AgWFnTYsg=
|
||||
@@ -1783,14 +1783,12 @@ github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3Adq
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.73.5 h1:r8a9JHYIWUqk7hNRJuMJ3cROkKfB2zmfkADg8ZLYh6I=
|
||||
github.com/rclone/rclone v1.73.5/go.mod h1:WVv8gvA/lEl/Y37e8I8yosm7ZY+Szq7ujXbJS8Ol63o=
|
||||
github.com/rclone/rclone v1.73.1 h1:FCGZUd7sXO8+nIpbuXGPkd+f23o/ZACefrHtRGDAiDM=
|
||||
github.com/rclone/rclone v1.73.1/go.mod h1:T9HeSXv/SI9qJFkYLqsjY/5ljIFWJTVmTrG7H/4JrPY=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
@@ -1840,12 +1838,14 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3 h1:rJufGrHImTx7yoGUmetUi+To4LrmTQJROJnBHWt92ic=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/seaweedfs/raft v1.1.8 h1:a5f+wSnriRjINLqLqMY7DDNNfU1+kv6TKBuOZa0KDwU=
|
||||
github.com/seaweedfs/raft v1.1.8/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/seaweedfs/raft v1.1.7 h1:3mVJZ2p4rdvBtbbrHROPjYKtH+q5qjMBc56G6VRu1kA=
|
||||
github.com/seaweedfs/raft v1.1.7/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
@@ -2028,10 +2028,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/willscott/go-nfs v0.0.3 h1:Z5fHVxMsppgEucdkKBN26Vou19MtEM875NmRwj156RE=
|
||||
github.com/willscott/go-nfs v0.0.3/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
@@ -2042,8 +2038,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
|
||||
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@@ -2064,14 +2060,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 h1:avIdi8eGXjKbn1WLokNR1Ofnz1k8t7tJ88YQLD/iCi8=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2 h1:Bzra5hUQIWrxknQ0oV6HE5q/eZU1VhHMKSZ/XWpqMgM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2/go.mod h1:VYUUkRJkKuQPkIpgtZJj6+58Fa2g8ccAqdmaaK6HP5k=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3 h1:hTwpF+PdbuR7vcixN+4AC6yu4asaUIAljQCxL51eDII=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
|
||||
@@ -2103,16 +2099,16 @@ github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
|
||||
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
|
||||
go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
|
||||
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10/go.mod h1:pdV4VeFmvhdNjB4LWRkC8ReLyRBAxUOze3GarMhE2sk=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10 h1:tBT7podcPhuVbCVkAEzx8bC5I+aqxfLwBN8/As1arrA=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10 h1:J598zJ+C/ZPvImypmq5waj84+bovePrlZERHklf34y0=
|
||||
go.etcd.io/etcd/client/v3 v3.6.10/go.mod h1:iHhUDUcEwaKs1YFq3MgmI9U4zhTVasp/vgdVbFf1RS8=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
|
||||
go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
@@ -2134,8 +2130,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
|
||||
@@ -2150,14 +2146,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
@@ -2214,8 +2210,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -2246,8 +2242,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2281,8 +2277,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2350,8 +2346,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -2509,11 +2505,11 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc=
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2529,8 +2525,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
|
||||
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -2551,15 +2547,15 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
@@ -2630,8 +2626,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -2709,8 +2705,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
|
||||
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -2844,12 +2840,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -2981,19 +2977,19 @@ modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
|
||||
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=
|
||||
modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8=
|
||||
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
|
||||
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
|
||||
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
|
||||
modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
|
||||
modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU=
|
||||
modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0=
|
||||
modernc.org/ccgo/v4 v4.30.2 h1:4yPaaq9dXYXZ2V8s1UgrC3KIj580l2N4ClrLwnbv2so=
|
||||
modernc.org/ccgo/v4 v4.30.2/go.mod h1:yZMnhWEdW0qw3EtCndG1+ldRrVGS+bIwyWmAWzS0XEw=
|
||||
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
|
||||
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
|
||||
@@ -3008,8 +3004,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
|
||||
modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
|
||||
modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
|
||||
modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
|
||||
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
|
||||
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
|
||||
modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ=
|
||||
modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0=
|
||||
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
@@ -3028,8 +3024,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
|
||||
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
|
||||
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU=
|
||||
modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
|
||||
-275
@@ -1,275 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# SeaweedFS Installer
|
||||
# Downloads Go and/or Rust binaries from GitHub releases.
|
||||
#
|
||||
# Usage:
|
||||
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
|
||||
# curl -fsSL ... | bash -s -- --version v3.93 --dir /usr/local/bin
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, all (default: weed)
|
||||
# --version VER Release version tag (default: latest)
|
||||
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
|
||||
# --dir DIR Installation directory (default: /usr/local/bin)
|
||||
# --help Show this help message
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO="seaweedfs/seaweedfs"
|
||||
COMPONENT="weed"
|
||||
VERSION=""
|
||||
LARGE_DISK=false
|
||||
INSTALL_DIR="/usr/local/bin"
|
||||
|
||||
# Colors (if terminal supports them)
|
||||
if [ -t 1 ]; then
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
else
|
||||
RED='' GREEN='' YELLOW='' BLUE='' NC=''
|
||||
fi
|
||||
|
||||
info() { echo -e "${BLUE}[info]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[ok]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[warn]${NC} $*"; }
|
||||
error() { echo -e "${RED}[error]${NC} $*" >&2; exit 1; }
|
||||
|
||||
usage() {
|
||||
sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//'
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--component) COMPONENT="$2"; shift 2 ;;
|
||||
--version) VERSION="$2"; shift 2 ;;
|
||||
--large-disk) LARGE_DISK=true; shift ;;
|
||||
--dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||
--help|-h) usage ;;
|
||||
*) error "Unknown option: $1. Use --help for usage." ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Detect OS and architecture
|
||||
detect_platform() {
|
||||
local os arch
|
||||
|
||||
case "$(uname -s)" in
|
||||
Linux*) os="linux" ;;
|
||||
Darwin*) os="darwin" ;;
|
||||
MINGW*|MSYS*|CYGWIN*) os="windows" ;;
|
||||
FreeBSD*) os="freebsd" ;;
|
||||
*) error "Unsupported OS: $(uname -s)" ;;
|
||||
esac
|
||||
|
||||
case "$(uname -m)" in
|
||||
x86_64|amd64) arch="amd64" ;;
|
||||
aarch64|arm64) arch="arm64" ;;
|
||||
armv7l|armv6l) arch="arm" ;;
|
||||
*) error "Unsupported architecture: $(uname -m)" ;;
|
||||
esac
|
||||
|
||||
echo "${os}" "${arch}"
|
||||
}
|
||||
|
||||
# Get latest release tag from GitHub API
|
||||
get_latest_version() {
|
||||
local url="https://api.github.com/repos/${REPO}/releases/latest"
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
else
|
||||
error "Neither curl nor wget found. Please install one."
|
||||
fi
|
||||
}
|
||||
|
||||
# Download a file
|
||||
download() {
|
||||
local url="$1" dest="$2"
|
||||
info "Downloading ${url}"
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -fsSL -o "$dest" "$url"
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -qO "$dest" "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
# Build Go weed binary asset name
|
||||
go_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
local suffix="${os}_${arch}"
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
suffix="${suffix}_large_disk"
|
||||
fi
|
||||
echo "${suffix}.tar.gz"
|
||||
}
|
||||
|
||||
# Build Rust volume server asset name
|
||||
rust_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
local prefix="weed-volume"
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
prefix="weed-volume_large_disk"
|
||||
else
|
||||
prefix="weed-volume"
|
||||
fi
|
||||
local suffix="${os}_${arch}"
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "${prefix}_${suffix}.zip"
|
||||
else
|
||||
echo "${prefix}_${suffix}.tar.gz"
|
||||
fi
|
||||
}
|
||||
|
||||
# Install a single component
|
||||
install_component() {
|
||||
local component="$1" os="$2" arch="$3"
|
||||
local asset_name download_url tmpdir
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
trap "rm -rf '$tmpdir'" EXIT
|
||||
|
||||
case "$component" in
|
||||
weed)
|
||||
asset_name="$(go_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
|
||||
# The Go release action puts the binary inside a directory
|
||||
local weed_bin
|
||||
weed_bin="$(find "$tmpdir" -name 'weed' -type f | head -1)"
|
||||
if [ -z "$weed_bin" ]; then
|
||||
weed_bin="$(find "$tmpdir" -name 'weed.exe' -type f | head -1)"
|
||||
fi
|
||||
if [ -z "$weed_bin" ]; then
|
||||
error "Could not find weed binary in archive"
|
||||
fi
|
||||
|
||||
chmod +x "$weed_bin"
|
||||
install_binary "$weed_bin" "weed"
|
||||
ok "Installed weed to ${INSTALL_DIR}/weed"
|
||||
;;
|
||||
|
||||
volume-rust)
|
||||
# Check platform support for Rust volume server
|
||||
case "$os" in
|
||||
linux|darwin|windows) ;;
|
||||
*) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
amd64|arm64) ;;
|
||||
*) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
|
||||
esac
|
||||
|
||||
asset_name="$(rust_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
if [ "$os" = "windows" ]; then
|
||||
unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir"
|
||||
else
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
fi
|
||||
|
||||
local rust_bin
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
rust_bin="$(find "$tmpdir" -name 'weed-volume-large-disk*' -type f | head -1)"
|
||||
else
|
||||
rust_bin="$(find "$tmpdir" -name 'weed-volume-normal*' -type f | head -1)"
|
||||
fi
|
||||
if [ -z "$rust_bin" ]; then
|
||||
rust_bin="$(find "$tmpdir" -name 'weed-volume*' -type f | head -1)"
|
||||
fi
|
||||
if [ -z "$rust_bin" ]; then
|
||||
error "Could not find weed-volume binary in archive"
|
||||
fi
|
||||
|
||||
chmod +x "$rust_bin"
|
||||
local dest_name="weed-volume"
|
||||
if [ "$os" = "windows" ]; then
|
||||
dest_name="weed-volume.exe"
|
||||
fi
|
||||
install_binary "$rust_bin" "$dest_name"
|
||||
ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
|
||||
;;
|
||||
|
||||
*)
|
||||
error "Unknown component: ${component}. Use: weed, volume-rust, all"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Copy binary to install dir, using sudo if needed
|
||||
install_binary() {
|
||||
local src="$1" name="$2"
|
||||
local dest="${INSTALL_DIR}/${name}"
|
||||
|
||||
mkdir -p "$INSTALL_DIR" 2>/dev/null || true
|
||||
|
||||
if [ -w "$INSTALL_DIR" ]; then
|
||||
cp "$src" "$dest"
|
||||
else
|
||||
info "Need elevated permissions to write to ${INSTALL_DIR}"
|
||||
sudo cp "$src" "$dest"
|
||||
fi
|
||||
chmod +x "$dest" 2>/dev/null || sudo chmod +x "$dest"
|
||||
}
|
||||
|
||||
main() {
|
||||
info "SeaweedFS Installer"
|
||||
|
||||
read -r os arch <<< "$(detect_platform)"
|
||||
info "Detected platform: ${os}/${arch}"
|
||||
|
||||
if [ -z "$VERSION" ]; then
|
||||
info "Resolving latest release..."
|
||||
VERSION="$(get_latest_version)"
|
||||
if [ -z "$VERSION" ]; then
|
||||
error "Could not determine latest version. Specify with --version"
|
||||
fi
|
||||
fi
|
||||
info "Version: ${VERSION}"
|
||||
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
info "Variant: large disk (8TB max volume)"
|
||||
else
|
||||
info "Variant: normal (32GB max volume)"
|
||||
fi
|
||||
|
||||
case "$COMPONENT" in
|
||||
all)
|
||||
install_component "weed" "$os" "$arch"
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
;;
|
||||
*)
|
||||
install_component "$COMPONENT" "$os" "$arch"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
ok "Installation complete!"
|
||||
if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed: ${INSTALL_DIR}/weed"
|
||||
fi
|
||||
if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed-volume: ${INSTALL_DIR}/weed-volume"
|
||||
fi
|
||||
echo ""
|
||||
info "Quick start:"
|
||||
info " weed master # Start master server"
|
||||
info " weed volume -mserver=localhost:9333 # Start Go volume server"
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
}
|
||||
|
||||
main
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.23"
|
||||
appVersion: "4.17"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.23.0
|
||||
version: 4.17.0
|
||||
|
||||
@@ -49,35 +49,6 @@ CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
#### RocksDB variant
|
||||
|
||||
The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
|
||||
To use this image with the Helm chart, override the image on all three components and disable
|
||||
the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
|
||||
override the image's built-in RocksDB configuration:
|
||||
|
||||
```yaml
|
||||
# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
|
||||
master:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
volume:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
filer:
|
||||
enablePVC: true
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
extraEnvironmentVars:
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
* `master` and `volume` use the same image tag so that all components share a consistent
|
||||
SeaweedFS build; RocksDB itself is only used by the filer.
|
||||
* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
|
||||
so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
|
||||
@@ -1168,108 +1168,6 @@
|
||||
"title": "Filer QPS",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"id": 91,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true,
|
||||
"width": 250
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"exemplar": true,
|
||||
"expr": "sum by (code) (rate(SeaweedFS_upload_error_total{namespace=\"$NAMESPACE\"}[$__rate_interval]))",
|
||||
"format": "time_series",
|
||||
"interval": "",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{code}}",
|
||||
"refId": "A",
|
||||
"step": 30
|
||||
}
|
||||
],
|
||||
"title": "Upload Errors",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"datasource": {
|
||||
@@ -1280,7 +1178,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 35
|
||||
"y": 28
|
||||
},
|
||||
"id": 61,
|
||||
"panels": [],
|
||||
@@ -1353,7 +1251,7 @@
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 36
|
||||
"y": 29
|
||||
},
|
||||
"id": 65,
|
||||
"links": [],
|
||||
@@ -1459,7 +1357,7 @@
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 36
|
||||
"y": 29
|
||||
},
|
||||
"id": 56,
|
||||
"links": [],
|
||||
@@ -1565,7 +1463,7 @@
|
||||
"h": 7,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 36
|
||||
"y": 29
|
||||
},
|
||||
"id": 58,
|
||||
"links": [],
|
||||
@@ -1622,7 +1520,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 43
|
||||
"y": 36
|
||||
},
|
||||
"id": 84,
|
||||
"links": [],
|
||||
@@ -1667,7 +1565,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 43
|
||||
"y": 36
|
||||
},
|
||||
"id": 85,
|
||||
"links": [],
|
||||
@@ -1763,7 +1661,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
"y": 41
|
||||
},
|
||||
"id": 86,
|
||||
"options": {
|
||||
@@ -1858,7 +1756,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
"y": 41
|
||||
},
|
||||
"id": 72,
|
||||
"links": [],
|
||||
@@ -1976,7 +1874,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 57
|
||||
"y": 50
|
||||
},
|
||||
"id": 73,
|
||||
"links": [],
|
||||
@@ -2132,7 +2030,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 64
|
||||
"y": 57
|
||||
},
|
||||
"id": 55,
|
||||
"links": [],
|
||||
@@ -2289,7 +2187,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 71
|
||||
"y": 64
|
||||
},
|
||||
"hideTimeOverride": false,
|
||||
"id": 59,
|
||||
@@ -2361,7 +2259,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 78
|
||||
"y": 71
|
||||
},
|
||||
"id": 62,
|
||||
"panels": [],
|
||||
@@ -2433,7 +2331,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 79
|
||||
"y": 72
|
||||
},
|
||||
"id": 47,
|
||||
"links": [],
|
||||
@@ -2576,7 +2474,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 79
|
||||
"y": 72
|
||||
},
|
||||
"id": 40,
|
||||
"links": [],
|
||||
@@ -2673,7 +2571,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 86
|
||||
"y": 79
|
||||
},
|
||||
"id": 48,
|
||||
"links": [],
|
||||
@@ -2783,7 +2681,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 93
|
||||
"y": 86
|
||||
},
|
||||
"id": 50,
|
||||
"links": [],
|
||||
@@ -2885,7 +2783,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 100
|
||||
"y": 93
|
||||
},
|
||||
"id": 51,
|
||||
"links": [],
|
||||
@@ -2925,7 +2823,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 100
|
||||
"y": 93
|
||||
},
|
||||
"id": 63,
|
||||
"panels": [],
|
||||
@@ -2998,7 +2896,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 108
|
||||
"y": 101
|
||||
},
|
||||
"id": 12,
|
||||
"links": [],
|
||||
@@ -3093,7 +2991,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 108
|
||||
"y": 101
|
||||
},
|
||||
"id": 14,
|
||||
"links": [],
|
||||
@@ -3135,7 +3033,7 @@
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 115
|
||||
"y": 108
|
||||
},
|
||||
"id": 64,
|
||||
"panels": [],
|
||||
@@ -3208,7 +3106,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 116
|
||||
"y": 109
|
||||
},
|
||||
"id": 52,
|
||||
"links": [],
|
||||
@@ -3336,7 +3234,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 116
|
||||
"y": 109
|
||||
},
|
||||
"id": 54,
|
||||
"links": [],
|
||||
@@ -3433,7 +3331,7 @@
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 123
|
||||
"y": 116
|
||||
},
|
||||
"id": 53,
|
||||
"links": [],
|
||||
@@ -3528,7 +3426,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 55
|
||||
"y": 48
|
||||
},
|
||||
"id": 89,
|
||||
"options": {
|
||||
@@ -3635,7 +3533,7 @@
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 55
|
||||
"y": 48
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
@@ -3756,4 +3654,4 @@
|
||||
"uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
@@ -82,7 +82,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.admin.image" . }}
|
||||
image: {{ template "admin.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
{{- $adminAuthEnabled := or .Values.admin.secret.existingSecret .Values.admin.secret.adminPassword }}
|
||||
{{- $urlPrefix := .Values.admin.urlPrefix }}
|
||||
@@ -242,7 +242,7 @@ spec:
|
||||
securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.admin.data.type "hostPath" }}
|
||||
@@ -303,7 +303,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.admin.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.admin.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "admin.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.admin.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
{{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
|
||||
@@ -247,8 +247,8 @@ spec:
|
||||
{{- $httpsPort := .Values.allInOne.s3.httpsPort | default .Values.s3.httpsPort }}
|
||||
{{- if $httpsPort }}
|
||||
-s3.port.https={{ $httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
|
||||
@@ -418,7 +418,7 @@ spec:
|
||||
{{- omit .Values.allInOne.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: data
|
||||
|
||||
@@ -164,7 +164,7 @@ spec:
|
||||
securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.cosi.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: socket
|
||||
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.filer.image" . }}
|
||||
image: {{ template "filer.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -200,8 +200,8 @@ spec:
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- if .Values.filer.s3.httpsPort }}
|
||||
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/seaweedfs_s3_config \
|
||||
@@ -317,7 +317,7 @@ spec:
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
@@ -413,7 +413,7 @@ spec:
|
||||
storageClassName: {{ .Values.filer.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.filer.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "filer.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -80,7 +80,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -251,7 +251,7 @@ spec:
|
||||
securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
@@ -312,7 +312,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.master.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "master.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.master.data.type "persistentVolumeClaim"}}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.s3.image" . }}
|
||||
image: {{ template "s3.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -127,8 +127,8 @@ spec:
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
-port.https={{ .Values.s3.httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.domainName }}
|
||||
-domainName={{ .Values.s3.domainName }} \
|
||||
@@ -226,7 +226,7 @@ spec:
|
||||
securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.s3.enableAuth }}
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
{{- $access_key_admin := $adminCreds.accessKey -}}
|
||||
{{- $secret_key_admin := $adminCreds.secretKey -}}
|
||||
{{- if not (and $access_key_admin $secret_key_admin) -}}
|
||||
{{- $access_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
{{- $readCreds := $creds.read | default dict -}}
|
||||
{{- $access_key_read := $readCreds.accessKey -}}
|
||||
{{- $secret_key_read := $readCreds.secretKey -}}
|
||||
{{- if not (and $access_key_read $secret_key_read) -}}
|
||||
{{- $access_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- /*
|
||||
The seaweedfs-s3 Service only gains a "metrics" port when the standalone S3
|
||||
gateway is enabled. With only the embedded filer S3 gateway, metrics live on
|
||||
the filer process and are already scraped by the filer ServiceMonitor.
|
||||
*/ -}}
|
||||
{{- if and .Values.s3.enabled .Values.s3.metricsPort .Values.global.seaweedfs.monitoring.enabled }}
|
||||
{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
|
||||
{{- if .Values.s3.metricsPort }}
|
||||
{{- if .Values.global.seaweedfs.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
@@ -34,3 +31,5 @@ spec:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.sftp.image" . }}
|
||||
image: {{ template "sftp.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -233,7 +233,7 @@ spec:
|
||||
securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.sftp.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.sftp.enableAuth }}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{{- if or .Values.sftp.enabled .Values.allInOne.enabled }}
|
||||
{{- $secretName := printf "%s-sftp-secret" (include "seaweedfs.fullname" .) }}
|
||||
{{- $admin_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" "length" 20) -}}
|
||||
{{- $read_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" "length" 20) -}}
|
||||
{{- $public_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" "length" 20) -}}
|
||||
{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" 20) -}}
|
||||
{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" 20) -}}
|
||||
{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" 20) -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
type: Opaque
|
||||
|
||||
@@ -72,77 +72,77 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper filer image */}}
|
||||
{{- define "seaweedfs.filer.image" -}}
|
||||
{{- define "filer.image" -}}
|
||||
{{- if .Values.filer.imageOverride -}}
|
||||
{{- $imageOverride := .Values.filer.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper master image */}}
|
||||
{{- define "seaweedfs.master.image" -}}
|
||||
{{- define "master.image" -}}
|
||||
{{- if .Values.master.imageOverride -}}
|
||||
{{- $imageOverride := .Values.master.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper s3 image */}}
|
||||
{{- define "seaweedfs.s3.image" -}}
|
||||
{{- define "s3.image" -}}
|
||||
{{- if .Values.s3.imageOverride -}}
|
||||
{{- $imageOverride := .Values.s3.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper sftp image */}}
|
||||
{{- define "seaweedfs.sftp.image" -}}
|
||||
{{- define "sftp.image" -}}
|
||||
{{- if .Values.sftp.imageOverride -}}
|
||||
{{- $imageOverride := .Values.sftp.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper admin image */}}
|
||||
{{- define "seaweedfs.admin.image" -}}
|
||||
{{- define "admin.image" -}}
|
||||
{{- if .Values.admin.imageOverride -}}
|
||||
{{- $imageOverride := .Values.admin.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper worker image */}}
|
||||
{{- define "seaweedfs.worker.image" -}}
|
||||
{{- define "worker.image" -}}
|
||||
{{- if .Values.worker.imageOverride -}}
|
||||
{{- $imageOverride := .Values.worker.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper volume image */}}
|
||||
{{- define "seaweedfs.volume.image" -}}
|
||||
{{- define "volume.image" -}}
|
||||
{{- if .Values.volume.imageOverride -}}
|
||||
{{- $imageOverride := .Values.volume.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Computes the container image name for all components (if they are not overridden) */}}
|
||||
{{- define "seaweedfs.image" -}}
|
||||
{{- define "common.image" -}}
|
||||
{{- $registryName := default .Values.image.registry .Values.global.imageRegistry | toString -}}
|
||||
{{- $repositoryName := default .Values.image.repository .Values.global.seaweedfs.image.repository | toString -}}
|
||||
{{- $name := .Values.global.seaweedfs.image.name | toString -}}
|
||||
@@ -160,7 +160,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Volume PVC exists */}}
|
||||
{{- define "seaweedfs.volume.pvc_exists" -}}
|
||||
{{- define "volume.pvc_exists" -}}
|
||||
{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -169,7 +169,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Filer PVC exists */}}
|
||||
{{- define "seaweedfs.filer.pvc_exists" -}}
|
||||
{{- define "filer.pvc_exists" -}}
|
||||
{{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -178,7 +178,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Master PVC exists */}}
|
||||
{{- define "seaweedfs.master.pvc_exists" -}}
|
||||
{{- define "master.pvc_exists" -}}
|
||||
{{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -187,7 +187,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Admin PVC exists */}}
|
||||
{{- define "seaweedfs.admin.pvc_exists" -}}
|
||||
{{- define "admin.pvc_exists" -}}
|
||||
{{- if or (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -196,7 +196,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any InitContainers exist for Volumes */}}
|
||||
{{- define "seaweedfs.volume.initContainers_exists" -}}
|
||||
{{- define "volume.initContainers_exists" -}}
|
||||
{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -225,10 +225,10 @@ imagePullSecrets:
|
||||
{{/*
|
||||
Renders a value that contains template perhaps with scope if the scope is present.
|
||||
Usage:
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.tplvalues.render" -}}
|
||||
{{- define "common.tplvalues.render" -}}
|
||||
{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
|
||||
{{- if contains "{{" (toJson .value) }}
|
||||
{{- if .scope }}
|
||||
@@ -245,9 +245,9 @@ Usage:
|
||||
Converts a Kubernetes quantity like "256Mi" or "2G" to a float64 in base units,
|
||||
handling both binary (Ki, Mi, Gi) and decimal (m, k, M) suffixes; numeric inputs
|
||||
Usage:
|
||||
{{ include "seaweedfs.resource-quantity" "10Gi" }}
|
||||
{{ include "common.resource-quantity" "10Gi" }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.resource-quantity" -}}
|
||||
{{- define "common.resource-quantity" -}}
|
||||
{{- $value := . -}}
|
||||
{{- $unit := 1.0 -}}
|
||||
{{- if typeIs "string" . -}}
|
||||
@@ -267,7 +267,7 @@ Usage:
|
||||
getOrGeneratePassword will check if a password exists in a secret and return it,
|
||||
or generate a new random password if it doesn't exist.
|
||||
*/}}
|
||||
{{- define "seaweedfs.getOrGeneratePassword" -}}
|
||||
{{- define "getOrGeneratePassword" -}}
|
||||
{{- $params := . -}}
|
||||
{{- $namespace := $params.namespace -}}
|
||||
{{- $secretName := $params.secretName -}}
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
{{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
|
||||
containers:
|
||||
- name: post-install-job
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
env:
|
||||
- name: WEED_CLUSTER_DEFAULT
|
||||
|
||||
@@ -35,8 +35,8 @@
|
||||
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
|
||||
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
|
||||
{{- if and $currentPVC }}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
|
||||
{{- $oldSize := include "common.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "common.resource-quantity" $desiredSize }}
|
||||
{{- if gt $newSize $oldSize }}
|
||||
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
|
||||
{{- end }}
|
||||
|
||||
@@ -71,12 +71,12 @@ spec:
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
|
||||
{{- $initContainers_exists := include "seaweedfs.volume.initContainers_exists" $ -}}
|
||||
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
|
||||
{{- if $initContainers_exists }}
|
||||
initContainers:
|
||||
{{- if $volume.idx }}
|
||||
- name: seaweedfs-vol-move-idx
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
command: [ '/bin/sh', '-c' ]
|
||||
args: [ '{{range $dir := $volume.dataDirs }}if ls /{{$dir.name}}/*.idx >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
|
||||
@@ -104,7 +104,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" $.Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_NAME
|
||||
@@ -274,7 +274,7 @@ spec:
|
||||
securityContext: {{- omit $volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if $volume.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.worker.image" . }}
|
||||
image: {{ template "worker.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -219,7 +219,7 @@ spec:
|
||||
securityContext: {{- omit .Values.worker.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.worker.data.type "hostPath" }}
|
||||
|
||||
@@ -1339,14 +1339,9 @@ worker:
|
||||
# Admin server to connect to
|
||||
adminServer: ""
|
||||
|
||||
# Worker job types - categories or comma-separated list of names/aliases
|
||||
# Categories: all, default, heavy
|
||||
# default: vacuum, volume_balance, ec_balance, admin_script
|
||||
# heavy: erasure_coding, iceberg_maintenance
|
||||
# all: every registered job type
|
||||
# Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
|
||||
# Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
|
||||
jobType: "all"
|
||||
# Worker job types - comma-separated list
|
||||
# Available: vacuum, volume_balance, erasure_coding
|
||||
jobType: "vacuum,volume_balance,erasure_coding"
|
||||
|
||||
# Maximum number of concurrent detection requests
|
||||
maxDetect: 1
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
MARP ?= npx @marp-team/marp-cli
|
||||
SRC := seaweedfs-production-setup.md
|
||||
PDF := $(SRC:.md=.pdf)
|
||||
HTML := $(SRC:.md=.html)
|
||||
PPTX := $(SRC:.md=.pptx)
|
||||
FLAGS := --allow-local-files
|
||||
|
||||
.PHONY: all pdf html pptx watch clean
|
||||
|
||||
all: pdf
|
||||
|
||||
pdf: $(PDF)
|
||||
|
||||
html: $(HTML)
|
||||
|
||||
pptx: $(PPTX)
|
||||
|
||||
$(PDF): $(SRC)
|
||||
$(MARP) $(SRC) --pdf $(FLAGS) -o $@
|
||||
|
||||
$(HTML): $(SRC)
|
||||
$(MARP) $(SRC) --html $(FLAGS) -o $@
|
||||
|
||||
$(PPTX): $(SRC)
|
||||
$(MARP) $(SRC) --pptx $(FLAGS) -o $@
|
||||
|
||||
watch:
|
||||
$(MARP) $(SRC) --watch --html $(FLAGS)
|
||||
|
||||
clean:
|
||||
rm -f $(PDF) $(HTML) $(PPTX)
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 738 KiB |
@@ -1,118 +0,0 @@
|
||||
---
|
||||
marp: true
|
||||
theme: default
|
||||
paginate: true
|
||||
---
|
||||
|
||||
# SeaweedFS Production Setup
|
||||
|
||||
A walkthrough of the recommended deployment topology
|
||||
|
||||
- Storage layer
|
||||
- File access layer
|
||||
- Backend operations
|
||||
- Erasure coding for durability
|
||||
|
||||
---
|
||||
|
||||

|
||||
|
||||
---
|
||||
|
||||
# Storage Layer
|
||||
|
||||
**3 Masters + Volume Servers**
|
||||
|
||||
- **Masters (run 3)**
|
||||
- Form a Raft quorum; tolerate 1 failure
|
||||
- Track volume locations and cluster topology
|
||||
- Lightweight; not on the data path for reads/writes after lookup
|
||||
|
||||
- **Volume Servers (scale out)**
|
||||
- Hold the actual needle data on local disks
|
||||
- Add more servers for capacity and throughput
|
||||
- Heartbeat to masters; clients talk to them directly for I/O
|
||||
|
||||
Rule of thumb: start with 3 masters, then grow volume servers with data.
|
||||
|
||||
---
|
||||
|
||||
# File Access Layer
|
||||
|
||||
**Filer + DB for metadata**
|
||||
|
||||
- Filer serves POSIX-like directory and file metadata
|
||||
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
|
||||
- Run **3 filers** for HA
|
||||
- Filers are stateless relative to each other — state lives in the DB
|
||||
|
||||
**S3 Servers**
|
||||
|
||||
- Expose the S3 API on top of the filer
|
||||
- Scale the count based on throughput requirements
|
||||
- Can be co-located with filers or run independently
|
||||
|
||||
---
|
||||
|
||||
# Backend Operations
|
||||
|
||||
**1 Admin Server + a few Workers**
|
||||
|
||||
- **Admin server (1 is enough)**
|
||||
- Coordinates background jobs: balancing, EC encoding, vacuum, replication fixes
|
||||
- **Not on the data path** — a restart does not affect object store reads/writes
|
||||
- No HA required for normal operation
|
||||
|
||||
- **Workers (a few)**
|
||||
- Execute long-running tasks issued by the admin server
|
||||
- Horizontally scalable based on maintenance workload
|
||||
|
||||
Keeps housekeeping off the hot path.
|
||||
|
||||
---
|
||||
|
||||
# Production Topology Summary
|
||||
|
||||
| Component | Count | Role |
|
||||
|----------------|----------------------|-----------------------------------|
|
||||
| Master | 3 | Raft quorum, topology, volume map |
|
||||
| Volume Server | N (scale out) | Stores data needles |
|
||||
| Filer | 3 | Metadata gateway |
|
||||
| Metadata DB | HA cluster | Persists filer metadata |
|
||||
| S3 Server | N (by throughput) | S3 API frontend |
|
||||
| Admin | 1 | Background job coordinator |
|
||||
| Worker | A few | Executes admin-scheduled tasks |
|
||||
|
||||
---
|
||||
|
||||
# Erasure Coding for Durability
|
||||
|
||||
Prefer **many volume servers** so EC shards spread across failure domains.
|
||||
|
||||
**Example: 8 volume servers, EC 5+3**
|
||||
|
||||
- Each volume becomes 5 data shards + 3 parity shards
|
||||
- Shards are placed on 8 distinct volume servers
|
||||
- **Tolerates up to 3 simultaneous volume server failures**
|
||||
- Storage overhead: 1.6x vs 3x for full replication
|
||||
|
||||
```
|
||||
Volume -> [D1][D2][D3][D4][D5] + [P1][P2][P3]
|
||||
| | | | | | | |
|
||||
VS1 VS2 VS3 VS4 VS5 VS6 VS7 VS8
|
||||
```
|
||||
|
||||
More volume servers = more EC layouts available (e.g. 10+4, 6+3).
|
||||
|
||||
---
|
||||
|
||||
# Getting Started Checklist
|
||||
|
||||
1. Provision 3 master nodes (small boxes are fine)
|
||||
2. Provision volume servers sized to your data footprint
|
||||
3. Stand up an HA metadata DB, then 3 filers pointing at it
|
||||
4. Add S3 servers sized to required throughput
|
||||
5. Run 1 admin + a few workers for background maintenance
|
||||
6. Once you have enough volume servers, enable EC (e.g. 5+3 on 8 servers)
|
||||
|
||||
Scale each layer independently as usage grows.
|
||||
@@ -84,15 +84,6 @@ service SeaweedFiler {
|
||||
// distributed lock management internal use only
|
||||
rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
|
||||
}
|
||||
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
|
||||
}
|
||||
|
||||
// Peer chunk sharing — tier 1: mount-server registry.
|
||||
// See design-weed-mount-peer-chunk-sharing.md for details.
|
||||
rpc MountRegister (MountRegisterRequest) returns (MountRegisterResponse) {
|
||||
}
|
||||
rpc MountList (MountListRequest) returns (MountListResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
@@ -204,10 +195,6 @@ message FuseAttributes {
|
||||
bytes md5 = 14;
|
||||
uint32 rdev = 16;
|
||||
uint64 inode = 17;
|
||||
int64 ctime = 18; // unix time in seconds, inode change time
|
||||
int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
|
||||
int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
|
||||
int32 crtime_ns = 21; // nanosecond component of crtime (0-999999999)
|
||||
}
|
||||
|
||||
message CreateEntryRequest {
|
||||
@@ -305,7 +292,6 @@ message AssignVolumeRequest {
|
||||
string rack = 7;
|
||||
string data_node = 9;
|
||||
string disk_type = 8;
|
||||
uint64 expected_data_size = 10; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message AssignVolumeResponse {
|
||||
@@ -404,23 +390,11 @@ message SubscribeMetadataRequest {
|
||||
int64 until_ns = 8;
|
||||
int32 client_epoch = 9;
|
||||
repeated string directories = 10; // exact directory to watch
|
||||
bool client_supports_batching = 11; // client can unpack SubscribeMetadataResponse.events
|
||||
bool client_supports_metadata_chunks = 12; // client can read log file chunks from volume servers
|
||||
}
|
||||
message SubscribeMetadataResponse {
|
||||
string directory = 1;
|
||||
EventNotification event_notification = 2;
|
||||
int64 ts_ns = 3;
|
||||
repeated SubscribeMetadataResponse events = 4; // batch of additional events (backlog catch-up)
|
||||
repeated LogFileChunkRef log_file_refs = 5; // log file chunk refs for client direct-read
|
||||
}
|
||||
// A persisted log file that the client can read directly from volume servers.
|
||||
// The file format is: [4-byte size | protobuf LogEntry] repeated.
|
||||
// Each LogEntry.Data contains a marshaled SubscribeMetadataResponse.
|
||||
message LogFileChunkRef {
|
||||
repeated FileChunk chunks = 1; // chunk references (fids) to read from volume servers
|
||||
int64 file_ts_ns = 2; // minute-level timestamp of the log file
|
||||
string filer_id = 3; // filer signature suffix from log filename
|
||||
}
|
||||
|
||||
message TraverseBfsMetadataRequest {
|
||||
@@ -536,7 +510,6 @@ message LockResponse {
|
||||
string lock_owner = 2;
|
||||
string lock_host_moved_to = 3;
|
||||
string error = 4;
|
||||
int64 generation = 5;
|
||||
}
|
||||
message UnlockRequest {
|
||||
string name = 1;
|
||||
@@ -559,26 +532,12 @@ message Lock {
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_backup = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message TransferLocksRequest {
|
||||
repeated Lock locks = 1;
|
||||
}
|
||||
message TransferLocksResponse {
|
||||
}
|
||||
message ReplicateLockRequest {
|
||||
string name = 1;
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
string owner = 4;
|
||||
int64 generation = 5;
|
||||
bool is_unlock = 6;
|
||||
int64 seq = 7;
|
||||
}
|
||||
message ReplicateLockResponse {
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
|
||||
@@ -608,31 +567,3 @@ message StreamMutateEntryResponse {
|
||||
string error = 7; // human-readable error message when the operation failed
|
||||
int32 errno = 8; // POSIX errno (e.g. ENOENT=2, ENOTEMPTY=66) for direct FUSE status mapping
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Peer chunk sharing — mount-server registry
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
message MountRegisterRequest {
|
||||
string peer_addr = 1; // host:port where this mount serves peer chunk requests
|
||||
string rack = 2; // locality label (rack); used for peer ranking
|
||||
int32 ttl_seconds = 3; // how long the filer should keep this entry without a heartbeat
|
||||
string data_center = 4; // locality label (data center); coarser than rack
|
||||
}
|
||||
|
||||
message MountRegisterResponse {
|
||||
}
|
||||
|
||||
message MountListRequest {
|
||||
}
|
||||
|
||||
message MountListResponse {
|
||||
repeated MountInfo mounts = 1;
|
||||
}
|
||||
|
||||
message MountInfo {
|
||||
string peer_addr = 1;
|
||||
string rack = 2;
|
||||
int64 last_seen_ns = 3;
|
||||
string data_center = 4;
|
||||
}
|
||||
|
||||
Generated
-5095
File diff suppressed because it is too large
Load Diff
@@ -1,139 +0,0 @@
|
||||
[package]
|
||||
name = "weed-volume"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "SeaweedFS Volume Server — Rust implementation"
|
||||
|
||||
[lib]
|
||||
name = "seaweed_volume"
|
||||
|
||||
[[bin]]
|
||||
name = "weed-volume"
|
||||
path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
# Default: 5-byte offsets (8TB max volume size), matching production Go builds (-tags 5BytesOffset).
|
||||
# Disable with --no-default-features for 4-byte offsets (32GB max volume size).
|
||||
default = ["5bytes"]
|
||||
5bytes = []
|
||||
|
||||
[dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
tokio-io-timeout = "1"
|
||||
|
||||
# gRPC + protobuf
|
||||
tonic = { version = "0.12", features = ["tls"] }
|
||||
tonic-reflection = "0.12"
|
||||
prost = "0.13"
|
||||
prost-types = "0.13"
|
||||
|
||||
# HTTP server
|
||||
axum = { version = "0.7", features = ["multipart"] }
|
||||
http-body = "1"
|
||||
hyper = { version = "1", features = ["full"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio", "service", "server-auto", "http1", "http2"] }
|
||||
tower = "0.4"
|
||||
tower-http = { version = "0.5", features = ["cors", "trace"] }
|
||||
|
||||
# CLI
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
||||
# Metrics
|
||||
prometheus = { version = "0.13", default-features = false, features = ["process"] }
|
||||
lazy_static = "1"
|
||||
|
||||
# JWT
|
||||
jsonwebtoken = { version = "10", features = ["rust_crypto"] }
|
||||
|
||||
# TLS
|
||||
rustls = "0.23"
|
||||
tokio-rustls = "0.26"
|
||||
rustls-pemfile = "2"
|
||||
|
||||
# LevelDB (via RocksDB for better Rust support)
|
||||
# Using rusty-leveldb for pure Rust LevelDB
|
||||
rusty-leveldb = "3"
|
||||
|
||||
# Disk-backed needle map (alternative to in-memory HashMap)
|
||||
redb = "3"
|
||||
|
||||
# Reed-Solomon erasure coding
|
||||
reed-solomon-erasure = "6"
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
# Config
|
||||
toml = "0.8"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
serde_urlencoded = "0.7"
|
||||
|
||||
# CRC32 — using Castagnoli polynomial (CRC32-C), matching Go's crc32.Castagnoli
|
||||
crc32c = "0.6"
|
||||
crc32fast = "1"
|
||||
|
||||
# Memory-mapped files
|
||||
memmap2 = "0.9"
|
||||
|
||||
# UUID
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
# HTTP client (for proxying, remote fetch)
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart", "json"] }
|
||||
|
||||
# Content hashing
|
||||
md-5 = "0.10"
|
||||
base64 = "0.22"
|
||||
|
||||
# Compression
|
||||
flate2 = "1"
|
||||
|
||||
# Image processing
|
||||
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "gif", "webp"] }
|
||||
kamadak-exif = "0.5"
|
||||
|
||||
# Multipart form-data parsing
|
||||
multer = "3"
|
||||
|
||||
# MIME type guessing from file extensions
|
||||
mime_guess = "2"
|
||||
|
||||
# Misc
|
||||
bytes = "1"
|
||||
rand = "0.8"
|
||||
chrono = "0.4"
|
||||
hex = "0.4"
|
||||
parking_lot = "0.12"
|
||||
dashmap = "6"
|
||||
thiserror = "1"
|
||||
anyhow = "1"
|
||||
async-trait = "0.1"
|
||||
futures = "0.3"
|
||||
async-stream = "0.3"
|
||||
x509-parser = "0.16"
|
||||
|
||||
# Disk space checking
|
||||
sysinfo = "0.31"
|
||||
libc = "0.2"
|
||||
|
||||
# AWS S3 SDK (for remote storage backends)
|
||||
aws-config = { version = "1", features = ["behavior-version-latest"] }
|
||||
aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4a", "http-1x", "default-https-client", "rt-tokio"] }
|
||||
aws-credential-types = "1"
|
||||
aws-types = "1"
|
||||
|
||||
# pprof is Unix-only (requires libc/nix APIs not available on Windows)
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
pprof = { version = "0.15", features = ["prost-codec"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build = "0.12"
|
||||
|
||||
[patch.crates-io]
|
||||
reed-solomon-erasure = { path = "vendor/reed-solomon-erasure" }
|
||||
@@ -1,105 +0,0 @@
|
||||
# Rust Volume Server — Dev Plan
|
||||
|
||||
## Current Status (2026-03-07)
|
||||
|
||||
**HTTP tests**: 53/53 pass (100%)
|
||||
**gRPC tests**: 56/56 pass (100%) — includes TestVolumeMoveHandlesInFlightWrites with Rust multi-volume cluster
|
||||
**Rust integration tests**: 8/8 pass
|
||||
**S3 remote storage tests**: 3/3 pass
|
||||
**Total**: 117/117 (100%) + 8 Rust + 3 S3 tests
|
||||
**Rust unit tests**: 137 lib + 7 integration = 144
|
||||
|
||||
## Completed Features
|
||||
|
||||
All phases from the original plan are complete:
|
||||
|
||||
- **Phase 1** — HTTP Core: CORS, OPTIONS, unsupported methods, static assets, path routing,
|
||||
cookie validation, conditional headers, range requests, dedup 204, content-encoding,
|
||||
readDeleted, chunk manifests, multipart validation, MD5 check, file size limit,
|
||||
upload/download throttling, image resize/crop, download disposition
|
||||
- **Phase 2** — JWT/Security: signing keys from security.toml, token source precedence
|
||||
(query > header > cookie), file_id claims, leeway=0
|
||||
- **Phase 3** — gRPC: maintenance mode, error message parity, ping routing, batch delete,
|
||||
VolumeServerStatus (with real disk stats, data_center, rack), ReadVolumeFileStatus
|
||||
(with timestamps)
|
||||
- **Phase 4** — Streaming gRPC: VolumeIncrementalCopy, CopyFile, ReceiveFile, ReadAllNeedles,
|
||||
VolumeTailSender, VolumeCopy, VolumeTailReceiver, VacuumVolumeCheck
|
||||
- **Phase 5** — EC Shards: mount/unmount, delete, read, blob delete, rebuild, shards-to-volume,
|
||||
copy, info
|
||||
- **Phase 6** — Advanced gRPC: ScrubVolume, ScrubEcVolume, Query, FetchAndWriteNeedle,
|
||||
VolumeTierMoveDat (error paths)
|
||||
- **Phase 7** — Remote Storage: S3-compatible backend via aws-sdk-s3,
|
||||
FetchAndWriteNeedle reads from S3/MinIO/SeaweedFS S3 and writes locally.
|
||||
Supports all S3-compatible providers (AWS, Wasabi, Backblaze, Aliyun, etc.)
|
||||
- **Master Heartbeat** — Bidirectional streaming SendHeartbeat RPC, volume/EC registration,
|
||||
leader changes, shutdown deregistration. Tested end-to-end with Go master.
|
||||
- **Production Sprint 1** — Quick wins:
|
||||
- VolumeMarkReadonly master notification (triggers immediate heartbeat)
|
||||
- Compaction throttling (`maybe_throttle_compaction()`)
|
||||
- File size limit enforcement on upload
|
||||
- `ts` query param for custom timestamps (upload + delete)
|
||||
- TTL expiration check (was already implemented)
|
||||
- Health check heartbeat status (returns 503 if disconnected from master)
|
||||
- preStopSeconds graceful drain before shutdown
|
||||
- S3 response passthrough headers (content-encoding, expires, content-language, content-disposition)
|
||||
- .vif persistence for readonly state across restarts
|
||||
- Webp image support for resize
|
||||
- **Production Sprint 2** — Compatibility:
|
||||
- MIME type extraction from Content-Type header
|
||||
- Stats endpoints (/stats/counter, /stats/memory, /stats/disk)
|
||||
- JSON pretty print (?pretty=y) and JSONP (?callback=fn)
|
||||
- Request ID generation (UUID if x-amz-request-id missing)
|
||||
- Advanced Prometheus metrics (INFLIGHT_REQUESTS, VOLUME_FILE_COUNT gauges)
|
||||
- **Production Sprint 3** — Streaming & Multi-node:
|
||||
- Streaming reads for large files (>1MB) via http_body::Body trait with spawn_blocking
|
||||
- Meta-only needle reads (NeedleStreamInfo) to avoid loading full body for streaming
|
||||
- Multi-volume Rust cluster support (RustMultiVolumeCluster test framework)
|
||||
- TestVolumeMoveHandlesInFlightWrites now uses Rust volume servers
|
||||
- CI skip list cleaned up (all tests pass with Rust)
|
||||
|
||||
- **Production Sprint 4** — Advanced Features:
|
||||
- BatchDelete EC shard support (ecx index lookup + ecj journal deletion)
|
||||
- JPEG EXIF orientation auto-fix on upload (kamadak-exif + image crate)
|
||||
- Async batched write processing (mpsc queue, up to 128 entries per batch)
|
||||
- VolumeTierMoveDatToRemote/FromRemote (S3 multipart upload/download)
|
||||
- S3TierRegistry for managing remote storage backends
|
||||
- VolumeInfo (.vif) persistence for remote file references
|
||||
- **Production Sprint 5** — Upload Compatibility:
|
||||
- TTL query parameter extraction during upload (`ttl=3m`)
|
||||
- Auto-compression for compressible file types (text/*, .js, .css, .json, .svg, etc.)
|
||||
- Seaweed-* custom metadata headers stored as needle pairs (JSON, max 64KB)
|
||||
- Filename extraction from URL path stored in needle name field
|
||||
- Upload response includes filename
|
||||
|
||||
- **Production Sprint 6** — Storage & Networking:
|
||||
- Redb disk-backed needle maps (pure Rust, no C deps) via `NeedleMap` enum
|
||||
- Binary search for `VolumeIncrementalCopy` with `since_ns > 0`
|
||||
- Proxy/redirect read modes for non-local volumes (master lookup, HTTP proxy, 301 redirect)
|
||||
|
||||
## Remaining Work (Production Readiness)
|
||||
|
||||
No major remaining items. All phases and production sprints are complete.
|
||||
|
||||
## Test Commands
|
||||
|
||||
```bash
|
||||
# Build
|
||||
cd seaweed-volume && cargo build --release
|
||||
|
||||
# Run all Go integration tests with Rust volume server
|
||||
VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 1200s ./test/volume_server/grpc/... ./test/volume_server/http/...
|
||||
|
||||
# Run S3 remote storage tests
|
||||
VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 180s -run "TestFetchAndWriteNeedle(FromS3|S3NotFound)" ./test/volume_server/grpc/...
|
||||
|
||||
# Run specific test
|
||||
VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 60s -run "TestName" ./test/volume_server/http/...
|
||||
|
||||
# Run Rust unit tests
|
||||
cd seaweed-volume && cargo test
|
||||
|
||||
# Test heartbeat with Go master
|
||||
weed master -port=9333 &
|
||||
seaweed-volume --port 8080 --master localhost:9333 --dir /tmp/vol1 --max 7
|
||||
curl http://localhost:9333/dir/status # should show Rust volume server registered
|
||||
```
|
||||
@@ -1,288 +0,0 @@
|
||||
# Rust Volume Server — Missing Features Audit
|
||||
|
||||
Comprehensive line-by-line comparison of Go vs Rust volume server.
|
||||
Generated 2026-03-07 from 4 parallel audits covering HTTP, gRPC, storage, and infrastructure.
|
||||
|
||||
## Executive Summary
|
||||
|
||||
| Area | Total Features | Implemented | Partial | Missing |
|
||||
|------|---------------|-------------|---------|---------|
|
||||
| gRPC RPCs | 48 | 43 (90%) | 2 (4%) | 3 (6%) |
|
||||
| HTTP Handlers | 31 | 12 (39%) | 10 (32%) | 9 (29%) |
|
||||
| Storage Layer | 22 | 6 (27%) | 7 (32%) | 9 (41%) |
|
||||
| Infrastructure | 14 | 5 (36%) | 4 (29%) | 5 (36%) |
|
||||
|
||||
---
|
||||
|
||||
## Priority 1 — Critical for Production
|
||||
|
||||
### P1.1 Streaming / Meta-Only Reads
|
||||
- **Go**: `ReadNeedleMeta()`, `ReadNeedleData()`, `ReadPagedData()` — reads only metadata or pages of large files
|
||||
- **Go**: `streamWriteResponseContent()` streams needle data in chunks
|
||||
- **Go**: `AttemptMetaOnly` / `MustMetaOnly` flags in `ReadOption`
|
||||
- **Rust**: Reads entire needle into memory always
|
||||
- **Impact**: OOM on large files; 8MB file = 8MB heap per request
|
||||
- **Files**: `weed/storage/needle/needle_read.go`, `weed/server/volume_server_handlers_read.go`
|
||||
- **Effort**: Medium
|
||||
|
||||
### P1.2 Download Proxy/Redirect Fallback (ReadMode)
|
||||
- **Go**: `ReadMode` config: "local" | "proxy" | "redirect"
|
||||
- **Go**: `tryProxyToReplica()` probes replicas, `proxyReqToTargetServer()` streams response
|
||||
- **Rust**: Always returns 404 for non-local volumes
|
||||
- **Impact**: Clients must handle volume placement themselves; breaks transparent replication
|
||||
- **Files**: `weed/server/volume_server_handlers_read.go:138-250`
|
||||
- **Effort**: Medium
|
||||
|
||||
### P1.3 TLS/HTTPS Support
|
||||
- **Go**: `LoadServerTLS()`, `LoadClientTLS()`, cert/key loading from security.toml
|
||||
- **Go**: Applied to both HTTP and gRPC servers
|
||||
- **Rust**: No TLS at all — plain TCP only
|
||||
- **Impact**: Cannot deploy in secure clusters
|
||||
- **Files**: `weed/security/tls.go`, `weed/command/volume.go`
|
||||
- **Effort**: Medium (rustls + tokio-rustls already in Cargo.toml)
|
||||
|
||||
### P1.4 VolumeMarkReadonly/Writable Master Notification
|
||||
- **Go**: `notifyMasterVolumeReadonly()` updates master with readonly state
|
||||
- **Rust**: Only sets local in-memory flag
|
||||
- **Impact**: Master keeps directing writes to readonly volume
|
||||
- **Files**: `weed/server/volume_grpc_admin.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P1.5 Compaction/Maintenance Throttling
|
||||
- **Go**: `WriteThrottler` with `MaybeSlowdown()` for MB/s rate limiting
|
||||
- **Rust**: Flags parsed but no throttle implementation
|
||||
- **Impact**: Compaction/copy operations can saturate disk IO
|
||||
- **Files**: `weed/util/throttler.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P1.6 File Size Limit Enforcement
|
||||
- **Go**: `fileSizeLimitBytes` checked on upload, returns 400
|
||||
- **Rust**: No enforcement — accepts any size
|
||||
- **Impact**: Can write files larger than volume size limit
|
||||
- **Files**: `weed/server/volume_server_handlers_write.go`
|
||||
- **Effort**: Low
|
||||
|
||||
---
|
||||
|
||||
## Priority 2 — Important for Compatibility
|
||||
|
||||
### P2.1 `ts` Query Param (Custom Timestamps)
|
||||
- **Go**: Upload and delete accept `ts` query param for custom Last-Modified time
|
||||
- **Rust**: Always uses current time
|
||||
- **Impact**: Replication timestamp fidelity; sync from external sources
|
||||
- **Files**: `weed/server/volume_server_handlers_write.go`, `volume_server_handlers_admin.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P2.2 Multipart Form Upload Parsing
|
||||
- **Go**: `needle.CreateNeedleFromRequest()` parses multipart forms, extracts MIME type, custom headers/pairs
|
||||
- **Rust**: Reads raw body bytes only — no multipart form parsing for metadata
|
||||
- **Impact**: MIME type not stored; custom needle pairs not supported
|
||||
- **Files**: `weed/storage/needle/needle.go:CreateNeedleFromRequest`
|
||||
- **Effort**: Medium
|
||||
|
||||
### P2.3 JPEG Orientation Auto-Fix
|
||||
- **Go**: `images.FixJpgOrientation()` on upload when enabled
|
||||
- **Rust**: Not implemented (flag exists but unused)
|
||||
- **Impact**: Mobile uploads may display rotated
|
||||
- **Files**: `weed/images/orientation.go`
|
||||
- **Effort**: Low (exif crate)
|
||||
|
||||
### P2.4 TTL Expiration Enforcement
|
||||
- **Go**: Checks `HasTtl()` + `AppendAtNs` against current time on read path
|
||||
- **Rust**: TTL struct exists but no expiration checking
|
||||
- **Impact**: Expired needles still served
|
||||
- **Files**: `weed/storage/needle/volume_ttl.go`, `weed/storage/volume_read.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P2.5 Health Check — Master Heartbeat Status
|
||||
- **Go**: Returns 503 if not heartbeating (can't reach master)
|
||||
- **Rust**: Only checks `is_stopping` flag
|
||||
- **Impact**: Load balancers won't detect disconnected volume servers
|
||||
- **Files**: `weed/server/volume_server.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P2.6 Stats Endpoints
|
||||
- **Go**: `/stats/counter`, `/stats/memory`, `/stats/disk` (whitelist-guarded)
|
||||
- **Rust**: Not implemented
|
||||
- **Impact**: No operational visibility
|
||||
- **Files**: `weed/server/volume_server.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P2.7 Webp Image Support
|
||||
- **Go**: `.webp` included in resize-eligible extensions
|
||||
- **Rust**: Only `.png`, `.jpg`, `.jpeg`, `.gif`
|
||||
- **Impact**: Webp images can't be resized on read
|
||||
- **Files**: `weed/server/volume_server_handlers_read.go`
|
||||
- **Effort**: Low (add webp feature to image crate)
|
||||
|
||||
### P2.8 preStopSeconds Graceful Drain
|
||||
- **Go**: Stops heartbeat, waits N seconds, then shuts down servers
|
||||
- **Rust**: Immediate shutdown on signal
|
||||
- **Impact**: In-flight requests dropped; Kubernetes readiness race
|
||||
- **Files**: `weed/command/volume.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P2.9 S3 Response Passthrough Headers
|
||||
- **Go**: `response-content-encoding`, `response-expires`, `response-content-language` query params
|
||||
- **Rust**: Only handles `response-content-type`, `response-cache-control`, `dl`
|
||||
- **Impact**: S3-compatible GET requests missing some override headers
|
||||
- **Files**: `weed/server/volume_server_handlers_read.go`
|
||||
- **Effort**: Low
|
||||
|
||||
---
|
||||
|
||||
## Priority 3 — Storage Layer Gaps
|
||||
|
||||
### P3.1 LevelDB Needle Maps
|
||||
- **Go**: 5 needle map variants: memory, LevelDB, LevelDB-medium, LevelDB-large, sorted-file
|
||||
- **Rust**: Memory-only needle map
|
||||
- **Impact**: Large volumes (millions of needles) require too much RAM
|
||||
- **Files**: `weed/storage/needle_map_leveldb.go`
|
||||
- **Effort**: High (need LevelDB binding or alternative)
|
||||
|
||||
### P3.2 Async Request Processing
|
||||
- **Go**: `asyncRequestsChan` with 128-entry queue, worker goroutine for batched writes
|
||||
- **Rust**: All writes synchronous
|
||||
- **Impact**: Write throughput limited by fsync latency
|
||||
- **Files**: `weed/storage/needle/async_request.go`
|
||||
- **Effort**: Medium
|
||||
|
||||
### P3.3 Volume Scrubbing (Data Integrity)
|
||||
- **Go**: `ScrubIndex()`, `scrubVolumeData()` — full data + index verification
|
||||
- **Rust**: Stub only in gRPC (returns OK without actual scrubbing)
|
||||
- **Impact**: No way to verify data integrity
|
||||
- **Files**: `weed/storage/volume_checking.go`, `weed/storage/idx/check.go`
|
||||
- **Effort**: Medium
|
||||
|
||||
### P3.4 Volume Backup / Sync
|
||||
- **Go**: Streaming backup, binary search for last modification, index generation scanner
|
||||
- **Rust**: Not implemented
|
||||
- **Impact**: No backup/restore capability
|
||||
- **Files**: `weed/storage/volume_backup.go`
|
||||
- **Effort**: Medium
|
||||
|
||||
### P3.5 Volume Info (.vif) Persistence
|
||||
- **Go**: `.vif` files store tier/remote metadata, readonly state persists across restarts
|
||||
- **Rust**: No `.vif` support; readonly is in-memory only
|
||||
- **Impact**: Readonly state lost on restart; no tier metadata
|
||||
- **Files**: `weed/storage/volume_info/volume_info.go`
|
||||
- **Effort**: Low
|
||||
|
||||
### P3.6 Disk Location Features
|
||||
- **Go**: Directory UUID tracking, disk space monitoring, min-free-space enforcement, tag-based grouping
|
||||
- **Rust**: Basic directory only
|
||||
- **Impact**: No disk-full protection
|
||||
- **Files**: `weed/storage/disk_location.go`
|
||||
- **Effort**: Medium
|
||||
|
||||
### P3.7 Compact Map (Memory-Efficient Needle Map)
|
||||
- **Go**: `CompactMap` with overflow handling for memory optimization
|
||||
- **Rust**: Uses standard HashMap
|
||||
- **Impact**: Higher memory usage for index
|
||||
- **Files**: `weed/storage/needle_map/compact_map.go`
|
||||
- **Effort**: Medium
|
||||
|
||||
---
|
||||
|
||||
## Priority 4 — Nice to Have
|
||||
|
||||
### P4.1 gRPC: VolumeTierMoveDatToRemote / FromRemote
|
||||
- **Go**: Full streaming implementation for tiering volumes to/from S3
|
||||
- **Rust**: Stub returning error
|
||||
- **Files**: `weed/server/volume_grpc_tier_upload.go`, `volume_grpc_tier_download.go`
|
||||
- **Effort**: High
|
||||
|
||||
### P4.2 gRPC: Query (S3 Select)
|
||||
- **Go**: JSON/CSV query over needle data (S3 Select compatible)
|
||||
- **Rust**: Stub returning error
|
||||
- **Files**: `weed/server/volume_grpc_query.go`
|
||||
- **Effort**: High
|
||||
|
||||
### P4.3 FetchAndWriteNeedle — Already Implemented
|
||||
- **Note**: The gRPC audit incorrectly flagged this as missing. It was implemented in a prior session with full S3 remote storage support.
|
||||
|
||||
### P4.4 JSON Pretty Print + JSONP
|
||||
- **Go**: `?pretty` query param for indented JSON; `?callback=fn` for JSONP
|
||||
- **Rust**: Neither supported
|
||||
- **Effort**: Low
|
||||
|
||||
### P4.5 Request ID Generation
|
||||
- **Go**: Generates UUID if `x-amz-request-id` header missing, propagates to gRPC context
|
||||
- **Rust**: Only echoes existing header
|
||||
- **Effort**: Low
|
||||
|
||||
### P4.6 UI Status Page
|
||||
- **Go**: Full HTML template with volumes, disks, stats, uptime
|
||||
- **Rust**: Stub HTML
|
||||
- **Effort**: Medium
|
||||
|
||||
### P4.7 Advanced Prometheus Metrics
|
||||
- **Go**: InFlightRequestsGauge, ConcurrentUploadLimit/DownloadLimit gauges, metrics push gateway
|
||||
- **Rust**: Basic request counter and histogram only
|
||||
- **Effort**: Low
|
||||
|
||||
### P4.8 Profiling (pprof)
|
||||
- **Go**: CPU/memory profiling, /debug/pprof endpoints
|
||||
- **Rust**: Flags parsed but not wired
|
||||
- **Effort**: Medium (tokio-console or pprof-rs)
|
||||
|
||||
### P4.9 EC Distribution / Rebalancing
|
||||
- **Go**: 17 files for EC operations including placement strategies, recovery, scrubbing
|
||||
- **Rust**: 6 files with basic encoder/decoder
|
||||
- **Effort**: High
|
||||
|
||||
### P4.10 Cookie Mismatch Status Code
|
||||
- **Go**: Returns 406 Not Acceptable
|
||||
- **Rust**: Returns 400 Bad Request
|
||||
- **Effort**: Trivial
|
||||
|
||||
---
|
||||
|
||||
## Implementation Order Recommendation
|
||||
|
||||
### Sprint 1 — Quick Wins (Low effort, high impact) ✅ DONE
|
||||
1. ✅ P1.4 VolumeMarkReadonly master notification — triggers immediate heartbeat
|
||||
2. ✅ P1.5 Compaction throttling — `maybe_throttle_compaction()` method added
|
||||
3. ✅ P1.6 File size limit enforcement — checks `file_size_limit_bytes` on upload
|
||||
4. ✅ P2.1 `ts` query param — custom timestamps for upload and delete
|
||||
5. ✅ P2.4 TTL expiration check — was already implemented
|
||||
6. ✅ P2.5 Health check heartbeat status — returns 503 if not heartbeating
|
||||
7. ✅ P2.8 preStopSeconds — graceful drain delay before shutdown
|
||||
8. ✅ P2.9 S3 passthrough headers — content-encoding, expires, content-language, content-disposition
|
||||
9. ✅ P3.5 .vif persistence — readonly state persists across restarts
|
||||
10. ✅ P2.7 Webp support — added to image resize-eligible extensions
|
||||
11. ~~P4.10 Cookie 406~~ — Go actually uses 404 for HTTP cookie mismatch (406 is gRPC batch delete only)
|
||||
|
||||
### Sprint 2 — Core Read Path (Medium effort) — Partially Done
|
||||
1. P1.1 Streaming / meta-only reads — TODO (medium effort, no test coverage yet)
|
||||
2. ✅ P1.2 ReadMode proxy/redirect — was already implemented and tested
|
||||
3. ✅ P2.2 Multipart form parsing — MIME type extraction from Content-Type header
|
||||
4. P2.3 JPEG orientation fix — TODO (low effort, needs exif crate)
|
||||
5. ✅ P2.6 Stats endpoints — /stats/counter, /stats/memory, /stats/disk
|
||||
6. ✅ P2.7 Webp support — done in Sprint 1
|
||||
7. ✅ P4.4 JSON pretty print + JSONP — ?pretty=y and ?callback=fn
|
||||
8. ✅ P4.5 Request ID generation — generates UUID if x-amz-request-id missing
|
||||
9. ✅ P4.7 Advanced Prometheus metrics — INFLIGHT_REQUESTS gauge, VOLUME_FILE_COUNT gauge
|
||||
|
||||
### Sprint 3 — Infrastructure (Medium effort) — Partially Done
|
||||
1. ✅ P1.3 TLS/HTTPS — rustls + tokio-rustls for HTTP, tonic ServerTlsConfig for gRPC
|
||||
2. P3.2 Async request processing — TODO (medium effort)
|
||||
3. ✅ P3.3 Volume scrubbing — CRC checksum verification of all needles
|
||||
4. ✅ P3.6 Disk location features — MinFreeSpace enforcement, background disk monitor
|
||||
|
||||
### Sprint 4 — Storage Advanced (High effort) — Deferred
|
||||
No integration test coverage for these items. All existing tests pass.
|
||||
1. P3.1 LevelDB needle maps — needed only for volumes with millions of needles
|
||||
2. P3.4 Volume backup/sync — streaming backup, binary search
|
||||
3. P3.7 Compact map — memory optimization for needle index
|
||||
4. P4.1 VolumeTierMoveDat — full S3 tiering (currently error stub)
|
||||
5. P4.9 EC distribution — advanced EC placement/rebalancing
|
||||
|
||||
### Sprint 5 — Polish — Deferred
|
||||
No integration test coverage for these items.
|
||||
1. P4.2 Query (S3 Select) — JSON/CSV query over needle data
|
||||
2. ✅ P4.4 JSON pretty/JSONP — done in Sprint 2
|
||||
3. ✅ P4.5 Request ID generation — done in Sprint 2
|
||||
4. P4.6 UI status page — HTML template with volume/disk/stats info
|
||||
5. ✅ P4.7 Advanced metrics — done in Sprint 2
|
||||
6. P4.8 Profiling — pprof-rs or tokio-console
|
||||
@@ -1,230 +0,0 @@
|
||||
# Rust Volume Server Parity Plan
|
||||
|
||||
Generated: 2026-03-16
|
||||
|
||||
## Goal
|
||||
|
||||
Make `seaweed-volume` a drop-in replacement for the Go volume server by:
|
||||
|
||||
- comparing every Go volume-server code path against the Rust implementation,
|
||||
- recording file-level ownership and verification status,
|
||||
- closing verified behavior gaps one logic change per commit,
|
||||
- extending tests so regressions are caught by Go parity suites and Rust unit/integration tests.
|
||||
|
||||
## Ground Truth
|
||||
|
||||
Primary Go sources:
|
||||
|
||||
- `weed/server/volume_server.go`
|
||||
- `weed/server/volume_server_handlers*.go`
|
||||
- `weed/server/volume_grpc_*.go`
|
||||
- `weed/server/constants/volume.go`
|
||||
- `weed/storage/store*.go`
|
||||
- `weed/storage/disk_location*.go`
|
||||
- `weed/storage/volume*.go`
|
||||
- `weed/storage/needle/*.go`
|
||||
- `weed/storage/idx/*.go`
|
||||
- `weed/storage/needle_map*.go`
|
||||
- `weed/storage/needle_map/*.go`
|
||||
- `weed/storage/super_block/*.go`
|
||||
- `weed/storage/erasure_coding/*.go`
|
||||
|
||||
Supporting Go dependencies that affect drop-in behavior:
|
||||
|
||||
- `weed/command/volume.go`
|
||||
- `weed/security/*.go`
|
||||
- `weed/images/*.go`
|
||||
- `weed/stats/*.go`
|
||||
|
||||
Primary Rust sources:
|
||||
|
||||
- `seaweed-volume/src/main.rs`
|
||||
- `seaweed-volume/src/config.rs`
|
||||
- `seaweed-volume/src/security.rs`
|
||||
- `seaweed-volume/src/images.rs`
|
||||
- `seaweed-volume/src/server/*.rs`
|
||||
- `seaweed-volume/src/storage/*.rs`
|
||||
- `seaweed-volume/src/storage/needle/*.rs`
|
||||
- `seaweed-volume/src/storage/idx/*.rs`
|
||||
- `seaweed-volume/src/storage/erasure_coding/*.rs`
|
||||
- `seaweed-volume/src/remote_storage/*.rs`
|
||||
|
||||
## Audit Method
|
||||
|
||||
For each Go file:
|
||||
|
||||
1. Map it to the Rust file or files that should own the same behavior.
|
||||
2. Compare exported entry points, helper functions, state transitions, wire fields, and persistence side effects.
|
||||
3. Mark each file `implemented`, `partial`, `missing`, or `needs verification`.
|
||||
4. Link each behavior to an existing test or add a missing test.
|
||||
5. Only treat a gap as closed after code review plus local verification.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
The Rust server is a drop-in replacement only when all of these hold:
|
||||
|
||||
- HTTP routes, status codes, headers, and body semantics match Go.
|
||||
- gRPC RPCs match Go request validation, response fields, streaming behavior, and maintenance/read-only semantics.
|
||||
- Master heartbeat and topology metadata match Go closely enough that the Go master treats Rust and Go volume servers the same.
|
||||
- On-disk volume behavior matches Go for normal volumes, EC shards, tiering metadata, and readonly persistence.
|
||||
- Startup flags and operational endpoints that affect production deployment behave equivalently or are explicitly documented as unsupported.
|
||||
- Existing Go integration suites pass with `VOLUME_SERVER_IMPL=rust`.
|
||||
|
||||
## File Matrix
|
||||
|
||||
### HTTP server surface
|
||||
|
||||
| Go file | Rust counterpart | Status | Comparison focus |
|
||||
| --- | --- | --- | --- |
|
||||
| `weed/server/volume_server.go` | `seaweed-volume/src/main.rs`, `seaweed-volume/src/server/volume_server.rs`, `seaweed-volume/src/server/heartbeat.rs` | partial | startup wiring, routers, heartbeat, shutdown, metrics/debug listeners |
|
||||
| `weed/server/volume_server_handlers.go` | `seaweed-volume/src/server/volume_server.rs`, `seaweed-volume/src/server/handlers.rs` | needs verification | method dispatch, OPTIONS behavior, public/admin split |
|
||||
| `weed/server/volume_server_handlers_admin.go` | `seaweed-volume/src/server/handlers.rs` | implemented | `/status`, `/healthz`, stats, server headers |
|
||||
| `weed/server/volume_server_handlers_helper.go` | `seaweed-volume/src/server/handlers.rs` | needs verification | JSON encoding, request parsing, helper parity |
|
||||
| `weed/server/volume_server_handlers_read.go` | `seaweed-volume/src/server/handlers.rs` | needs verification | JWT, conditional reads, range reads, proxy/redirect, chunk manifests, image transforms |
|
||||
| `weed/server/volume_server_handlers_ui.go` | `seaweed-volume/src/server/handlers.rs`, embedded assets | partial | UI payload and HTML parity |
|
||||
| `weed/server/volume_server_handlers_write.go` | `seaweed-volume/src/server/handlers.rs`, `seaweed-volume/src/images.rs` | needs verification | multipart parsing, metadata, compression, ts, delete semantics |
|
||||
| `weed/server/constants/volume.go` | `seaweed-volume/src/server/heartbeat.rs`, config defaults | needs verification | heartbeat timing, constants parity |
|
||||
|
||||
### gRPC server surface
|
||||
|
||||
| Go file | Rust counterpart | Status | Comparison focus |
|
||||
| --- | --- | --- | --- |
|
||||
| `weed/server/volume_grpc_admin.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | readonly/writable, allocate/delete/configure/mount/unmount |
|
||||
| `weed/server/volume_grpc_batch_delete.go` | `seaweed-volume/src/server/grpc_server.rs` | implemented | batch delete, EC delete path |
|
||||
| `weed/server/volume_grpc_client_to_master.go` | `seaweed-volume/src/server/heartbeat.rs` | partial | heartbeat fields, leader changes, metrics settings from master |
|
||||
| `weed/server/volume_grpc_copy.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | full copy streams |
|
||||
| `weed/server/volume_grpc_copy_incremental.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | incremental copy binary search, timestamps |
|
||||
| `weed/server/volume_grpc_erasure_coding.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/erasure_coding/*.rs` | needs verification | shard read/write/delete/mount/unmount/rebuild |
|
||||
| `weed/server/volume_grpc_query.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | query validation and error parity |
|
||||
| `weed/server/volume_grpc_read_all.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | read-all ordering and tail semantics |
|
||||
| `weed/server/volume_grpc_read_write.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/*.rs` | needs verification | blob/meta/page reads, write blob semantics |
|
||||
| `weed/server/volume_grpc_remote.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | remote fetch/write and tier metadata |
|
||||
| `weed/server/volume_grpc_scrub.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/*.rs` | needs verification | scrub result semantics |
|
||||
| `weed/server/volume_grpc_state.go` | `seaweed-volume/src/server/grpc_server.rs` | implemented | GetState/SetState/Status |
|
||||
| `weed/server/volume_grpc_tail.go` | `seaweed-volume/src/server/grpc_server.rs` | needs verification | tail streaming and idle timeout |
|
||||
| `weed/server/volume_grpc_tier_download.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | tier download stream/error paths |
|
||||
| `weed/server/volume_grpc_tier_upload.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | tier upload stream/error paths |
|
||||
| `weed/server/volume_grpc_vacuum.go` | `seaweed-volume/src/server/grpc_server.rs`, `seaweed-volume/src/storage/*.rs` | needs verification | compact/commit/cleanup progress and readonly transitions |
|
||||
|
||||
### Storage and persistence surface
|
||||
|
||||
| Go file group | Rust counterpart | Status | Comparison focus |
|
||||
| --- | --- | --- | --- |
|
||||
| `weed/storage/store.go`, `store_state.go` | `seaweed-volume/src/storage/store.rs`, `seaweed-volume/src/server/heartbeat.rs` | partial | topology metadata, disk tags, server id, state persistence |
|
||||
| `weed/storage/store_vacuum.go` | `seaweed-volume/src/storage/store.rs`, `seaweed-volume/src/storage/volume.rs` | needs verification | vacuum sequencing |
|
||||
| `weed/storage/store_ec.go`, `store_ec_delete.go`, `store_ec_scrub.go` | `seaweed-volume/src/storage/store.rs`, `seaweed-volume/src/storage/erasure_coding/*.rs` | needs verification | EC lifecycle and scrub behavior |
|
||||
| `weed/storage/disk_location.go`, `disk_location_ec.go` | `seaweed-volume/src/storage/disk_location.rs`, `seaweed-volume/src/storage/store.rs` | partial | directory UUIDs, tags, load rules, disk space checks |
|
||||
| `weed/storage/volume.go`, `volume_loading.go` | `seaweed-volume/src/storage/volume.rs` | needs verification | load/reload/readonly/remote metadata |
|
||||
| `weed/storage/volume_super_block.go` | `seaweed-volume/src/storage/super_block.rs`, `seaweed-volume/src/storage/volume.rs` | implemented | super block parity |
|
||||
| `weed/storage/volume_read.go`, `volume_read_all.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/server/handlers.rs` | needs verification | full/meta/page reads, TTL, streaming |
|
||||
| `weed/storage/volume_write.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/server/write_queue.rs` | needs verification | dedup, sync/async writes, metadata flags |
|
||||
| `weed/storage/volume_vacuum.go` | `seaweed-volume/src/storage/volume.rs` | needs verification | compact and commit parity |
|
||||
| `weed/storage/volume_backup.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/server/grpc_server.rs` | needs verification | backup/search logic |
|
||||
| `weed/storage/volume_checking.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/storage/idx/mod.rs`, `seaweed-volume/src/server/grpc_server.rs` | needs verification | scrub and integrity checks |
|
||||
| `weed/storage/volume_info.go`, `volume_info/volume_info.go`, `volume_tier.go` | `seaweed-volume/src/storage/volume.rs`, `seaweed-volume/src/remote_storage/*.rs` | needs verification | `.vif` format and tiered file metadata |
|
||||
| `weed/storage/needle/*.go` | `seaweed-volume/src/storage/needle/*.rs` | needs verification | needle parsing, CRC, TTL, multipart metadata |
|
||||
| `weed/storage/idx/*.go` | `seaweed-volume/src/storage/idx/*.rs` | needs verification | index walking and binary search |
|
||||
| `weed/storage/needle_map*.go`, `needle_map/*.go` | `seaweed-volume/src/storage/needle_map.rs` | needs verification | map kind parity, persistence, memory behavior |
|
||||
| `weed/storage/super_block/*.go` | `seaweed-volume/src/storage/super_block.rs` | implemented | replica placement and TTL metadata |
|
||||
| `weed/storage/erasure_coding/*.go` | `seaweed-volume/src/storage/erasure_coding/*.rs` | needs verification | EC shard placement, encode/decode, journal deletes |
|
||||
|
||||
### Supporting runtime surface
|
||||
|
||||
| Go file | Rust counterpart | Status | Comparison focus |
|
||||
| --- | --- | --- | --- |
|
||||
| `weed/command/volume.go` | `seaweed-volume/src/config.rs`, `seaweed-volume/src/main.rs` | partial | flags, metrics/debug listeners, startup behavior |
|
||||
| `weed/security/*.go` | `seaweed-volume/src/security.rs`, `seaweed-volume/src/main.rs` | implemented | JWT and TLS loading |
|
||||
| `weed/images/*.go` | `seaweed-volume/src/images.rs`, `seaweed-volume/src/server/handlers.rs` | implemented | JPEG orientation and transforms |
|
||||
| `weed/stats/*.go` | `seaweed-volume/src/metrics.rs`, `seaweed-volume/src/server/handlers.rs` | partial | metrics endpoints, push-gateway integration |
|
||||
|
||||
## Verified Gaps As Of 2026-03-08
|
||||
|
||||
The startup/runtime gaps that were verified in the initial audit are now closed:
|
||||
|
||||
1. Heartbeat metadata parity
|
||||
Closed by `8ade1c51d` and retained in current HEAD.
|
||||
|
||||
2. Dedicated metrics/debug listener parity
|
||||
Closed by `fbe0e5829`.
|
||||
|
||||
3. Master-provided metrics push settings
|
||||
Closed by `fbe0e5829`.
|
||||
|
||||
4. Slow-read tuning parity
|
||||
Closed by `66e3900dc`.
|
||||
|
||||
There are no remaining verified gaps from the initial startup/runtime audit. The broader line-by-line comparison batches below are still required to either confirm parity or surface new gaps.
|
||||
|
||||
## Execution Status As Of 2026-03-16
|
||||
|
||||
The file-by-file comparison and verification work executed in this round was:
|
||||
|
||||
1. Startup and harness alignment
|
||||
Compared `weed/command/volume.go`, `test/volume_server/framework/cluster*.go`, `seaweed-volume/src/config.rs`, and `seaweed-volume/src/main.rs` to ensure the Rust server is invoked with Go-compatible flags and is rebuilt from the current source during parity runs.
|
||||
|
||||
2. HTTP admin surface
|
||||
Compared `weed/server/volume_server_handlers_admin.go` against `seaweed-volume/src/server/handlers.rs` with emphasis on `/status` payload shape, disk-status fields, and volume ordering.
|
||||
|
||||
3. gRPC admin surface
|
||||
Compared `weed/server/volume_grpc_admin.go` against `seaweed-volume/src/server/grpc_server.rs` with emphasis on `Ping`, `VolumeConfigure`, readonly/writable flows, and error wrapping.
|
||||
|
||||
4. Storage/index layout
|
||||
Compared Go index-entry defaults in `weed/storage/types` and `weed/storage/idx/*.go` against the Rust default feature set in `seaweed-volume/Cargo.toml` and the Rust index reader/writer paths to confirm default binaries use the same offset width.
|
||||
|
||||
5. End-to-end parity verification
|
||||
Re-ran the Go HTTP and gRPC integration suites with `VOLUME_SERVER_IMPL=rust` after each fix to confirm wire-level compatibility.
|
||||
|
||||
### Verified mismatches closed in this round
|
||||
|
||||
- Rust parity runs could reuse a stale `weed-volume` binary across test invocations, hiding source and feature changes from the Go harness.
|
||||
- Rust defaulted to 5-byte index offsets, while the default Go `go build` path uses 4-byte offsets unless built with `-tags 5BytesOffset`.
|
||||
- Rust `/status` omitted Go fields in both `Volumes` and `DiskStatuses`, and did not sort volumes by `Id`.
|
||||
- Rust `Ping` treated an empty target as a self-ping and only performed a raw gRPC connect for filer targets; Go returns `remote_time_ns=0` for the empty request and performs a real filer `Ping` RPC.
|
||||
- Rust `VolumeNeedleStatus` dropped stored TTL metadata and reported `data_size` instead of Go’s `Size` field.
|
||||
- Rust multipart uploads ignored form fields such as `ts`, `ttl`, and `cm`, and also ignored part-level `Content-Encoding` and `Content-MD5`.
|
||||
- Rust only treated `dl=true` and `dl=1` as truthy, while Go accepts the full `strconv.ParseBool` set such as `dl=t` and `dl=True`.
|
||||
|
||||
### Verification commands
|
||||
|
||||
- `VOLUME_SERVER_IMPL=rust go test -count=1 -timeout 1200s ./test/volume_server/http/...`
|
||||
- `VOLUME_SERVER_IMPL=rust go test -count=1 -timeout 1200s ./test/volume_server/grpc/...`
|
||||
|
||||
## Execution Plan
|
||||
|
||||
### Batch 1: startup and heartbeat
|
||||
|
||||
- Compare `weed/command/volume.go`, `weed/server/volume_server.go`, `weed/server/volume_grpc_client_to_master.go`, `weed/storage/store.go`, and `weed/storage/disk_location.go`.
|
||||
- Close metadata and startup parity gaps that affect master registration and deployment compatibility.
|
||||
- Add Rust unit tests for heartbeat payloads and config wiring.
|
||||
|
||||
### Batch 2: HTTP read path
|
||||
|
||||
- Compare `volume_server_handlers_read.go`, `volume_server_handlers_helper.go`, and related storage read functions line by line.
|
||||
- Verify JWT, path parsing, proxy/redirect, ranges, streaming, chunk manifests, image transforms, and response-header overrides.
|
||||
- Extend `test/volume_server/http/...` and Rust handler tests where parity is not covered.
|
||||
|
||||
### Batch 3: HTTP write/delete path
|
||||
|
||||
- Compare `volume_server_handlers_write.go` and write-related storage functions.
|
||||
- Verify multipart behavior, metadata, md5, compression, unchanged writes, delete edge cases, and timestamp handling.
|
||||
|
||||
### Batch 4: gRPC admin and lifecycle
|
||||
|
||||
- Compare `volume_grpc_admin.go`, `volume_grpc_state.go`, and `volume_grpc_vacuum.go`.
|
||||
- Verify readonly/writable flows, maintenance mode, status payloads, mount/unmount/delete/configure, and vacuum transitions.
|
||||
|
||||
### Batch 5: gRPC data movement
|
||||
|
||||
- Compare `volume_grpc_read_write.go`, `copy*.go`, `read_all.go`, `tail.go`, `remote.go`, and `query.go`.
|
||||
- Verify stream framing, binary search, idle timeout, and remote-storage semantics.
|
||||
|
||||
### Batch 6: storage internals
|
||||
|
||||
- Compare all `weed/storage` volume, needle, idx, needle map, and EC files line by line.
|
||||
- Focus on persistence rules, readonly semantics, TTL, recovery/scrub, backup, and memory/disk map behavior.
|
||||
|
||||
## Commit Strategy
|
||||
|
||||
- One commit for the audit/plan document if the document itself changes.
|
||||
- One commit per logic fix.
|
||||
- Every logic commit must include the smallest test addition that proves the new parity claim.
|
||||
@@ -1,140 +0,0 @@
|
||||
# SeaweedFS Volume Server (Rust)
|
||||
|
||||
A drop-in replacement for the [SeaweedFS](https://github.com/seaweedfs/seaweedfs) Go volume server, rewritten in Rust. It uses binary-compatible storage formats (`.dat`, `.idx`, `.vif`) and speaks the same HTTP and gRPC protocols, so it works with an unmodified Go master server.
|
||||
|
||||
## Building
|
||||
|
||||
Requires Rust 1.75+ (2021 edition).
|
||||
|
||||
```bash
|
||||
cd seaweed-volume
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
The binary is produced at `target/release/seaweed-volume`.
|
||||
|
||||
## Running
|
||||
|
||||
Start a Go master server first, then point the Rust volume server at it:
|
||||
|
||||
```bash
|
||||
# Minimal
|
||||
seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7
|
||||
|
||||
# Multiple data directories
|
||||
seaweed-volume --port 8080 --master localhost:9333 \
|
||||
--dir /mnt/ssd1,/mnt/ssd2 --max 100,100 --disk ssd
|
||||
|
||||
# With datacenter/rack topology
|
||||
seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
|
||||
--dataCenter dc1 --rack rack1
|
||||
|
||||
# With JWT authentication
|
||||
seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
|
||||
--securityFile /etc/seaweedfs/security.toml
|
||||
|
||||
# With TLS (configured in security.toml via [https.volume] and [grpc.volume] sections)
|
||||
seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
|
||||
--securityFile /etc/seaweedfs/security.toml
|
||||
```
|
||||
|
||||
### Common flags
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--port` | `8080` | HTTP listen port |
|
||||
| `--port.grpc` | `port+10000` | gRPC listen port |
|
||||
| `--master` | `localhost:9333` | Comma-separated master server addresses |
|
||||
| `--dir` | `/tmp` | Comma-separated data directories |
|
||||
| `--max` | `8` | Max volumes per directory (comma-separated) |
|
||||
| `--ip` | auto-detect | Server IP / identifier |
|
||||
| `--ip.bind` | same as `--ip` | Bind address |
|
||||
| `--dataCenter` | | Datacenter name |
|
||||
| `--rack` | | Rack name |
|
||||
| `--disk` | | Disk type tag: `hdd`, `ssd`, or custom |
|
||||
| `--index` | `memory` | Needle map type: `memory`, `leveldb`, `leveldbMedium`, `leveldbLarge` |
|
||||
| `--readMode` | `proxy` | Non-local read mode: `local`, `proxy`, `redirect` |
|
||||
| `--fileSizeLimitMB` | `256` | Max upload file size |
|
||||
| `--minFreeSpace` | `1` (percent) | Min free disk space before marking volumes read-only |
|
||||
| `--securityFile` | | Path to `security.toml` for JWT keys and TLS certs |
|
||||
| `--metricsPort` | `0` (disabled) | Prometheus metrics endpoint port |
|
||||
| `--whiteList` | | Comma-separated IPs with write permission |
|
||||
| `--preStopSeconds` | `10` | Graceful drain period before shutdown |
|
||||
| `--compactionMBps` | `0` (unlimited) | Compaction I/O rate limit |
|
||||
| `--pprof` | `false` | Enable pprof HTTP handlers |
|
||||
|
||||
Set `RUST_LOG=debug` (or `trace`, `info`, `warn`) for log level control.
|
||||
Set `SEAWEED_WRITE_QUEUE=1` to enable batched async write processing.
|
||||
|
||||
## Features
|
||||
|
||||
- **Binary compatible** -- reads and writes the same `.dat`/`.idx`/`.vif` files as the Go server; seamless migration with no data conversion.
|
||||
- **HTTP + gRPC** -- full implementation of the volume server HTTP API and all gRPC RPCs including streaming operations (copy, tail, incremental copy, vacuum).
|
||||
- **Master heartbeat** -- bidirectional streaming heartbeat with the Go master server; volume and EC shard registration, leader failover, graceful shutdown deregistration.
|
||||
- **JWT authentication** -- signing key configuration via `security.toml` with token source precedence (query > header > cookie), file_id claims validation, and separate read/write keys.
|
||||
- **TLS** -- HTTPS for the HTTP API and mTLS for gRPC, configured through `security.toml`.
|
||||
- **Erasure coding** -- Reed-Solomon EC shard management: mount/unmount, read, rebuild, copy, delete, and shard-to-volume reconstruction.
|
||||
- **S3 remote storage** -- `FetchAndWriteNeedle` reads from any S3-compatible backend (AWS, MinIO, Wasabi, Backblaze, etc.) and writes locally. Supports `VolumeTierMoveDatToRemote`/`FromRemote` for tiered storage.
|
||||
- **Needle map backends** -- in-memory HashMap, LevelDB (via `rusty-leveldb`), or redb (pure Rust disk-backed) needle maps.
|
||||
- **Image processing** -- on-the-fly resize/crop, JPEG EXIF orientation auto-fix, WebP support.
|
||||
- **Streaming reads** -- large files (>1MB) are streamed via `spawn_blocking` to avoid blocking the async runtime.
|
||||
- **Auto-compression** -- compressible file types (text, JSON, CSS, JS, SVG, etc.) are gzip-compressed on upload.
|
||||
- **Prometheus metrics** -- counters, histograms, and gauges exported at a dedicated metrics port; optional push gateway support.
|
||||
- **Graceful shutdown** -- SIGINT/SIGTERM handling with configurable `preStopSeconds` drain period.
|
||||
|
||||
## Testing
|
||||
|
||||
### Rust unit tests
|
||||
|
||||
```bash
|
||||
cd seaweed-volume
|
||||
cargo test
|
||||
```
|
||||
|
||||
### Go integration tests
|
||||
|
||||
The Go test suite can target either the Go or Rust volume server via the `VOLUME_SERVER_IMPL` environment variable:
|
||||
|
||||
```bash
|
||||
# Run all HTTP + gRPC integration tests against the Rust server
|
||||
VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 1200s \
|
||||
./test/volume_server/grpc/... ./test/volume_server/http/...
|
||||
|
||||
# Run a single test
|
||||
VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 60s \
|
||||
-run "TestName" ./test/volume_server/http/...
|
||||
|
||||
# Run S3 remote storage tests
|
||||
VOLUME_SERVER_IMPL=rust go test -v -count=1 -timeout 180s \
|
||||
-run "TestFetchAndWriteNeedle" ./test/volume_server/grpc/...
|
||||
```
|
||||
|
||||
## Load testing
|
||||
|
||||
A load test harness is available at `test/volume_server/loadtest/`. See that directory for usage instructions and scenarios.
|
||||
|
||||
## Architecture
|
||||
|
||||
The server runs three listeners concurrently:
|
||||
|
||||
- **HTTP** (Axum 0.7) -- admin and public routers for file upload/download, status, and stats endpoints.
|
||||
- **gRPC** (Tonic 0.12) -- all `VolumeServer` RPCs from the SeaweedFS protobuf definition.
|
||||
- **Metrics** (optional) -- Prometheus scrape endpoint on a separate port.
|
||||
|
||||
Key source modules:
|
||||
|
||||
| Path | Description |
|
||||
|------|-------------|
|
||||
| `src/main.rs` | Entry point, server startup, signal handling |
|
||||
| `src/config.rs` | CLI parsing and configuration resolution |
|
||||
| `src/server/volume_server.rs` | HTTP router setup and middleware |
|
||||
| `src/server/handlers.rs` | HTTP request handlers (read, write, delete, status) |
|
||||
| `src/server/grpc_server.rs` | gRPC service implementation |
|
||||
| `src/server/heartbeat.rs` | Master heartbeat loop |
|
||||
| `src/storage/volume.rs` | Volume read/write/delete logic |
|
||||
| `src/storage/needle.rs` | Needle (file entry) serialization |
|
||||
| `src/storage/store.rs` | Multi-volume store management |
|
||||
| `src/security.rs` | JWT validation and IP whitelist guard |
|
||||
| `src/remote_storage/` | S3 remote storage backend |
|
||||
|
||||
See [DEV_PLAN.md](DEV_PLAN.md) for the full development history and feature checklist.
|
||||
@@ -1,17 +0,0 @@
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?);
|
||||
tonic_build::configure()
|
||||
.build_server(true)
|
||||
.build_client(true)
|
||||
.file_descriptor_set_path(out_dir.join("seaweed_descriptor.bin"))
|
||||
.compile_protos(
|
||||
&[
|
||||
"proto/volume_server.proto",
|
||||
"proto/master.proto",
|
||||
"proto/remote.proto",
|
||||
"../weed/pb/filer.proto",
|
||||
],
|
||||
&["proto/", "../weed/pb/"],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,480 +0,0 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package master_pb;
|
||||
|
||||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/master_pb";
|
||||
|
||||
import "volume_server.proto";
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
service Seaweed {
|
||||
rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) {
|
||||
}
|
||||
rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) {
|
||||
}
|
||||
rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
|
||||
}
|
||||
rpc Assign (AssignRequest) returns (AssignResponse) {
|
||||
}
|
||||
rpc StreamAssign (stream AssignRequest) returns (stream AssignResponse) {
|
||||
}
|
||||
rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
|
||||
}
|
||||
rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
|
||||
}
|
||||
rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
|
||||
}
|
||||
rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
|
||||
}
|
||||
rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
|
||||
}
|
||||
rpc VacuumVolume (VacuumVolumeRequest) returns (VacuumVolumeResponse) {
|
||||
}
|
||||
rpc DisableVacuum (DisableVacuumRequest) returns (DisableVacuumResponse) {
|
||||
}
|
||||
rpc EnableVacuum (EnableVacuumRequest) returns (EnableVacuumResponse) {
|
||||
}
|
||||
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
|
||||
}
|
||||
rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
|
||||
}
|
||||
rpc ListClusterNodes (ListClusterNodesRequest) returns (ListClusterNodesResponse) {
|
||||
}
|
||||
rpc LeaseAdminToken (LeaseAdminTokenRequest) returns (LeaseAdminTokenResponse) {
|
||||
}
|
||||
rpc ReleaseAdminToken (ReleaseAdminTokenRequest) returns (ReleaseAdminTokenResponse) {
|
||||
}
|
||||
rpc Ping (PingRequest) returns (PingResponse) {
|
||||
}
|
||||
rpc RaftListClusterServers (RaftListClusterServersRequest) returns (RaftListClusterServersResponse) {
|
||||
}
|
||||
rpc RaftAddServer (RaftAddServerRequest) returns (RaftAddServerResponse) {
|
||||
}
|
||||
rpc RaftRemoveServer (RaftRemoveServerRequest) returns (RaftRemoveServerResponse) {
|
||||
}
|
||||
rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
|
||||
}
|
||||
rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
message DiskTag {
|
||||
uint32 disk_id = 1;
|
||||
repeated string tags = 2;
|
||||
}
|
||||
|
||||
message Heartbeat {
|
||||
string ip = 1;
|
||||
uint32 port = 2;
|
||||
string public_url = 3;
|
||||
uint64 max_file_key = 5;
|
||||
string data_center = 6;
|
||||
string rack = 7;
|
||||
uint32 admin_port = 8;
|
||||
repeated VolumeInformationMessage volumes = 9;
|
||||
// delta volumes
|
||||
repeated VolumeShortInformationMessage new_volumes = 10;
|
||||
repeated VolumeShortInformationMessage deleted_volumes = 11;
|
||||
bool has_no_volumes = 12;
|
||||
|
||||
// erasure coding
|
||||
repeated VolumeEcShardInformationMessage ec_shards = 16;
|
||||
// delta erasure coding shards
|
||||
repeated VolumeEcShardInformationMessage new_ec_shards = 17;
|
||||
repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
|
||||
bool has_no_ec_shards = 19;
|
||||
|
||||
map<string, uint32> max_volume_counts = 4;
|
||||
uint32 grpc_port = 20;
|
||||
repeated string location_uuids = 21;
|
||||
string id = 22; // volume server id, independent of ip:port for stable identification
|
||||
|
||||
// state flags
|
||||
volume_server_pb.VolumeServerState state = 23;
|
||||
|
||||
repeated DiskTag disk_tags = 24;
|
||||
}
|
||||
|
||||
message HeartbeatResponse {
|
||||
uint64 volume_size_limit = 1;
|
||||
string leader = 2;
|
||||
string metrics_address = 3;
|
||||
uint32 metrics_interval_seconds = 4;
|
||||
repeated StorageBackend storage_backends = 5;
|
||||
repeated string duplicated_uuids = 6;
|
||||
bool preallocate = 7;
|
||||
}
|
||||
|
||||
message VolumeInformationMessage {
|
||||
uint32 id = 1;
|
||||
uint64 size = 2;
|
||||
string collection = 3;
|
||||
uint64 file_count = 4;
|
||||
uint64 delete_count = 5;
|
||||
uint64 deleted_byte_count = 6;
|
||||
bool read_only = 7;
|
||||
uint32 replica_placement = 8;
|
||||
uint32 version = 9;
|
||||
uint32 ttl = 10;
|
||||
uint32 compact_revision = 11;
|
||||
int64 modified_at_second = 12;
|
||||
string remote_storage_name = 13;
|
||||
string remote_storage_key = 14;
|
||||
string disk_type = 15;
|
||||
uint32 disk_id = 16;
|
||||
}
|
||||
|
||||
message VolumeShortInformationMessage {
|
||||
uint32 id = 1;
|
||||
string collection = 3;
|
||||
uint32 replica_placement = 8;
|
||||
uint32 version = 9;
|
||||
uint32 ttl = 10;
|
||||
string disk_type = 15;
|
||||
uint32 disk_id = 16;
|
||||
}
|
||||
|
||||
message VolumeEcShardInformationMessage {
|
||||
uint32 id = 1;
|
||||
string collection = 2;
|
||||
uint32 ec_index_bits = 3;
|
||||
string disk_type = 4;
|
||||
uint64 expire_at_sec = 5; // used to record the destruction time of ec volume
|
||||
uint32 disk_id = 6;
|
||||
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
|
||||
uint64 file_count = 8; // total needles in the .ecx index (live + tombstoned)
|
||||
uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
|
||||
// fields 10-19 reserved for future upstream open-source additions.
|
||||
// fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
|
||||
// and must not be used here without coordination.
|
||||
}
|
||||
|
||||
message StorageBackend {
|
||||
string type = 1;
|
||||
string id = 2;
|
||||
map<string, string> properties = 3;
|
||||
}
|
||||
|
||||
message Empty {
|
||||
}
|
||||
|
||||
message SuperBlockExtra {
|
||||
message ErasureCoding {
|
||||
uint32 data = 1;
|
||||
uint32 parity = 2;
|
||||
repeated uint32 volume_ids = 3;
|
||||
}
|
||||
ErasureCoding erasure_coding = 1;
|
||||
}
|
||||
|
||||
message KeepConnectedRequest {
|
||||
string client_type = 1;
|
||||
string client_address = 3;
|
||||
string version = 4;
|
||||
string filer_group = 5;
|
||||
string data_center = 6;
|
||||
string rack = 7;
|
||||
}
|
||||
|
||||
message VolumeLocation {
|
||||
string url = 1;
|
||||
string public_url = 2;
|
||||
repeated uint32 new_vids = 3;
|
||||
repeated uint32 deleted_vids = 4;
|
||||
string leader = 5; // optional when leader is not itself
|
||||
string data_center = 6; // optional when DataCenter is in use
|
||||
uint32 grpc_port = 7;
|
||||
repeated uint32 new_ec_vids = 8;
|
||||
repeated uint32 deleted_ec_vids = 9;
|
||||
}
|
||||
|
||||
message ClusterNodeUpdate {
|
||||
string node_type = 1;
|
||||
string address = 2;
|
||||
bool is_add = 4;
|
||||
string filer_group = 5;
|
||||
int64 created_at_ns = 6;
|
||||
}
|
||||
|
||||
message KeepConnectedResponse {
|
||||
VolumeLocation volume_location = 1;
|
||||
ClusterNodeUpdate cluster_node_update = 2;
|
||||
}
|
||||
|
||||
message LookupVolumeRequest {
|
||||
repeated string volume_or_file_ids = 1;
|
||||
string collection = 2; // optional, a bit faster if provided.
|
||||
}
|
||||
message LookupVolumeResponse {
|
||||
message VolumeIdLocation {
|
||||
string volume_or_file_id = 1;
|
||||
repeated Location locations = 2;
|
||||
string error = 3;
|
||||
string auth = 4;
|
||||
}
|
||||
repeated VolumeIdLocation volume_id_locations = 1;
|
||||
}
|
||||
|
||||
message Location {
|
||||
string url = 1;
|
||||
string public_url = 2;
|
||||
uint32 grpc_port = 3;
|
||||
string data_center = 4;
|
||||
}
|
||||
|
||||
message AssignRequest {
|
||||
uint64 count = 1;
|
||||
string replication = 2;
|
||||
string collection = 3;
|
||||
string ttl = 4;
|
||||
string data_center = 5;
|
||||
string rack = 6;
|
||||
string data_node = 7;
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
uint32 writable_volume_count = 9;
|
||||
string disk_type = 10;
|
||||
uint64 expected_data_size = 11; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message VolumeGrowRequest {
|
||||
uint32 writable_volume_count = 1;
|
||||
string replication = 2;
|
||||
string collection = 3;
|
||||
string ttl = 4;
|
||||
string data_center = 5;
|
||||
string rack = 6;
|
||||
string data_node = 7;
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
string disk_type = 9;
|
||||
}
|
||||
|
||||
message AssignResponse {
|
||||
string fid = 1;
|
||||
uint64 count = 4;
|
||||
string error = 5;
|
||||
string auth = 6;
|
||||
repeated Location replicas = 7;
|
||||
Location location = 8;
|
||||
}
|
||||
|
||||
message StatisticsRequest {
|
||||
string replication = 1;
|
||||
string collection = 2;
|
||||
string ttl = 3;
|
||||
string disk_type = 4;
|
||||
}
|
||||
message StatisticsResponse {
|
||||
uint64 total_size = 4;
|
||||
uint64 used_size = 5;
|
||||
uint64 file_count = 6;
|
||||
}
|
||||
|
||||
//
|
||||
// collection related
|
||||
//
|
||||
message Collection {
|
||||
string name = 1;
|
||||
}
|
||||
message CollectionListRequest {
|
||||
bool include_normal_volumes = 1;
|
||||
bool include_ec_volumes = 2;
|
||||
}
|
||||
message CollectionListResponse {
|
||||
repeated Collection collections = 1;
|
||||
}
|
||||
|
||||
message CollectionDeleteRequest {
|
||||
string name = 1;
|
||||
}
|
||||
message CollectionDeleteResponse {
|
||||
}
|
||||
|
||||
//
|
||||
// volume related
|
||||
//
|
||||
message DiskInfo {
|
||||
string type = 1;
|
||||
int64 volume_count = 2;
|
||||
int64 max_volume_count = 3;
|
||||
int64 free_volume_count = 4;
|
||||
int64 active_volume_count = 5;
|
||||
repeated VolumeInformationMessage volume_infos = 6;
|
||||
repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
|
||||
int64 remote_volume_count = 8;
|
||||
uint32 disk_id = 9;
|
||||
repeated string tags = 10;
|
||||
}
|
||||
message DataNodeInfo {
|
||||
string id = 1;
|
||||
map<string, DiskInfo> diskInfos = 2;
|
||||
uint32 grpc_port = 3;
|
||||
string address = 4; // ip:port for connecting to the volume server
|
||||
}
|
||||
message RackInfo {
|
||||
string id = 1;
|
||||
repeated DataNodeInfo data_node_infos = 2;
|
||||
map<string, DiskInfo> diskInfos = 3;
|
||||
}
|
||||
message DataCenterInfo {
|
||||
string id = 1;
|
||||
repeated RackInfo rack_infos = 2;
|
||||
map<string, DiskInfo> diskInfos = 3;
|
||||
}
|
||||
message TopologyInfo {
|
||||
string id = 1;
|
||||
repeated DataCenterInfo data_center_infos = 2;
|
||||
map<string, DiskInfo> diskInfos = 3;
|
||||
}
|
||||
message VolumeListRequest {
|
||||
}
|
||||
message VolumeListResponse {
|
||||
TopologyInfo topology_info = 1;
|
||||
uint64 volume_size_limit_mb = 2;
|
||||
}
|
||||
|
||||
message LookupEcVolumeRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message LookupEcVolumeResponse {
|
||||
uint32 volume_id = 1;
|
||||
message EcShardIdLocation {
|
||||
uint32 shard_id = 1;
|
||||
repeated Location locations = 2;
|
||||
}
|
||||
repeated EcShardIdLocation shard_id_locations = 2;
|
||||
}
|
||||
|
||||
message VacuumVolumeRequest {
|
||||
float garbage_threshold = 1;
|
||||
uint32 volume_id = 2;
|
||||
string collection = 3;
|
||||
}
|
||||
message VacuumVolumeResponse {
|
||||
}
|
||||
|
||||
message DisableVacuumRequest {
|
||||
}
|
||||
message DisableVacuumResponse {
|
||||
}
|
||||
|
||||
message EnableVacuumRequest {
|
||||
}
|
||||
message EnableVacuumResponse {
|
||||
}
|
||||
|
||||
message VolumeMarkReadonlyRequest {
|
||||
string ip = 1;
|
||||
uint32 port = 2;
|
||||
uint32 volume_id = 4;
|
||||
string collection = 5;
|
||||
uint32 replica_placement = 6;
|
||||
uint32 version = 7;
|
||||
uint32 ttl = 8;
|
||||
string disk_type = 9;
|
||||
bool is_readonly = 10;
|
||||
}
|
||||
message VolumeMarkReadonlyResponse {
|
||||
}
|
||||
|
||||
message GetMasterConfigurationRequest {
|
||||
}
|
||||
message GetMasterConfigurationResponse {
|
||||
string metrics_address = 1;
|
||||
uint32 metrics_interval_seconds = 2;
|
||||
repeated StorageBackend storage_backends = 3;
|
||||
string default_replication = 4;
|
||||
string leader = 5;
|
||||
uint32 volume_size_limit_m_b = 6;
|
||||
bool volume_preallocate = 7;
|
||||
// MIGRATION: fields 8-9 help migrate master.toml [master.maintenance] to admin script plugin. Remove after March 2027.
|
||||
string maintenance_scripts = 8;
|
||||
uint32 maintenance_sleep_minutes = 9;
|
||||
}
|
||||
|
||||
message ListClusterNodesRequest {
|
||||
string client_type = 1;
|
||||
string filer_group = 2;
|
||||
int32 limit = 4;
|
||||
}
|
||||
message ListClusterNodesResponse {
|
||||
message ClusterNode {
|
||||
string address = 1;
|
||||
string version = 2;
|
||||
int64 created_at_ns = 4;
|
||||
string data_center = 5;
|
||||
string rack = 6;
|
||||
}
|
||||
repeated ClusterNode cluster_nodes = 1;
|
||||
}
|
||||
|
||||
message LeaseAdminTokenRequest {
|
||||
int64 previous_token = 1;
|
||||
int64 previous_lock_time = 2;
|
||||
string lock_name = 3;
|
||||
string client_name = 4;
|
||||
string message = 5;
|
||||
}
|
||||
message LeaseAdminTokenResponse {
|
||||
int64 token = 1;
|
||||
int64 lock_ts_ns = 2;
|
||||
}
|
||||
|
||||
message ReleaseAdminTokenRequest {
|
||||
int64 previous_token = 1;
|
||||
int64 previous_lock_time = 2;
|
||||
string lock_name = 3;
|
||||
}
|
||||
message ReleaseAdminTokenResponse {
|
||||
}
|
||||
|
||||
message PingRequest {
|
||||
string target = 1; // default to ping itself
|
||||
string target_type = 2;
|
||||
}
|
||||
message PingResponse {
|
||||
int64 start_time_ns = 1;
|
||||
int64 remote_time_ns = 2;
|
||||
int64 stop_time_ns = 3;
|
||||
}
|
||||
|
||||
message RaftAddServerRequest {
|
||||
string id = 1;
|
||||
string address = 2;
|
||||
bool voter = 3;
|
||||
}
|
||||
message RaftAddServerResponse {
|
||||
}
|
||||
|
||||
message RaftRemoveServerRequest {
|
||||
string id = 1;
|
||||
bool force = 2;
|
||||
}
|
||||
message RaftRemoveServerResponse {
|
||||
}
|
||||
|
||||
message RaftListClusterServersRequest {
|
||||
}
|
||||
message RaftListClusterServersResponse {
|
||||
message ClusterServers {
|
||||
string id = 1;
|
||||
string address = 2;
|
||||
string suffrage = 3;
|
||||
bool isLeader = 4;
|
||||
}
|
||||
repeated ClusterServers cluster_servers = 1;
|
||||
}
|
||||
|
||||
message RaftLeadershipTransferRequest {
|
||||
string target_id = 1; // Optional: target server ID. If empty, transfers to any eligible follower
|
||||
string target_address = 2; // Optional: target server address. Required if target_id is specified
|
||||
}
|
||||
message RaftLeadershipTransferResponse {
|
||||
string previous_leader = 1;
|
||||
string new_leader = 2;
|
||||
}
|
||||
|
||||
message VolumeGrowResponse {
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package remote_pb;
|
||||
|
||||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/remote_pb";
|
||||
option java_package = "seaweedfs.client";
|
||||
option java_outer_classname = "FilerProto";
|
||||
|
||||
/////////////////////////
|
||||
// Remote Storage related
|
||||
/////////////////////////
|
||||
message RemoteConf {
|
||||
string type = 1;
|
||||
string name = 2;
|
||||
string s3_access_key = 4;
|
||||
string s3_secret_key = 5;
|
||||
string s3_region = 6;
|
||||
string s3_endpoint = 7;
|
||||
string s3_storage_class = 8;
|
||||
bool s3_force_path_style = 9;
|
||||
bool s3_support_tagging = 13;
|
||||
bool s3_v4_signature = 11;
|
||||
|
||||
string gcs_google_application_credentials = 10;
|
||||
string gcs_project_id = 12;
|
||||
|
||||
string azure_account_name = 15;
|
||||
string azure_account_key = 16;
|
||||
|
||||
string backblaze_key_id = 20;
|
||||
string backblaze_application_key = 21;
|
||||
string backblaze_endpoint = 22;
|
||||
string backblaze_region = 23;
|
||||
|
||||
string aliyun_access_key = 25;
|
||||
string aliyun_secret_key = 26;
|
||||
string aliyun_endpoint = 27;
|
||||
string aliyun_region = 28;
|
||||
|
||||
string tencent_secret_id = 30;
|
||||
string tencent_secret_key = 31;
|
||||
string tencent_endpoint = 32;
|
||||
|
||||
string baidu_access_key = 35;
|
||||
string baidu_secret_key = 36;
|
||||
string baidu_endpoint = 37;
|
||||
string baidu_region = 38;
|
||||
|
||||
string wasabi_access_key = 40;
|
||||
string wasabi_secret_key = 41;
|
||||
string wasabi_endpoint = 42;
|
||||
string wasabi_region = 43;
|
||||
|
||||
string filebase_access_key = 60;
|
||||
string filebase_secret_key = 61;
|
||||
string filebase_endpoint = 62;
|
||||
|
||||
string storj_access_key = 65;
|
||||
string storj_secret_key = 66;
|
||||
string storj_endpoint = 67;
|
||||
|
||||
string contabo_access_key = 68;
|
||||
string contabo_secret_key = 69;
|
||||
string contabo_endpoint = 70;
|
||||
string contabo_region = 71;
|
||||
}
|
||||
|
||||
message RemoteStorageMapping {
|
||||
map<string,RemoteStorageLocation> mappings = 1;
|
||||
string primary_bucket_storage_name = 2;
|
||||
}
|
||||
message RemoteStorageLocation {
|
||||
string name = 1;
|
||||
string bucket = 2;
|
||||
string path = 3;
|
||||
}
|
||||
@@ -1,760 +0,0 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package volume_server_pb;
|
||||
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb";
|
||||
|
||||
import "remote.proto";
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
// Persistent state for volume servers.
|
||||
message VolumeServerState {
|
||||
// whether the server is in maintenance (i.e. read-only) mode.
|
||||
bool maintenance = 1;
|
||||
// incremental version counter
|
||||
uint32 version = 2;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
service VolumeServer {
|
||||
//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
|
||||
rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
|
||||
}
|
||||
|
||||
rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
|
||||
}
|
||||
rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (stream VacuumVolumeCompactResponse) {
|
||||
}
|
||||
rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
|
||||
}
|
||||
rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
|
||||
}
|
||||
|
||||
rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
|
||||
}
|
||||
rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
|
||||
}
|
||||
|
||||
rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
|
||||
}
|
||||
rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
|
||||
}
|
||||
|
||||
rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
|
||||
}
|
||||
rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
|
||||
}
|
||||
rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
|
||||
}
|
||||
rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
|
||||
}
|
||||
rpc VolumeMarkWritable (VolumeMarkWritableRequest) returns (VolumeMarkWritableResponse) {
|
||||
}
|
||||
rpc VolumeConfigure (VolumeConfigureRequest) returns (VolumeConfigureResponse) {
|
||||
}
|
||||
rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
|
||||
}
|
||||
|
||||
rpc GetState (GetStateRequest) returns (GetStateResponse) {
|
||||
}
|
||||
rpc SetState (SetStateRequest) returns (SetStateResponse) {
|
||||
}
|
||||
|
||||
// copy the .idx .dat files, and mount this volume
|
||||
rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
|
||||
}
|
||||
rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
|
||||
}
|
||||
rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
|
||||
}
|
||||
rpc ReceiveFile (stream ReceiveFileRequest) returns (ReceiveFileResponse) {
|
||||
}
|
||||
|
||||
rpc ReadNeedleBlob (ReadNeedleBlobRequest) returns (ReadNeedleBlobResponse) {
|
||||
}
|
||||
rpc ReadNeedleMeta (ReadNeedleMetaRequest) returns (ReadNeedleMetaResponse) {
|
||||
}
|
||||
rpc WriteNeedleBlob (WriteNeedleBlobRequest) returns (WriteNeedleBlobResponse) {
|
||||
}
|
||||
rpc ReadAllNeedles (ReadAllNeedlesRequest) returns (stream ReadAllNeedlesResponse) {
|
||||
}
|
||||
|
||||
rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
|
||||
}
|
||||
rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
|
||||
}
|
||||
|
||||
// erasure coding
|
||||
rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
|
||||
}
|
||||
rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
|
||||
}
|
||||
rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
|
||||
}
|
||||
rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
|
||||
}
|
||||
rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
|
||||
}
|
||||
rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
|
||||
}
|
||||
rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
|
||||
}
|
||||
rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
|
||||
}
|
||||
rpc VolumeEcShardsToVolume (VolumeEcShardsToVolumeRequest) returns (VolumeEcShardsToVolumeResponse) {
|
||||
}
|
||||
rpc VolumeEcShardsInfo (VolumeEcShardsInfoRequest) returns (VolumeEcShardsInfoResponse) {
|
||||
}
|
||||
|
||||
// tiered storage
|
||||
rpc VolumeTierMoveDatToRemote (VolumeTierMoveDatToRemoteRequest) returns (stream VolumeTierMoveDatToRemoteResponse) {
|
||||
}
|
||||
rpc VolumeTierMoveDatFromRemote (VolumeTierMoveDatFromRemoteRequest) returns (stream VolumeTierMoveDatFromRemoteResponse) {
|
||||
}
|
||||
|
||||
rpc VolumeServerStatus (VolumeServerStatusRequest) returns (VolumeServerStatusResponse) {
|
||||
}
|
||||
rpc VolumeServerLeave (VolumeServerLeaveRequest) returns (VolumeServerLeaveResponse) {
|
||||
}
|
||||
|
||||
// remote storage
|
||||
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
|
||||
}
|
||||
|
||||
// scrubbing
|
||||
rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) {
|
||||
}
|
||||
rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) {
|
||||
}
|
||||
|
||||
// <experimental> query
|
||||
rpc Query (QueryRequest) returns (stream QueriedStripe) {
|
||||
}
|
||||
|
||||
rpc VolumeNeedleStatus (VolumeNeedleStatusRequest) returns (VolumeNeedleStatusResponse) {
|
||||
}
|
||||
|
||||
rpc Ping (PingRequest) returns (PingResponse) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
message BatchDeleteRequest {
|
||||
repeated string file_ids = 1;
|
||||
bool skip_cookie_check = 2;
|
||||
}
|
||||
|
||||
message BatchDeleteResponse {
|
||||
repeated DeleteResult results = 1;
|
||||
}
|
||||
message DeleteResult {
|
||||
string file_id = 1;
|
||||
int32 status = 2;
|
||||
string error = 3;
|
||||
uint32 size = 4;
|
||||
uint32 version = 5;
|
||||
}
|
||||
|
||||
message Empty {
|
||||
}
|
||||
|
||||
message VacuumVolumeCheckRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VacuumVolumeCheckResponse {
|
||||
double garbage_ratio = 1;
|
||||
}
|
||||
|
||||
message VacuumVolumeCompactRequest {
|
||||
uint32 volume_id = 1;
|
||||
int64 preallocate = 2;
|
||||
}
|
||||
message VacuumVolumeCompactResponse {
|
||||
int64 processed_bytes = 1;
|
||||
float load_avg_1m = 2;
|
||||
}
|
||||
|
||||
message VacuumVolumeCommitRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VacuumVolumeCommitResponse {
|
||||
bool is_read_only = 1;
|
||||
uint64 volume_size = 2;
|
||||
}
|
||||
|
||||
message VacuumVolumeCleanupRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VacuumVolumeCleanupResponse {
|
||||
}
|
||||
|
||||
message DeleteCollectionRequest {
|
||||
string collection = 1;
|
||||
}
|
||||
message DeleteCollectionResponse {
|
||||
}
|
||||
|
||||
message AllocateVolumeRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
int64 preallocate = 3;
|
||||
string replication = 4;
|
||||
string ttl = 5;
|
||||
uint32 memory_map_max_size_mb = 6;
|
||||
string disk_type = 7;
|
||||
uint32 version = 8;
|
||||
}
|
||||
message AllocateVolumeResponse {
|
||||
}
|
||||
|
||||
message VolumeSyncStatusRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VolumeSyncStatusResponse {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
string replication = 4;
|
||||
string ttl = 5;
|
||||
uint64 tail_offset = 6;
|
||||
uint32 compact_revision = 7;
|
||||
uint64 idx_file_size = 8;
|
||||
uint32 version = 9;
|
||||
}
|
||||
|
||||
message VolumeIncrementalCopyRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 since_ns = 2;
|
||||
}
|
||||
message VolumeIncrementalCopyResponse {
|
||||
bytes file_content = 1;
|
||||
}
|
||||
|
||||
message VolumeMountRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VolumeMountResponse {
|
||||
}
|
||||
|
||||
message VolumeUnmountRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VolumeUnmountResponse {
|
||||
}
|
||||
|
||||
message VolumeDeleteRequest {
|
||||
uint32 volume_id = 1;
|
||||
bool only_empty = 2;
|
||||
}
|
||||
message VolumeDeleteResponse {
|
||||
}
|
||||
|
||||
message VolumeMarkReadonlyRequest {
|
||||
uint32 volume_id = 1;
|
||||
bool persist = 2;
|
||||
}
|
||||
message VolumeMarkReadonlyResponse {
|
||||
}
|
||||
|
||||
message VolumeMarkWritableRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VolumeMarkWritableResponse {
|
||||
}
|
||||
|
||||
message VolumeConfigureRequest {
|
||||
uint32 volume_id = 1;
|
||||
string replication = 2;
|
||||
}
|
||||
message VolumeConfigureResponse {
|
||||
string error = 1;
|
||||
}
|
||||
|
||||
message VolumeStatusRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VolumeStatusResponse {
|
||||
bool is_read_only = 1;
|
||||
uint64 volume_size = 2;
|
||||
uint64 file_count = 3;
|
||||
uint64 file_deleted_count = 4;
|
||||
}
|
||||
|
||||
message GetStateRequest {
|
||||
}
|
||||
message GetStateResponse {
|
||||
VolumeServerState state = 1;
|
||||
}
|
||||
|
||||
message SetStateRequest {
|
||||
// SetState updates *all* volume server flags at once. Retrieve state with GetState(),
|
||||
// modify individual flags as required, then call this RPC to update.
|
||||
VolumeServerState state = 1;
|
||||
}
|
||||
message SetStateResponse {
|
||||
VolumeServerState state = 1;
|
||||
}
|
||||
|
||||
message VolumeCopyRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
string replication = 3;
|
||||
string ttl = 4;
|
||||
string source_data_node = 5;
|
||||
string disk_type = 6;
|
||||
int64 io_byte_per_second = 7;
|
||||
}
|
||||
message VolumeCopyResponse {
|
||||
uint64 last_append_at_ns = 1;
|
||||
int64 processed_bytes = 2;
|
||||
}
|
||||
|
||||
message CopyFileRequest {
|
||||
uint32 volume_id = 1;
|
||||
string ext = 2;
|
||||
uint32 compaction_revision = 3;
|
||||
uint64 stop_offset = 4;
|
||||
string collection = 5;
|
||||
bool is_ec_volume = 6;
|
||||
bool ignore_source_file_not_found = 7;
|
||||
}
|
||||
message CopyFileResponse {
|
||||
bytes file_content = 1;
|
||||
int64 modified_ts_ns = 2;
|
||||
}
|
||||
|
||||
message ReceiveFileRequest {
|
||||
oneof data {
|
||||
ReceiveFileInfo info = 1;
|
||||
bytes file_content = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message ReceiveFileInfo {
|
||||
uint32 volume_id = 1;
|
||||
string ext = 2;
|
||||
string collection = 3;
|
||||
bool is_ec_volume = 4;
|
||||
uint32 shard_id = 5;
|
||||
uint64 file_size = 6;
|
||||
uint32 disk_id = 7; // EC shard disk; 0 = auto-select (see VolumeEcShardsCopyRequest.disk_id)
|
||||
}
|
||||
|
||||
message ReceiveFileResponse {
|
||||
uint64 bytes_written = 1;
|
||||
string error = 2;
|
||||
}
|
||||
|
||||
message ReadNeedleBlobRequest {
|
||||
uint32 volume_id = 1;
|
||||
int64 offset = 3; // actual offset
|
||||
int32 size = 4;
|
||||
}
|
||||
message ReadNeedleBlobResponse {
|
||||
bytes needle_blob = 1;
|
||||
}
|
||||
|
||||
message ReadNeedleMetaRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
int64 offset = 3; // actual offset
|
||||
int32 size = 4;
|
||||
}
|
||||
message ReadNeedleMetaResponse {
|
||||
uint32 cookie = 1;
|
||||
uint64 last_modified = 2;
|
||||
uint32 crc = 3;
|
||||
string ttl = 4;
|
||||
uint64 append_at_ns = 5;
|
||||
}
|
||||
|
||||
message WriteNeedleBlobRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
int32 size = 3;
|
||||
bytes needle_blob = 4;
|
||||
}
|
||||
message WriteNeedleBlobResponse {
|
||||
}
|
||||
|
||||
message ReadAllNeedlesRequest {
|
||||
repeated uint32 volume_ids = 1;
|
||||
}
|
||||
message ReadAllNeedlesResponse {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
uint32 cookie = 3;
|
||||
bytes needle_blob = 5;
|
||||
bool needle_blob_compressed = 6;
|
||||
uint64 last_modified = 7;
|
||||
uint32 crc = 8;
|
||||
bytes name = 9;
|
||||
bytes mime = 10;
|
||||
}
|
||||
|
||||
message VolumeTailSenderRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 since_ns = 2;
|
||||
uint32 idle_timeout_seconds = 3;
|
||||
}
|
||||
message VolumeTailSenderResponse {
|
||||
bytes needle_header = 1;
|
||||
bytes needle_body = 2;
|
||||
bool is_last_chunk = 3;
|
||||
uint32 version = 4;
|
||||
}
|
||||
|
||||
message VolumeTailReceiverRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 since_ns = 2;
|
||||
uint32 idle_timeout_seconds = 3;
|
||||
string source_volume_server = 4;
|
||||
}
|
||||
message VolumeTailReceiverResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardsGenerateRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
}
|
||||
message VolumeEcShardsGenerateResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardsRebuildRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
}
|
||||
message VolumeEcShardsRebuildResponse {
|
||||
repeated uint32 rebuilt_shard_ids = 1;
|
||||
}
|
||||
|
||||
message VolumeEcShardsCopyRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
bool copy_ecx_file = 4;
|
||||
string source_data_node = 5;
|
||||
bool copy_ecj_file = 6;
|
||||
bool copy_vif_file = 7;
|
||||
uint32 disk_id = 8; // Target disk ID for storing EC shards
|
||||
}
|
||||
message VolumeEcShardsCopyResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardsDeleteRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
}
|
||||
message VolumeEcShardsDeleteResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardsMountRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
}
|
||||
message VolumeEcShardsMountResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardsUnmountRequest {
|
||||
uint32 volume_id = 1;
|
||||
repeated uint32 shard_ids = 3;
|
||||
}
|
||||
message VolumeEcShardsUnmountResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardReadRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint32 shard_id = 2;
|
||||
int64 offset = 3;
|
||||
int64 size = 4;
|
||||
uint64 file_key = 5;
|
||||
}
|
||||
message VolumeEcShardReadResponse {
|
||||
bytes data = 1;
|
||||
bool is_deleted = 2;
|
||||
}
|
||||
|
||||
message VolumeEcBlobDeleteRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
uint64 file_key = 3;
|
||||
uint32 version = 4;
|
||||
}
|
||||
message VolumeEcBlobDeleteResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardsToVolumeRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
}
|
||||
message VolumeEcShardsToVolumeResponse {
|
||||
}
|
||||
|
||||
message VolumeEcShardsInfoRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message VolumeEcShardsInfoResponse {
|
||||
repeated EcShardInfo ec_shard_infos = 1;
|
||||
uint64 volume_size = 2;
|
||||
uint64 file_count = 3;
|
||||
uint64 file_deleted_count = 4;
|
||||
}
|
||||
|
||||
message EcShardInfo {
|
||||
uint32 shard_id = 1;
|
||||
int64 size = 2;
|
||||
string collection = 3;
|
||||
uint32 volume_id = 4;
|
||||
}
|
||||
|
||||
message ReadVolumeFileStatusRequest {
|
||||
uint32 volume_id = 1;
|
||||
}
|
||||
message ReadVolumeFileStatusResponse {
|
||||
uint32 volume_id = 1;
|
||||
uint64 idx_file_timestamp_seconds = 2;
|
||||
uint64 idx_file_size = 3;
|
||||
uint64 dat_file_timestamp_seconds = 4;
|
||||
uint64 dat_file_size = 5;
|
||||
uint64 file_count = 6;
|
||||
uint32 compaction_revision = 7;
|
||||
string collection = 8;
|
||||
string disk_type = 9;
|
||||
VolumeInfo volume_info = 10;
|
||||
uint32 version = 11;
|
||||
}
|
||||
|
||||
message DiskStatus {
|
||||
string dir = 1;
|
||||
uint64 all = 2;
|
||||
uint64 used = 3;
|
||||
uint64 free = 4;
|
||||
float percent_free = 5;
|
||||
float percent_used = 6;
|
||||
string disk_type = 7;
|
||||
}
|
||||
|
||||
message MemStatus {
|
||||
int32 goroutines = 1;
|
||||
uint64 all = 2;
|
||||
uint64 used = 3;
|
||||
uint64 free = 4;
|
||||
uint64 self = 5;
|
||||
uint64 heap = 6;
|
||||
uint64 stack = 7;
|
||||
}
|
||||
|
||||
// tired storage on volume servers
|
||||
message RemoteFile {
|
||||
string backend_type = 1;
|
||||
string backend_id = 2;
|
||||
string key = 3;
|
||||
uint64 offset = 4;
|
||||
uint64 file_size = 5;
|
||||
uint64 modified_time = 6;
|
||||
string extension = 7;
|
||||
}
|
||||
message VolumeInfo {
|
||||
repeated RemoteFile files = 1;
|
||||
uint32 version = 2;
|
||||
string replication = 3;
|
||||
uint32 bytes_offset = 4;
|
||||
int64 dat_file_size = 5; // store the original dat file size
|
||||
uint64 expire_at_sec = 6; // expiration time of ec volume
|
||||
bool read_only = 7;
|
||||
EcShardConfig ec_shard_config = 8; // EC shard configuration (optional, null = use default 10+4)
|
||||
}
|
||||
|
||||
// EcShardConfig specifies erasure coding shard configuration
|
||||
message EcShardConfig {
|
||||
uint32 data_shards = 1; // Number of data shards (e.g., 10)
|
||||
uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
|
||||
}
|
||||
message OldVersionVolumeInfo {
|
||||
repeated RemoteFile files = 1;
|
||||
uint32 version = 2;
|
||||
string replication = 3;
|
||||
uint32 BytesOffset = 4;
|
||||
int64 dat_file_size = 5; // store the original dat file size
|
||||
uint64 DestroyTime = 6; // expiration time of ec volume
|
||||
bool read_only = 7;
|
||||
}
|
||||
|
||||
// tiered storage
|
||||
message VolumeTierMoveDatToRemoteRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
string destination_backend_name = 3;
|
||||
bool keep_local_dat_file = 4;
|
||||
}
|
||||
message VolumeTierMoveDatToRemoteResponse {
|
||||
int64 processed = 1;
|
||||
float processedPercentage = 2;
|
||||
}
|
||||
|
||||
message VolumeTierMoveDatFromRemoteRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
bool keep_remote_dat_file = 3;
|
||||
}
|
||||
message VolumeTierMoveDatFromRemoteResponse {
|
||||
int64 processed = 1;
|
||||
float processedPercentage = 2;
|
||||
}
|
||||
|
||||
message VolumeServerStatusRequest {
|
||||
|
||||
}
|
||||
message VolumeServerStatusResponse {
|
||||
repeated DiskStatus disk_statuses = 1;
|
||||
MemStatus memory_status = 2;
|
||||
string version = 3;
|
||||
string data_center = 4;
|
||||
string rack = 5;
|
||||
VolumeServerState state = 6;
|
||||
}
|
||||
|
||||
message VolumeServerLeaveRequest {
|
||||
}
|
||||
message VolumeServerLeaveResponse {
|
||||
}
|
||||
|
||||
// remote storage
|
||||
message FetchAndWriteNeedleRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
uint32 cookie = 3;
|
||||
int64 offset = 4;
|
||||
int64 size = 5;
|
||||
message Replica {
|
||||
string url = 1;
|
||||
string public_url = 2;
|
||||
int32 grpc_port = 3;
|
||||
}
|
||||
repeated Replica replicas = 6;
|
||||
string auth = 7;
|
||||
// remote conf
|
||||
remote_pb.RemoteConf remote_conf = 15;
|
||||
remote_pb.RemoteStorageLocation remote_location = 16;
|
||||
}
|
||||
message FetchAndWriteNeedleResponse {
|
||||
string e_tag = 1;
|
||||
}
|
||||
|
||||
enum VolumeScrubMode {
|
||||
UNKNOWN = 0;
|
||||
INDEX = 1;
|
||||
FULL = 2;
|
||||
LOCAL = 3;
|
||||
}
|
||||
|
||||
message ScrubVolumeRequest {
|
||||
VolumeScrubMode mode = 1;
|
||||
// optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed.
|
||||
repeated uint32 volume_ids = 2;
|
||||
bool mark_broken_volumes_readonly = 3;
|
||||
}
|
||||
message ScrubVolumeResponse {
|
||||
uint64 total_volumes = 1;
|
||||
uint64 total_files = 2;
|
||||
repeated uint32 broken_volume_ids = 3;
|
||||
repeated string details = 4;
|
||||
}
|
||||
|
||||
message ScrubEcVolumeRequest {
|
||||
VolumeScrubMode mode = 1;
|
||||
// optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed.
|
||||
repeated uint32 volume_ids = 2;
|
||||
}
|
||||
message ScrubEcVolumeResponse {
|
||||
uint64 total_volumes = 1;
|
||||
uint64 total_files = 2;
|
||||
repeated uint32 broken_volume_ids = 3;
|
||||
repeated EcShardInfo broken_shard_infos = 4;
|
||||
repeated string details = 5;
|
||||
}
|
||||
|
||||
// select on volume servers
|
||||
message QueryRequest {
|
||||
repeated string selections = 1;
|
||||
repeated string from_file_ids = 2;
|
||||
message Filter {
|
||||
string field = 1;
|
||||
string operand = 2;
|
||||
string value = 3;
|
||||
}
|
||||
Filter filter = 3;
|
||||
|
||||
message InputSerialization {
|
||||
// NONE | GZIP | BZIP2
|
||||
string compression_type = 1;
|
||||
message CSVInput {
|
||||
string file_header_info = 1; // Valid values: NONE | USE | IGNORE
|
||||
string record_delimiter = 2; // Default: \n
|
||||
string field_delimiter = 3; // Default: ,
|
||||
string quote_character = 4; // Default: "
|
||||
string quote_escape_character = 5; // Default: "
|
||||
string comments = 6; // Default: #
|
||||
// If true, records might contain record delimiters within quote characters
|
||||
bool allow_quoted_record_delimiter = 7; // default False.
|
||||
}
|
||||
message JSONInput {
|
||||
string type = 1; // Valid values: DOCUMENT | LINES
|
||||
}
|
||||
message ParquetInput {
|
||||
}
|
||||
|
||||
CSVInput csv_input = 2;
|
||||
JSONInput json_input = 3;
|
||||
ParquetInput parquet_input = 4;
|
||||
}
|
||||
InputSerialization input_serialization = 4;
|
||||
|
||||
message OutputSerialization {
|
||||
message CSVOutput {
|
||||
string quote_fields = 1; // Valid values: ALWAYS | ASNEEDED
|
||||
string record_delimiter = 2; // Default: \n
|
||||
string field_delimiter = 3; // Default: ,
|
||||
string quote_character = 4; // Default: "
|
||||
string quote_escape_character = 5; // Default: "
|
||||
}
|
||||
message JSONOutput {
|
||||
string record_delimiter = 1;
|
||||
}
|
||||
|
||||
CSVOutput csv_output = 2;
|
||||
JSONOutput json_output = 3;
|
||||
}
|
||||
|
||||
OutputSerialization output_serialization = 5;
|
||||
}
|
||||
message QueriedStripe {
|
||||
bytes records = 1;
|
||||
}
|
||||
|
||||
message VolumeNeedleStatusRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
}
|
||||
message VolumeNeedleStatusResponse {
|
||||
uint64 needle_id = 1;
|
||||
uint32 cookie = 2;
|
||||
uint32 size = 3;
|
||||
uint64 last_modified = 4;
|
||||
uint32 crc = 5;
|
||||
string ttl = 6;
|
||||
}
|
||||
|
||||
message PingRequest {
|
||||
string target = 1; // default to ping itself
|
||||
string target_type = 2;
|
||||
}
|
||||
message PingResponse {
|
||||
int64 start_time_ns = 1;
|
||||
int64 remote_time_ns = 2;
|
||||
int64 stop_time_ns = 3;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,275 +0,0 @@
|
||||
//! JPEG EXIF orientation auto-fix, matching Go's `FixJpgOrientation`.
|
||||
//!
|
||||
//! Reads the EXIF orientation tag from JPEG data and rotates/flips the image
|
||||
//! to normalize it to orientation 1 (top-left). If EXIF parsing fails or
|
||||
//! orientation is already normal, returns the original data unchanged.
|
||||
|
||||
use std::io::Cursor;
|
||||
|
||||
use image::{DynamicImage, GenericImageView, ImageFormat, RgbaImage};
|
||||
|
||||
/// EXIF orientation tag values.
|
||||
/// See: <http://sylvana.net/jpegcrop/exif_orientation.html>
|
||||
const TOP_LEFT_SIDE: u32 = 1;
|
||||
const TOP_RIGHT_SIDE: u32 = 2;
|
||||
const BOTTOM_RIGHT_SIDE: u32 = 3;
|
||||
const BOTTOM_LEFT_SIDE: u32 = 4;
|
||||
const LEFT_SIDE_TOP: u32 = 5;
|
||||
const RIGHT_SIDE_TOP: u32 = 6;
|
||||
const RIGHT_SIDE_BOTTOM: u32 = 7;
|
||||
const LEFT_SIDE_BOTTOM: u32 = 8;
|
||||
|
||||
/// Fix JPEG orientation based on EXIF data.
|
||||
///
|
||||
/// Reads the EXIF orientation tag and applies the appropriate rotation/flip
|
||||
/// to normalize the image to orientation 1 (top-left). Re-encodes as JPEG.
|
||||
///
|
||||
/// Returns the original data unchanged if:
|
||||
/// - EXIF data cannot be parsed
|
||||
/// - No orientation tag is present
|
||||
/// - Orientation is already 1 (normal)
|
||||
/// - Image decoding or re-encoding fails
|
||||
pub fn fix_jpg_orientation(data: &[u8]) -> Vec<u8> {
|
||||
// Parse EXIF data
|
||||
let orientation = match read_exif_orientation(data) {
|
||||
Some(o) => o,
|
||||
None => return data.to_vec(),
|
||||
};
|
||||
|
||||
// Orientation 1 means normal — no transformation needed
|
||||
if orientation == TOP_LEFT_SIDE {
|
||||
return data.to_vec();
|
||||
}
|
||||
|
||||
// Determine rotation angle and flip mode
|
||||
let (angle, flip_horizontal) = match orientation {
|
||||
TOP_RIGHT_SIDE => (0, true),
|
||||
BOTTOM_RIGHT_SIDE => (180, false),
|
||||
BOTTOM_LEFT_SIDE => (180, true),
|
||||
LEFT_SIDE_TOP => (-90, true),
|
||||
RIGHT_SIDE_TOP => (-90, false),
|
||||
RIGHT_SIDE_BOTTOM => (90, true),
|
||||
LEFT_SIDE_BOTTOM => (90, false),
|
||||
_ => return data.to_vec(),
|
||||
};
|
||||
|
||||
// Decode the image
|
||||
let src_image = match image::load_from_memory_with_format(data, ImageFormat::Jpeg) {
|
||||
Ok(img) => img,
|
||||
Err(_) => return data.to_vec(),
|
||||
};
|
||||
|
||||
// Apply rotation then flip (matching Go's flip(rotate(img, angle), flipMode))
|
||||
let transformed = flip_horizontal_if(rotate(src_image, angle), flip_horizontal);
|
||||
|
||||
// Re-encode as JPEG
|
||||
let mut buf = Cursor::new(Vec::new());
|
||||
match transformed.write_to(&mut buf, ImageFormat::Jpeg) {
|
||||
Ok(_) => buf.into_inner(),
|
||||
Err(_) => data.to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the EXIF orientation tag from JPEG data.
|
||||
/// Returns None if EXIF cannot be parsed or orientation tag is not present.
|
||||
fn read_exif_orientation(data: &[u8]) -> Option<u32> {
|
||||
let exif_reader = exif::Reader::new();
|
||||
let mut cursor = Cursor::new(data);
|
||||
let exif_data = exif_reader.read_from_container(&mut cursor).ok()?;
|
||||
|
||||
let orientation_field = exif_data.get_field(exif::Tag::Orientation, exif::In::PRIMARY)?;
|
||||
match orientation_field.value {
|
||||
exif::Value::Short(ref v) if !v.is_empty() => Some(v[0] as u32),
|
||||
_ => orientation_field.value.get_uint(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Rotate an image by the given angle (counter-clockwise, in degrees).
|
||||
/// Matches Go's rotate function.
|
||||
fn rotate(img: DynamicImage, angle: i32) -> DynamicImage {
|
||||
let (width, height) = img.dimensions();
|
||||
|
||||
match angle {
|
||||
90 => {
|
||||
// 90 degrees counter-clockwise
|
||||
let new_w = height;
|
||||
let new_h = width;
|
||||
let mut out = RgbaImage::new(new_w, new_h);
|
||||
for y in 0..new_h {
|
||||
for x in 0..new_w {
|
||||
out.put_pixel(x, y, img.get_pixel(new_h - 1 - y, x));
|
||||
}
|
||||
}
|
||||
DynamicImage::ImageRgba8(out)
|
||||
}
|
||||
-90 => {
|
||||
// 90 degrees clockwise (or 270 counter-clockwise)
|
||||
let new_w = height;
|
||||
let new_h = width;
|
||||
let mut out = RgbaImage::new(new_w, new_h);
|
||||
for y in 0..new_h {
|
||||
for x in 0..new_w {
|
||||
out.put_pixel(x, y, img.get_pixel(y, new_w - 1 - x));
|
||||
}
|
||||
}
|
||||
DynamicImage::ImageRgba8(out)
|
||||
}
|
||||
180 | -180 => {
|
||||
let mut out = RgbaImage::new(width, height);
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
out.put_pixel(x, y, img.get_pixel(width - 1 - x, height - 1 - y));
|
||||
}
|
||||
}
|
||||
DynamicImage::ImageRgba8(out)
|
||||
}
|
||||
_ => img,
|
||||
}
|
||||
}
|
||||
|
||||
/// Flip the image horizontally if requested.
|
||||
/// In Go, flipMode 2 == FlipHorizontal. We simplify since only horizontal flip is used.
|
||||
fn flip_horizontal_if(img: DynamicImage, do_flip: bool) -> DynamicImage {
|
||||
if !do_flip {
|
||||
return img;
|
||||
}
|
||||
let (width, height) = img.dimensions();
|
||||
let mut out = RgbaImage::new(width, height);
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
out.put_pixel(x, y, img.get_pixel(width - 1 - x, y));
|
||||
}
|
||||
}
|
||||
DynamicImage::ImageRgba8(out)
|
||||
}
|
||||
|
||||
/// Returns true if the given MIME type or file path extension indicates a JPEG file.
|
||||
pub fn is_jpeg(mime_type: &str, path: &str) -> bool {
|
||||
if mime_type == "image/jpeg" {
|
||||
return true;
|
||||
}
|
||||
let lower = path.to_lowercase();
|
||||
lower.ends_with(".jpg") || lower.ends_with(".jpeg")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_non_jpeg_data_returned_unchanged() {
|
||||
let data = b"not a jpeg file at all";
|
||||
let result = fix_jpg_orientation(data);
|
||||
assert_eq!(result, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jpeg_without_exif_returned_unchanged() {
|
||||
// Create a minimal JPEG without EXIF data
|
||||
let img = DynamicImage::ImageRgba8(RgbaImage::new(2, 2));
|
||||
let mut buf = Cursor::new(Vec::new());
|
||||
img.write_to(&mut buf, ImageFormat::Jpeg).unwrap();
|
||||
let jpeg_data = buf.into_inner();
|
||||
|
||||
let result = fix_jpg_orientation(&jpeg_data);
|
||||
// Should return data unchanged (no EXIF orientation tag)
|
||||
// Just verify it's still valid JPEG
|
||||
assert!(!result.is_empty());
|
||||
assert_eq!(&result[0..2], &[0xFF, 0xD8]); // JPEG magic bytes
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_jpeg() {
|
||||
assert!(is_jpeg("image/jpeg", ""));
|
||||
assert!(is_jpeg("", "/3,abc.jpg"));
|
||||
assert!(is_jpeg("", "/3,abc.JPEG"));
|
||||
assert!(is_jpeg("application/octet-stream", "/3,abc.JPG"));
|
||||
assert!(!is_jpeg("image/png", "/3,abc.png"));
|
||||
assert!(!is_jpeg("", "/3,abc.png"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rotate_180() {
|
||||
// Create a 2x2 image with distinct pixel colors
|
||||
let mut img = RgbaImage::new(2, 2);
|
||||
img.put_pixel(0, 0, image::Rgba([255, 0, 0, 255])); // red top-left
|
||||
img.put_pixel(1, 0, image::Rgba([0, 255, 0, 255])); // green top-right
|
||||
img.put_pixel(0, 1, image::Rgba([0, 0, 255, 255])); // blue bottom-left
|
||||
img.put_pixel(1, 1, image::Rgba([255, 255, 0, 255])); // yellow bottom-right
|
||||
let dynamic = DynamicImage::ImageRgba8(img);
|
||||
|
||||
let rotated = rotate(dynamic, 180);
|
||||
let (w, h) = rotated.dimensions();
|
||||
assert_eq!((w, h), (2, 2));
|
||||
// After 180 rotation: top-left should be yellow, top-right should be blue
|
||||
assert_eq!(rotated.get_pixel(0, 0), image::Rgba([255, 255, 0, 255]));
|
||||
assert_eq!(rotated.get_pixel(1, 0), image::Rgba([0, 0, 255, 255]));
|
||||
assert_eq!(rotated.get_pixel(0, 1), image::Rgba([0, 255, 0, 255]));
|
||||
assert_eq!(rotated.get_pixel(1, 1), image::Rgba([255, 0, 0, 255]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rotate_90_ccw() {
|
||||
// Create 3x2 image (width=3, height=2)
|
||||
let mut img = RgbaImage::new(3, 2);
|
||||
img.put_pixel(0, 0, image::Rgba([1, 0, 0, 255]));
|
||||
img.put_pixel(1, 0, image::Rgba([2, 0, 0, 255]));
|
||||
img.put_pixel(2, 0, image::Rgba([3, 0, 0, 255]));
|
||||
img.put_pixel(0, 1, image::Rgba([4, 0, 0, 255]));
|
||||
img.put_pixel(1, 1, image::Rgba([5, 0, 0, 255]));
|
||||
img.put_pixel(2, 1, image::Rgba([6, 0, 0, 255]));
|
||||
let dynamic = DynamicImage::ImageRgba8(img);
|
||||
|
||||
let rotated = rotate(dynamic, 90);
|
||||
let (w, h) = rotated.dimensions();
|
||||
// 90 CCW: width=3,height=2 -> new_w=2, new_h=3
|
||||
assert_eq!((w, h), (2, 3));
|
||||
// Top-right (2,0) should move to top-left (0,0) in CCW 90
|
||||
assert_eq!(rotated.get_pixel(0, 0)[0], 3);
|
||||
assert_eq!(rotated.get_pixel(1, 0)[0], 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rotate_neg90_cw() {
|
||||
// Create 3x2 image
|
||||
let mut img = RgbaImage::new(3, 2);
|
||||
img.put_pixel(0, 0, image::Rgba([1, 0, 0, 255]));
|
||||
img.put_pixel(1, 0, image::Rgba([2, 0, 0, 255]));
|
||||
img.put_pixel(2, 0, image::Rgba([3, 0, 0, 255]));
|
||||
img.put_pixel(0, 1, image::Rgba([4, 0, 0, 255]));
|
||||
img.put_pixel(1, 1, image::Rgba([5, 0, 0, 255]));
|
||||
img.put_pixel(2, 1, image::Rgba([6, 0, 0, 255]));
|
||||
let dynamic = DynamicImage::ImageRgba8(img);
|
||||
|
||||
let rotated = rotate(dynamic, -90);
|
||||
let (w, h) = rotated.dimensions();
|
||||
assert_eq!((w, h), (2, 3));
|
||||
// -90 (CW 90): top-left (0,0) should go to top-right
|
||||
assert_eq!(rotated.get_pixel(0, 0)[0], 4);
|
||||
assert_eq!(rotated.get_pixel(1, 0)[0], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flip_horizontal() {
|
||||
let mut img = RgbaImage::new(2, 1);
|
||||
img.put_pixel(0, 0, image::Rgba([10, 0, 0, 255]));
|
||||
img.put_pixel(1, 0, image::Rgba([20, 0, 0, 255]));
|
||||
let dynamic = DynamicImage::ImageRgba8(img);
|
||||
|
||||
let flipped = flip_horizontal_if(dynamic, true);
|
||||
assert_eq!(flipped.get_pixel(0, 0)[0], 20);
|
||||
assert_eq!(flipped.get_pixel(1, 0)[0], 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flip_horizontal_noop() {
|
||||
let mut img = RgbaImage::new(2, 1);
|
||||
img.put_pixel(0, 0, image::Rgba([10, 0, 0, 255]));
|
||||
img.put_pixel(1, 0, image::Rgba([20, 0, 0, 255]));
|
||||
let dynamic = DynamicImage::ImageRgba8(img);
|
||||
|
||||
let not_flipped = flip_horizontal_if(dynamic, false);
|
||||
assert_eq!(not_flipped.get_pixel(0, 0)[0], 10);
|
||||
assert_eq!(not_flipped.get_pixel(1, 0)[0], 20);
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
pub mod config;
|
||||
pub mod images;
|
||||
pub mod metrics;
|
||||
pub mod remote_storage;
|
||||
pub mod security;
|
||||
pub mod server;
|
||||
pub mod storage;
|
||||
pub mod version;
|
||||
|
||||
/// Generated protobuf modules.
|
||||
pub mod pb {
|
||||
pub const FILE_DESCRIPTOR_SET: &[u8] =
|
||||
tonic::include_file_descriptor_set!("seaweed_descriptor");
|
||||
|
||||
pub mod remote_pb {
|
||||
tonic::include_proto!("remote_pb");
|
||||
}
|
||||
pub mod volume_server_pb {
|
||||
tonic::include_proto!("volume_server_pb");
|
||||
}
|
||||
pub mod master_pb {
|
||||
tonic::include_proto!("master_pb");
|
||||
}
|
||||
pub mod filer_pb {
|
||||
tonic::include_proto!("filer_pb");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,488 +0,0 @@
|
||||
//! Prometheus metrics for the volume server.
|
||||
//!
|
||||
//! Mirrors the Go SeaweedFS volume server metrics.
|
||||
|
||||
use prometheus::{
|
||||
self, Encoder, GaugeVec, HistogramOpts, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec,
|
||||
Opts, Registry, TextEncoder,
|
||||
};
|
||||
use std::sync::Once;
|
||||
|
||||
use crate::version;
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct PushGatewayConfig {
|
||||
pub address: String,
|
||||
pub interval_seconds: u32,
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref REGISTRY: Registry = Registry::new();
|
||||
|
||||
// ---- Request metrics (Go: VolumeServerRequestCounter, VolumeServerRequestHistogram) ----
|
||||
|
||||
/// Request counter with labels `type` (HTTP method) and `code` (HTTP status).
|
||||
pub static ref REQUEST_COUNTER: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_request_total", "Volume server requests"),
|
||||
&["type", "code"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Request duration histogram with label `type` (HTTP method).
|
||||
pub static ref REQUEST_DURATION: HistogramVec = HistogramVec::new(
|
||||
HistogramOpts::new(
|
||||
"SeaweedFS_volumeServer_request_seconds",
|
||||
"Volume server request duration in seconds",
|
||||
).buckets(exponential_buckets(0.0001, 2.0, 24)),
|
||||
&["type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Handler counters (Go: VolumeServerHandlerCounter) ----
|
||||
|
||||
/// Handler-level operation counter with label `type`.
|
||||
pub static ref HANDLER_COUNTER: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_handler_total", "Volume server handler counters"),
|
||||
&["type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Vacuuming metrics (Go: VolumeServerVacuuming*) ----
|
||||
|
||||
/// Vacuuming compact counter with label `success` (true/false).
|
||||
pub static ref VACUUMING_COMPACT_COUNTER: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_vacuuming_compact_count", "Counter of volume vacuuming Compact counter"),
|
||||
&["success"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Vacuuming commit counter with label `success` (true/false).
|
||||
pub static ref VACUUMING_COMMIT_COUNTER: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_vacuuming_commit_count", "Counter of volume vacuuming commit counter"),
|
||||
&["success"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Vacuuming duration histogram with label `type` (compact/commit).
|
||||
pub static ref VACUUMING_HISTOGRAM: HistogramVec = HistogramVec::new(
|
||||
HistogramOpts::new(
|
||||
"SeaweedFS_volumeServer_vacuuming_seconds",
|
||||
"Volume vacuuming duration in seconds",
|
||||
).buckets(exponential_buckets(0.0001, 2.0, 24)),
|
||||
&["type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Volume gauges (Go: VolumeServerVolumeGauge, VolumeServerReadOnlyVolumeGauge) ----
|
||||
|
||||
/// Volumes per collection and type (volume/ec_shards).
|
||||
pub static ref VOLUME_GAUGE: GaugeVec = GaugeVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_volumes", "Number of volumes"),
|
||||
&["collection", "type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Read-only volumes per collection and type.
|
||||
pub static ref READ_ONLY_VOLUME_GAUGE: GaugeVec = GaugeVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_read_only_volumes", "Number of read-only volumes."),
|
||||
&["collection", "type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Maximum number of volumes this server can hold.
|
||||
pub static ref MAX_VOLUMES: IntGauge = IntGauge::new(
|
||||
"SeaweedFS_volumeServer_max_volumes",
|
||||
"Maximum number of volumes",
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Disk size gauges (Go: VolumeServerDiskSizeGauge) ----
|
||||
|
||||
/// Actual disk size used by volumes per collection and type (normal/deleted_bytes/ec).
|
||||
pub static ref DISK_SIZE_GAUGE: GaugeVec = GaugeVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_total_disk_size", "Actual disk size used by volumes"),
|
||||
&["collection", "type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Resource gauges (Go: VolumeServerResourceGauge) ----
|
||||
|
||||
/// Disk resource usage per directory and type (all/used/free/avail).
|
||||
pub static ref RESOURCE_GAUGE: GaugeVec = GaugeVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_resource", "Server resource usage"),
|
||||
&["name", "type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- In-flight gauges (Go: VolumeServerInFlightRequestsGauge, InFlightDownload/UploadSize) ----
|
||||
|
||||
/// In-flight requests per HTTP method.
|
||||
pub static ref INFLIGHT_REQUESTS_GAUGE: IntGaugeVec = IntGaugeVec::new(
|
||||
Opts::new("SeaweedFS_volumeServer_in_flight_requests", "Current number of in-flight requests being handled by volume server."),
|
||||
&["type"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Concurrent download limit in bytes.
|
||||
pub static ref CONCURRENT_DOWNLOAD_LIMIT: IntGauge = IntGauge::new(
|
||||
"SeaweedFS_volumeServer_concurrent_download_limit",
|
||||
"Limit for total concurrent download size in bytes",
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Concurrent upload limit in bytes.
|
||||
pub static ref CONCURRENT_UPLOAD_LIMIT: IntGauge = IntGauge::new(
|
||||
"SeaweedFS_volumeServer_concurrent_upload_limit",
|
||||
"Limit for total concurrent upload size in bytes",
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Current in-flight download bytes.
|
||||
pub static ref INFLIGHT_DOWNLOAD_SIZE: IntGauge = IntGauge::new(
|
||||
"SeaweedFS_volumeServer_in_flight_download_size",
|
||||
"In flight total download size.",
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Current in-flight upload bytes.
|
||||
pub static ref INFLIGHT_UPLOAD_SIZE: IntGauge = IntGauge::new(
|
||||
"SeaweedFS_volumeServer_in_flight_upload_size",
|
||||
"In flight total upload size.",
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Upload error counter by HTTP status code. Code "0" = transport error (no response).
|
||||
pub static ref UPLOAD_ERROR_COUNTER: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new("SeaweedFS_upload_error_total",
|
||||
"Counter of upload errors by HTTP status code. Code 0 means transport error (no response received)."),
|
||||
&["code"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Scrubbing metrics (Go: VolumeServerScrub*) ----
|
||||
|
||||
/// Last scrub execution time, as seconds since UNIX epoch, with label `mode`.
|
||||
pub static ref SCRUB_LAST_TIME_SECONDS: GaugeVec = GaugeVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_last_time_seconds",
|
||||
"Last scrub execution time, as seconds since UNIX epoch.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Counter of overall volumes with issues detected during scrubbing, with label `mode`.
|
||||
pub static ref SCRUB_VOLUME_FAILURES: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_volume_failures",
|
||||
"Counter of overall volumes with issues detected during scrubbing.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Counter of overall EC shards with issues detected during scrubbing, with label `mode`.
|
||||
pub static ref SCRUB_SHARD_FAILURES: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_shard_failures",
|
||||
"Counter of overall EC shards with issues detected during scrubbing.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Legacy aliases for backward compat with existing code ----
|
||||
|
||||
/// Total number of volumes on this server (flat gauge).
|
||||
pub static ref VOLUMES_TOTAL: IntGauge = IntGauge::new(
|
||||
"volume_server_volumes_total",
|
||||
"Total number of volumes",
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Disk size in bytes per directory.
|
||||
pub static ref DISK_SIZE_BYTES: IntGaugeVec = IntGaugeVec::new(
|
||||
Opts::new("volume_server_disk_size_bytes", "Disk size in bytes"),
|
||||
&["dir"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Disk free bytes per directory.
|
||||
pub static ref DISK_FREE_BYTES: IntGaugeVec = IntGaugeVec::new(
|
||||
Opts::new("volume_server_disk_free_bytes", "Disk free space in bytes"),
|
||||
&["dir"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Current number of in-flight requests (flat gauge).
|
||||
pub static ref INFLIGHT_REQUESTS: IntGauge = IntGauge::new(
|
||||
"volume_server_inflight_requests",
|
||||
"Current number of in-flight requests",
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Total number of files stored across all volumes.
|
||||
pub static ref VOLUME_FILE_COUNT: IntGauge = IntGauge::new(
|
||||
"volume_server_volume_file_count",
|
||||
"Total number of files stored across all volumes",
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Build info (Go: BuildInfo) ----
|
||||
|
||||
/// Build information gauge, always set to 1. Matches Go:
|
||||
/// Namespace="SeaweedFS", Subsystem="build", Name="info",
|
||||
/// labels: version, commit, sizelimit, goos, goarch.
|
||||
pub static ref BUILD_INFO: GaugeVec = GaugeVec::new(
|
||||
Opts::new("SeaweedFS_build_info", "A metric with a constant '1' value labeled by version, commit, sizelimit, goos, and goarch from which SeaweedFS was built."),
|
||||
&["version", "commit", "sizelimit", "goos", "goarch"],
|
||||
).expect("metric can be created");
|
||||
}
|
||||
|
||||
/// Generate exponential bucket boundaries for histograms.
|
||||
fn exponential_buckets(start: f64, factor: f64, count: usize) -> Vec<f64> {
|
||||
let mut buckets = Vec::with_capacity(count);
|
||||
let mut val = start;
|
||||
for _ in 0..count {
|
||||
buckets.push(val);
|
||||
val *= factor;
|
||||
}
|
||||
buckets
|
||||
}
|
||||
|
||||
// Handler counter type constants (matches Go's metrics_names.go).
|
||||
pub const WRITE_TO_LOCAL_DISK: &str = "writeToLocalDisk";
|
||||
pub const WRITE_TO_REPLICAS: &str = "writeToReplicas";
|
||||
pub const DOWNLOAD_LIMIT_COND: &str = "downloadLimitCondition";
|
||||
pub const UPLOAD_LIMIT_COND: &str = "uploadLimitCondition";
|
||||
pub const READ_PROXY_REQ: &str = "readProxyRequest";
|
||||
pub const READ_REDIRECT_REQ: &str = "readRedirectRequest";
|
||||
pub const EMPTY_READ_PROXY_LOC: &str = "emptyReadProxyLocaction";
|
||||
pub const FAILED_READ_PROXY_REQ: &str = "failedReadProxyRequest";
|
||||
|
||||
// Error metric name constants.
|
||||
pub const ERROR_SIZE_MISMATCH_OFFSET_SIZE: &str = "errorSizeMismatchOffsetSize";
|
||||
pub const ERROR_SIZE_MISMATCH: &str = "errorSizeMismatch";
|
||||
pub const ERROR_CRC: &str = "errorCRC";
|
||||
pub const ERROR_INDEX_OUT_OF_RANGE: &str = "errorIndexOutOfRange";
|
||||
pub const ERROR_GET_NOT_FOUND: &str = "errorGetNotFound";
|
||||
pub const ERROR_GET_INTERNAL: &str = "errorGetInternal";
|
||||
pub const ERROR_WRITE_TO_LOCAL_DISK: &str = "errorWriteToLocalDisk";
|
||||
pub const ERROR_UNMARSHAL_PAIRS: &str = "errorUnmarshalPairs";
|
||||
pub const ERROR_WRITE_TO_REPLICAS: &str = "errorWriteToReplicas";
|
||||
|
||||
// Go volume heartbeat metric label values.
|
||||
pub const READ_ONLY_LABEL_IS_READ_ONLY: &str = "IsReadOnly";
|
||||
pub const READ_ONLY_LABEL_NO_WRITE_OR_DELETE: &str = "noWriteOrDelete";
|
||||
pub const READ_ONLY_LABEL_NO_WRITE_CAN_DELETE: &str = "noWriteCanDelete";
|
||||
pub const READ_ONLY_LABEL_IS_DISK_SPACE_LOW: &str = "isDiskSpaceLow";
|
||||
pub const DISK_SIZE_LABEL_NORMAL: &str = "normal";
|
||||
pub const DISK_SIZE_LABEL_DELETED_BYTES: &str = "deleted_bytes";
|
||||
pub const DISK_SIZE_LABEL_EC: &str = "ec";
|
||||
|
||||
static REGISTER_METRICS: Once = Once::new();
|
||||
|
||||
/// Register all metrics with the custom registry.
|
||||
/// Call this once at startup.
|
||||
pub fn register_metrics() {
|
||||
REGISTER_METRICS.call_once(|| {
|
||||
let metrics: Vec<Box<dyn prometheus::core::Collector>> = vec![
|
||||
// New Go-compatible metrics
|
||||
Box::new(REQUEST_COUNTER.clone()),
|
||||
Box::new(REQUEST_DURATION.clone()),
|
||||
Box::new(HANDLER_COUNTER.clone()),
|
||||
Box::new(VACUUMING_COMPACT_COUNTER.clone()),
|
||||
Box::new(VACUUMING_COMMIT_COUNTER.clone()),
|
||||
Box::new(VACUUMING_HISTOGRAM.clone()),
|
||||
Box::new(VOLUME_GAUGE.clone()),
|
||||
Box::new(READ_ONLY_VOLUME_GAUGE.clone()),
|
||||
Box::new(MAX_VOLUMES.clone()),
|
||||
Box::new(DISK_SIZE_GAUGE.clone()),
|
||||
Box::new(RESOURCE_GAUGE.clone()),
|
||||
Box::new(INFLIGHT_REQUESTS_GAUGE.clone()),
|
||||
Box::new(CONCURRENT_DOWNLOAD_LIMIT.clone()),
|
||||
Box::new(CONCURRENT_UPLOAD_LIMIT.clone()),
|
||||
Box::new(INFLIGHT_DOWNLOAD_SIZE.clone()),
|
||||
Box::new(INFLIGHT_UPLOAD_SIZE.clone()),
|
||||
Box::new(UPLOAD_ERROR_COUNTER.clone()),
|
||||
Box::new(SCRUB_LAST_TIME_SECONDS.clone()),
|
||||
Box::new(SCRUB_VOLUME_FAILURES.clone()),
|
||||
Box::new(SCRUB_SHARD_FAILURES.clone()),
|
||||
// Legacy metrics
|
||||
Box::new(VOLUMES_TOTAL.clone()),
|
||||
Box::new(DISK_SIZE_BYTES.clone()),
|
||||
Box::new(DISK_FREE_BYTES.clone()),
|
||||
Box::new(INFLIGHT_REQUESTS.clone()),
|
||||
Box::new(VOLUME_FILE_COUNT.clone()),
|
||||
// Build info
|
||||
Box::new(BUILD_INFO.clone()),
|
||||
];
|
||||
for m in metrics {
|
||||
REGISTRY.register(m).expect("metric registered");
|
||||
}
|
||||
|
||||
// Set build info gauge to 1 with version/commit/sizelimit/os/arch labels (matches Go).
|
||||
BUILD_INFO
|
||||
.with_label_values(&[
|
||||
version::version(),
|
||||
version::commit(),
|
||||
version::size_limit(),
|
||||
std::env::consts::OS,
|
||||
std::env::consts::ARCH,
|
||||
])
|
||||
.set(1.0);
|
||||
});
|
||||
}
|
||||
|
||||
/// Gather all metrics and encode them in Prometheus text exposition format.
|
||||
pub fn gather_metrics() -> String {
|
||||
let encoder = TextEncoder::new();
|
||||
let metric_families = REGISTRY.gather();
|
||||
let mut buffer = Vec::new();
|
||||
encoder
|
||||
.encode(&metric_families, &mut buffer)
|
||||
.expect("encoding metrics");
|
||||
String::from_utf8(buffer).expect("metrics are valid UTF-8")
|
||||
}
|
||||
|
||||
pub fn delete_collection_metrics(collection: &str) {
|
||||
// Mirrors Go's DeletePartialMatch(prometheus.Labels{"collection": collection})
|
||||
// which removes ALL metric entries matching the collection label, regardless
|
||||
// of other label values (like "type"). We gather the metric families to discover
|
||||
// all type values dynamically, matching Go's partial-match behavior.
|
||||
delete_partial_match_collection(&VOLUME_GAUGE, collection);
|
||||
delete_partial_match_collection(&READ_ONLY_VOLUME_GAUGE, collection);
|
||||
delete_partial_match_collection(&DISK_SIZE_GAUGE, collection);
|
||||
}
|
||||
|
||||
/// Remove all metric entries from a GaugeVec where the "collection" label matches.
|
||||
/// This emulates Go's `DeletePartialMatch(prometheus.Labels{"collection": collection})`.
|
||||
fn delete_partial_match_collection(gauge: &GaugeVec, collection: &str) {
|
||||
use prometheus::core::Collector;
|
||||
let families = gauge.collect();
|
||||
for family in &families {
|
||||
for metric in family.get_metric() {
|
||||
let labels = metric.get_label();
|
||||
let mut matches_collection = false;
|
||||
let mut type_value = None;
|
||||
for label in labels {
|
||||
if label.get_name() == "collection" && label.get_value() == collection {
|
||||
matches_collection = true;
|
||||
}
|
||||
if label.get_name() == "type" {
|
||||
type_value = Some(label.get_value().to_string());
|
||||
}
|
||||
}
|
||||
if matches_collection {
|
||||
if let Some(ref tv) = type_value {
|
||||
let _ = gauge.remove_label_values(&[collection, tv]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_pushgateway_url(address: &str, job: &str, instance: &str) -> String {
|
||||
let base = if address.starts_with("http://") || address.starts_with("https://") {
|
||||
address.to_string()
|
||||
} else {
|
||||
format!("http://{}", address)
|
||||
};
|
||||
let base = base.trim_end_matches('/');
|
||||
format!("{}/metrics/job/{}/instance/{}", base, job, instance)
|
||||
}
|
||||
|
||||
pub async fn push_metrics_once(
|
||||
client: &reqwest::Client,
|
||||
address: &str,
|
||||
job: &str,
|
||||
instance: &str,
|
||||
) -> Result<(), String> {
|
||||
let url = build_pushgateway_url(address, job, instance);
|
||||
let response = client
|
||||
.put(&url)
|
||||
.header(
|
||||
reqwest::header::CONTENT_TYPE,
|
||||
"text/plain; version=0.0.4; charset=utf-8",
|
||||
)
|
||||
.body(gather_metrics())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("push metrics request failed: {}", e))?;
|
||||
|
||||
if response.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"push metrics failed with status {}",
|
||||
response.status()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{routing::put, Router};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[test]
|
||||
fn test_gather_metrics_returns_text() {
|
||||
register_metrics();
|
||||
REQUEST_COUNTER.with_label_values(&["GET", "200"]).inc();
|
||||
let output = gather_metrics();
|
||||
assert!(output.contains("SeaweedFS_volumeServer_request_total"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_pushgateway_url() {
|
||||
assert_eq!(
|
||||
build_pushgateway_url("localhost:9091", "volumeServer", "test-instance"),
|
||||
"http://localhost:9091/metrics/job/volumeServer/instance/test-instance"
|
||||
);
|
||||
assert_eq!(
|
||||
build_pushgateway_url("https://push.example", "volumeServer", "node-a"),
|
||||
"https://push.example/metrics/job/volumeServer/instance/node-a"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_push_metrics_once() {
|
||||
register_metrics();
|
||||
|
||||
let captured = Arc::new(Mutex::new(None::<String>));
|
||||
let captured_clone = captured.clone();
|
||||
|
||||
let app = Router::new().route(
|
||||
"/metrics/job/volumeServer/instance/test-instance",
|
||||
put(move |body: String| {
|
||||
let captured = captured_clone.clone();
|
||||
async move {
|
||||
*captured.lock().unwrap() = Some(body);
|
||||
"ok"
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
push_metrics_once(
|
||||
&client,
|
||||
&format!("127.0.0.1:{}", addr.port()),
|
||||
"volumeServer",
|
||||
"test-instance",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let body = captured.lock().unwrap().clone().unwrap();
|
||||
assert!(body.contains("SeaweedFS_volumeServer_request_total"));
|
||||
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_collection_metrics_removes_collection_labelsets() {
|
||||
register_metrics();
|
||||
|
||||
VOLUME_GAUGE.with_label_values(&["pics", "volume"]).set(2.0);
|
||||
VOLUME_GAUGE.with_label_values(&["pics", "ec_shards"]).set(3.0);
|
||||
READ_ONLY_VOLUME_GAUGE
|
||||
.with_label_values(&["pics", "volume"])
|
||||
.set(1.0);
|
||||
DISK_SIZE_GAUGE
|
||||
.with_label_values(&["pics", "normal"])
|
||||
.set(10.0);
|
||||
DISK_SIZE_GAUGE
|
||||
.with_label_values(&["pics", "deleted_bytes"])
|
||||
.set(4.0);
|
||||
|
||||
delete_collection_metrics("pics");
|
||||
|
||||
let output = gather_metrics();
|
||||
assert!(!output.contains("collection=\"pics\",type=\"volume\""));
|
||||
assert!(!output.contains("collection=\"pics\",type=\"ec_shards\""));
|
||||
assert!(!output.contains("collection=\"pics\",type=\"normal\""));
|
||||
assert!(!output.contains("collection=\"pics\",type=\"deleted_bytes\""));
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
//! Remote storage backends for tiered storage support.
|
||||
//!
|
||||
//! Provides a trait-based abstraction over cloud storage providers (S3, GCS, Azure, etc.)
|
||||
//! and a registry to create clients from protobuf RemoteConf messages.
|
||||
|
||||
pub mod s3;
|
||||
pub mod s3_tier;
|
||||
|
||||
use crate::pb::remote_pb::{RemoteConf, RemoteStorageLocation};
|
||||
|
||||
/// Error type for remote storage operations.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RemoteStorageError {
|
||||
#[error("remote storage type {0} not found")]
|
||||
TypeNotFound(String),
|
||||
#[error("remote object not found: {0}")]
|
||||
ObjectNotFound(String),
|
||||
#[error("remote storage error: {0}")]
|
||||
Other(String),
|
||||
#[error("io error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
/// Metadata about a remote file entry.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RemoteEntry {
|
||||
pub size: i64,
|
||||
pub last_modified_at: i64, // Unix seconds
|
||||
pub e_tag: String,
|
||||
pub storage_name: String,
|
||||
}
|
||||
|
||||
/// Trait for remote storage clients. Matches Go's RemoteStorageClient interface.
|
||||
#[async_trait::async_trait]
|
||||
pub trait RemoteStorageClient: Send + Sync {
|
||||
/// Read (part of) a file from remote storage.
|
||||
async fn read_file(
|
||||
&self,
|
||||
loc: &RemoteStorageLocation,
|
||||
offset: i64,
|
||||
size: i64,
|
||||
) -> Result<Vec<u8>, RemoteStorageError>;
|
||||
|
||||
/// Write a file to remote storage.
|
||||
async fn write_file(
|
||||
&self,
|
||||
loc: &RemoteStorageLocation,
|
||||
data: &[u8],
|
||||
) -> Result<RemoteEntry, RemoteStorageError>;
|
||||
|
||||
/// Get metadata for a file in remote storage.
|
||||
async fn stat_file(
|
||||
&self,
|
||||
loc: &RemoteStorageLocation,
|
||||
) -> Result<RemoteEntry, RemoteStorageError>;
|
||||
|
||||
/// Delete a file from remote storage.
|
||||
async fn delete_file(&self, loc: &RemoteStorageLocation) -> Result<(), RemoteStorageError>;
|
||||
|
||||
/// List all buckets.
|
||||
async fn list_buckets(&self) -> Result<Vec<String>, RemoteStorageError>;
|
||||
|
||||
/// The RemoteConf used to create this client.
|
||||
fn remote_conf(&self) -> &RemoteConf;
|
||||
}
|
||||
|
||||
/// Create a new remote storage client from a RemoteConf.
|
||||
pub fn make_remote_storage_client(
|
||||
conf: &RemoteConf,
|
||||
) -> Result<Box<dyn RemoteStorageClient>, RemoteStorageError> {
|
||||
match conf.r#type.as_str() {
|
||||
// All S3-compatible backends use the same client with different credentials
|
||||
"s3" | "wasabi" | "backblaze" | "aliyun" | "tencent" | "baidu" | "filebase" | "storj"
|
||||
| "contabo" => {
|
||||
let (access_key, secret_key, endpoint, region) = extract_s3_credentials(conf);
|
||||
Ok(Box::new(s3::S3RemoteStorageClient::new(
|
||||
conf.clone(),
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion,
|
||||
&endpoint,
|
||||
conf.s3_force_path_style,
|
||||
)))
|
||||
}
|
||||
other => Err(RemoteStorageError::TypeNotFound(other.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract S3-compatible credentials from a RemoteConf based on its type.
|
||||
fn extract_s3_credentials(conf: &RemoteConf) -> (String, String, String, String) {
|
||||
match conf.r#type.as_str() {
|
||||
"s3" => (
|
||||
conf.s3_access_key.clone(),
|
||||
conf.s3_secret_key.clone(),
|
||||
conf.s3_endpoint.clone(),
|
||||
if conf.s3_region.is_empty() {
|
||||
"us-east-1".to_string()
|
||||
} else {
|
||||
conf.s3_region.clone()
|
||||
},
|
||||
),
|
||||
"wasabi" => (
|
||||
conf.wasabi_access_key.clone(),
|
||||
conf.wasabi_secret_key.clone(),
|
||||
conf.wasabi_endpoint.clone(),
|
||||
conf.wasabi_region.clone(),
|
||||
),
|
||||
"backblaze" => (
|
||||
conf.backblaze_key_id.clone(),
|
||||
conf.backblaze_application_key.clone(),
|
||||
conf.backblaze_endpoint.clone(),
|
||||
conf.backblaze_region.clone(),
|
||||
),
|
||||
"aliyun" => (
|
||||
conf.aliyun_access_key.clone(),
|
||||
conf.aliyun_secret_key.clone(),
|
||||
conf.aliyun_endpoint.clone(),
|
||||
conf.aliyun_region.clone(),
|
||||
),
|
||||
"tencent" => (
|
||||
conf.tencent_secret_id.clone(),
|
||||
conf.tencent_secret_key.clone(),
|
||||
conf.tencent_endpoint.clone(),
|
||||
String::new(),
|
||||
),
|
||||
"baidu" => (
|
||||
conf.baidu_access_key.clone(),
|
||||
conf.baidu_secret_key.clone(),
|
||||
conf.baidu_endpoint.clone(),
|
||||
conf.baidu_region.clone(),
|
||||
),
|
||||
"filebase" => (
|
||||
conf.filebase_access_key.clone(),
|
||||
conf.filebase_secret_key.clone(),
|
||||
conf.filebase_endpoint.clone(),
|
||||
String::new(),
|
||||
),
|
||||
"storj" => (
|
||||
conf.storj_access_key.clone(),
|
||||
conf.storj_secret_key.clone(),
|
||||
conf.storj_endpoint.clone(),
|
||||
String::new(),
|
||||
),
|
||||
"contabo" => (
|
||||
conf.contabo_access_key.clone(),
|
||||
conf.contabo_secret_key.clone(),
|
||||
conf.contabo_endpoint.clone(),
|
||||
conf.contabo_region.clone(),
|
||||
),
|
||||
_ => (
|
||||
conf.s3_access_key.clone(),
|
||||
conf.s3_secret_key.clone(),
|
||||
conf.s3_endpoint.clone(),
|
||||
conf.s3_region.clone(),
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
//! S3-compatible remote storage client.
|
||||
//!
|
||||
//! Works with AWS S3, MinIO, SeaweedFS S3, and all S3-compatible providers.
|
||||
|
||||
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::Client;
|
||||
|
||||
use super::{RemoteEntry, RemoteStorageClient, RemoteStorageError};
|
||||
use crate::pb::remote_pb::{RemoteConf, RemoteStorageLocation};
|
||||
|
||||
/// S3-compatible remote storage client.
|
||||
pub struct S3RemoteStorageClient {
|
||||
client: Client,
|
||||
conf: RemoteConf,
|
||||
}
|
||||
|
||||
impl S3RemoteStorageClient {
|
||||
/// Create a new S3 client from credentials and endpoint configuration.
|
||||
pub fn new(
|
||||
conf: RemoteConf,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
region: &str,
|
||||
endpoint: &str,
|
||||
force_path_style: bool,
|
||||
) -> Self {
|
||||
let region = if region.is_empty() {
|
||||
"us-east-1"
|
||||
} else {
|
||||
region
|
||||
};
|
||||
|
||||
let credentials = Credentials::new(
|
||||
access_key,
|
||||
secret_key,
|
||||
None, // session token
|
||||
None, // expiry
|
||||
"seaweedfs-volume",
|
||||
);
|
||||
|
||||
let mut s3_config = aws_sdk_s3::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(region.to_string()))
|
||||
.credentials_provider(credentials)
|
||||
.force_path_style(force_path_style);
|
||||
|
||||
if !endpoint.is_empty() {
|
||||
s3_config = s3_config.endpoint_url(endpoint);
|
||||
}
|
||||
|
||||
let client = Client::from_conf(s3_config.build());
|
||||
|
||||
S3RemoteStorageClient { client, conf }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl RemoteStorageClient for S3RemoteStorageClient {
|
||||
async fn read_file(
|
||||
&self,
|
||||
loc: &RemoteStorageLocation,
|
||||
offset: i64,
|
||||
size: i64,
|
||||
) -> Result<Vec<u8>, RemoteStorageError> {
|
||||
let key = loc.path.trim_start_matches('/');
|
||||
|
||||
let mut req = self.client.get_object().bucket(&loc.bucket).key(key);
|
||||
|
||||
// Set byte range if specified
|
||||
if size > 0 {
|
||||
let end = offset + size - 1;
|
||||
req = req.range(format!("bytes={}-{}", offset, end));
|
||||
} else if offset > 0 {
|
||||
req = req.range(format!("bytes={}-", offset));
|
||||
}
|
||||
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
let msg = format!("{}", e);
|
||||
if msg.contains("NoSuchKey") || msg.contains("404") {
|
||||
RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
|
||||
} else {
|
||||
RemoteStorageError::Other(format!("s3 get object: {}", e))
|
||||
}
|
||||
})?;
|
||||
|
||||
let data = resp
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|e| RemoteStorageError::Other(format!("s3 read body: {}", e)))?;
|
||||
|
||||
Ok(data.into_bytes().to_vec())
|
||||
}
|
||||
|
||||
async fn write_file(
|
||||
&self,
|
||||
loc: &RemoteStorageLocation,
|
||||
data: &[u8],
|
||||
) -> Result<RemoteEntry, RemoteStorageError> {
|
||||
let key = loc.path.trim_start_matches('/');
|
||||
|
||||
let resp = self
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(&loc.bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(data.to_vec()))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteStorageError::Other(format!("s3 put object: {}", e)))?;
|
||||
|
||||
Ok(RemoteEntry {
|
||||
size: data.len() as i64,
|
||||
last_modified_at: std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs() as i64,
|
||||
e_tag: resp.e_tag().unwrap_or_default().to_string(),
|
||||
storage_name: loc.name.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn stat_file(
|
||||
&self,
|
||||
loc: &RemoteStorageLocation,
|
||||
) -> Result<RemoteEntry, RemoteStorageError> {
|
||||
let key = loc.path.trim_start_matches('/');
|
||||
|
||||
let resp = self
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(&loc.bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
let msg = format!("{}", e);
|
||||
if msg.contains("404") || msg.contains("NotFound") {
|
||||
RemoteStorageError::ObjectNotFound(format!("{}/{}", loc.bucket, key))
|
||||
} else {
|
||||
RemoteStorageError::Other(format!("s3 head object: {}", e))
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(RemoteEntry {
|
||||
size: resp.content_length().unwrap_or(0),
|
||||
last_modified_at: resp.last_modified().map(|t| t.secs()).unwrap_or(0),
|
||||
e_tag: resp.e_tag().unwrap_or_default().to_string(),
|
||||
storage_name: loc.name.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_file(&self, loc: &RemoteStorageLocation) -> Result<(), RemoteStorageError> {
|
||||
let key = loc.path.trim_start_matches('/');
|
||||
|
||||
self.client
|
||||
.delete_object()
|
||||
.bucket(&loc.bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteStorageError::Other(format!("s3 delete object: {}", e)))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<String>, RemoteStorageError> {
|
||||
let resp = self
|
||||
.client
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteStorageError::Other(format!("s3 list buckets: {}", e)))?;
|
||||
|
||||
Ok(resp
|
||||
.buckets()
|
||||
.iter()
|
||||
.filter_map(|b| b.name().map(String::from))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn remote_conf(&self) -> &RemoteConf {
|
||||
&self.conf
|
||||
}
|
||||
}
|
||||
@@ -1,519 +0,0 @@
|
||||
//! S3-compatible tiered storage backend for volume .dat file upload/download.
|
||||
//!
|
||||
//! Provides multipart upload and concurrent download with progress callbacks,
|
||||
//! matching the Go SeaweedFS S3 backend behavior.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, OnceLock, RwLock};
|
||||
|
||||
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use aws_sdk_s3::Client;
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
/// Concurrency limit for multipart upload/download (matches Go's s3manager).
|
||||
const CONCURRENCY: usize = 5;
|
||||
|
||||
/// Configuration for an S3 tier backend.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct S3TierConfig {
|
||||
pub access_key: String,
|
||||
pub secret_key: String,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
pub endpoint: String,
|
||||
pub storage_class: String,
|
||||
pub force_path_style: bool,
|
||||
}
|
||||
|
||||
/// S3 tier backend for uploading/downloading volume .dat files.
|
||||
pub struct S3TierBackend {
|
||||
client: Client,
|
||||
pub bucket: String,
|
||||
pub storage_class: String,
|
||||
}
|
||||
|
||||
impl S3TierBackend {
|
||||
/// Create a new S3 tier backend from configuration.
|
||||
pub fn new(config: &S3TierConfig) -> Self {
|
||||
let region = if config.region.is_empty() {
|
||||
"us-east-1"
|
||||
} else {
|
||||
&config.region
|
||||
};
|
||||
|
||||
let credentials = Credentials::new(
|
||||
&config.access_key,
|
||||
&config.secret_key,
|
||||
None,
|
||||
None,
|
||||
"seaweedfs-volume-tier",
|
||||
);
|
||||
|
||||
let mut s3_config = aws_sdk_s3::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(region.to_string()))
|
||||
.credentials_provider(credentials)
|
||||
.force_path_style(config.force_path_style);
|
||||
|
||||
if !config.endpoint.is_empty() {
|
||||
s3_config = s3_config.endpoint_url(&config.endpoint);
|
||||
}
|
||||
|
||||
let client = Client::from_conf(s3_config.build());
|
||||
|
||||
S3TierBackend {
|
||||
client,
|
||||
bucket: config.bucket.clone(),
|
||||
storage_class: if config.storage_class.is_empty() {
|
||||
"STANDARD_IA".to_string()
|
||||
} else {
|
||||
config.storage_class.clone()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Upload a local file to S3 using multipart upload with concurrent parts
|
||||
/// and progress reporting.
|
||||
///
|
||||
/// Returns (s3_key, file_size) on success.
|
||||
/// The progress callback receives (bytes_uploaded, percentage).
|
||||
/// Uses 64MB part size and 5 concurrent uploads (matches Go s3manager).
|
||||
pub async fn upload_file<F>(
|
||||
&self,
|
||||
file_path: &str,
|
||||
progress_fn: F,
|
||||
) -> Result<(String, u64), String>
|
||||
where
|
||||
F: FnMut(i64, f32) + Send + Sync + 'static,
|
||||
{
|
||||
let key = uuid::Uuid::new_v4().to_string();
|
||||
|
||||
let metadata = tokio::fs::metadata(file_path)
|
||||
.await
|
||||
.map_err(|e| format!("failed to stat file {}: {}", file_path, e))?;
|
||||
let file_size = metadata.len();
|
||||
|
||||
// Calculate part size: start at 64MB, scale up for very large files (matches Go)
|
||||
let mut part_size: u64 = 64 * 1024 * 1024;
|
||||
while part_size * 1000 < file_size {
|
||||
part_size *= 4;
|
||||
}
|
||||
|
||||
// Initiate multipart upload
|
||||
let create_resp = self
|
||||
.client
|
||||
.create_multipart_upload()
|
||||
.bucket(&self.bucket)
|
||||
.key(&key)
|
||||
.storage_class(
|
||||
self.storage_class
|
||||
.parse()
|
||||
.unwrap_or(aws_sdk_s3::types::StorageClass::StandardIa),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to create multipart upload: {}", e))?;
|
||||
|
||||
let upload_id = create_resp
|
||||
.upload_id()
|
||||
.ok_or_else(|| "no upload_id in multipart upload response".to_string())?
|
||||
.to_string();
|
||||
|
||||
// Build list of (part_number, offset, size) for all parts
|
||||
let mut parts_plan: Vec<(i32, u64, usize)> = Vec::new();
|
||||
let mut offset: u64 = 0;
|
||||
let mut part_number: i32 = 1;
|
||||
while offset < file_size {
|
||||
let remaining = file_size - offset;
|
||||
let this_part_size = std::cmp::min(part_size, remaining) as usize;
|
||||
parts_plan.push((part_number, offset, this_part_size));
|
||||
offset += this_part_size as u64;
|
||||
part_number += 1;
|
||||
}
|
||||
|
||||
// Upload parts concurrently with a semaphore limiting to CONCURRENCY
|
||||
let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
|
||||
let client = &self.client;
|
||||
let bucket = &self.bucket;
|
||||
let file_path_owned = file_path.to_string();
|
||||
let progress = Arc::new(std::sync::Mutex::new((0u64, progress_fn)));
|
||||
|
||||
let mut handles = Vec::with_capacity(parts_plan.len());
|
||||
for (pn, off, size) in parts_plan {
|
||||
let sem = semaphore.clone();
|
||||
let client = client.clone();
|
||||
let bucket = bucket.clone();
|
||||
let key = key.clone();
|
||||
let upload_id = upload_id.clone();
|
||||
let fp = file_path_owned.clone();
|
||||
let progress = progress.clone();
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let _permit = sem
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|e| format!("semaphore error: {}", e))?;
|
||||
|
||||
// Read this part's data from the file at the correct offset
|
||||
let mut file = tokio::fs::File::open(&fp)
|
||||
.await
|
||||
.map_err(|e| format!("failed to open file {}: {}", fp, e))?;
|
||||
file.seek(std::io::SeekFrom::Start(off))
|
||||
.await
|
||||
.map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
|
||||
let mut buf = vec![0u8; size];
|
||||
file.read_exact(&mut buf)
|
||||
.await
|
||||
.map_err(|e| format!("failed to read file at offset {}: {}", off, e))?;
|
||||
|
||||
let upload_part_resp = client
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(pn)
|
||||
.body(buf.into())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
format!("failed to upload part {} at offset {}: {}", pn, off, e)
|
||||
})?;
|
||||
|
||||
let e_tag = upload_part_resp.e_tag().unwrap_or_default().to_string();
|
||||
|
||||
// Report progress
|
||||
{
|
||||
let mut guard = progress.lock().unwrap();
|
||||
guard.0 += size as u64;
|
||||
let uploaded = guard.0;
|
||||
let pct = if file_size > 0 {
|
||||
(uploaded as f32 * 100.0) / file_size as f32
|
||||
} else {
|
||||
100.0
|
||||
};
|
||||
(guard.1)(uploaded as i64, pct);
|
||||
}
|
||||
|
||||
Ok::<_, String>(
|
||||
CompletedPart::builder()
|
||||
.e_tag(e_tag)
|
||||
.part_number(pn)
|
||||
.build(),
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
// Collect results, preserving part order
|
||||
let mut completed_parts = Vec::with_capacity(handles.len());
|
||||
for handle in handles {
|
||||
let part = handle
|
||||
.await
|
||||
.map_err(|e| format!("upload task panicked: {}", e))??;
|
||||
completed_parts.push(part);
|
||||
}
|
||||
|
||||
// Complete multipart upload
|
||||
let completed_upload = CompletedMultipartUpload::builder()
|
||||
.set_parts(Some(completed_parts))
|
||||
.build();
|
||||
|
||||
self.client
|
||||
.complete_multipart_upload()
|
||||
.bucket(&self.bucket)
|
||||
.key(&key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(completed_upload)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to complete multipart upload: {}", e))?;
|
||||
|
||||
Ok((key, file_size))
|
||||
}
|
||||
|
||||
/// Download a file from S3 to a local path with concurrent range requests
|
||||
/// and progress reporting.
|
||||
///
|
||||
/// Returns the file size on success.
|
||||
/// Uses 64MB part size and 5 concurrent downloads (matches Go s3manager).
|
||||
pub async fn download_file<F>(
|
||||
&self,
|
||||
dest_path: &str,
|
||||
key: &str,
|
||||
progress_fn: F,
|
||||
) -> Result<u64, String>
|
||||
where
|
||||
F: FnMut(i64, f32) + Send + Sync + 'static,
|
||||
{
|
||||
// Get file size first
|
||||
let head_resp = self
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to head object {}: {}", key, e))?;
|
||||
|
||||
let file_size = head_resp.content_length().unwrap_or(0) as u64;
|
||||
|
||||
// Pre-allocate file to full size so concurrent WriteAt-style writes work
|
||||
{
|
||||
let file = tokio::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(dest_path)
|
||||
.await
|
||||
.map_err(|e| format!("failed to open dest file {}: {}", dest_path, e))?;
|
||||
file.set_len(file_size)
|
||||
.await
|
||||
.map_err(|e| format!("failed to set file length: {}", e))?;
|
||||
}
|
||||
|
||||
let part_size: u64 = 64 * 1024 * 1024;
|
||||
|
||||
// Build list of (offset, size) for all parts
|
||||
let mut parts_plan: Vec<(u64, u64)> = Vec::new();
|
||||
let mut offset: u64 = 0;
|
||||
while offset < file_size {
|
||||
let remaining = file_size - offset;
|
||||
let this_part_size = std::cmp::min(part_size, remaining);
|
||||
parts_plan.push((offset, this_part_size));
|
||||
offset += this_part_size;
|
||||
}
|
||||
|
||||
// Download parts concurrently with a semaphore limiting to CONCURRENCY
|
||||
let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
|
||||
let client = &self.client;
|
||||
let bucket = &self.bucket;
|
||||
let dest_path_owned = dest_path.to_string();
|
||||
let key_owned = key.to_string();
|
||||
let progress = Arc::new(std::sync::Mutex::new((0u64, progress_fn)));
|
||||
|
||||
let mut handles = Vec::with_capacity(parts_plan.len());
|
||||
for (off, size) in parts_plan {
|
||||
let sem = semaphore.clone();
|
||||
let client = client.clone();
|
||||
let bucket = bucket.clone();
|
||||
let key = key_owned.clone();
|
||||
let dp = dest_path_owned.clone();
|
||||
let progress = progress.clone();
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let _permit = sem
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|e| format!("semaphore error: {}", e))?;
|
||||
|
||||
let end = off + size - 1;
|
||||
let range = format!("bytes={}-{}", off, end);
|
||||
|
||||
let get_resp = client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.range(&range)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
|
||||
|
||||
let body = get_resp
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|e| format!("failed to read body: {}", e))?;
|
||||
let bytes = body.into_bytes();
|
||||
|
||||
// Write at the correct offset (like Go's WriteAt)
|
||||
let mut file = tokio::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(&dp)
|
||||
.await
|
||||
.map_err(|e| format!("failed to open dest file {}: {}", dp, e))?;
|
||||
file.seek(std::io::SeekFrom::Start(off))
|
||||
.await
|
||||
.map_err(|e| format!("failed to seek to offset {}: {}", off, e))?;
|
||||
file.write_all(&bytes)
|
||||
.await
|
||||
.map_err(|e| format!("failed to write to {}: {}", dp, e))?;
|
||||
|
||||
// Report progress
|
||||
{
|
||||
let mut guard = progress.lock().unwrap();
|
||||
guard.0 += bytes.len() as u64;
|
||||
let downloaded = guard.0;
|
||||
let pct = if file_size > 0 {
|
||||
(downloaded as f32 * 100.0) / file_size as f32
|
||||
} else {
|
||||
100.0
|
||||
};
|
||||
(guard.1)(downloaded as i64, pct);
|
||||
}
|
||||
|
||||
Ok::<_, String>(())
|
||||
}));
|
||||
}
|
||||
|
||||
// Wait for all download tasks
|
||||
for handle in handles {
|
||||
handle
|
||||
.await
|
||||
.map_err(|e| format!("download task panicked: {}", e))??;
|
||||
}
|
||||
|
||||
Ok(file_size)
|
||||
}
|
||||
|
||||
pub async fn read_range(&self, key: &str, offset: u64, size: usize) -> Result<Vec<u8>, String> {
|
||||
let end = offset + (size as u64).saturating_sub(1);
|
||||
let range = format!("bytes={}-{}", offset, end);
|
||||
let resp = self
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(key)
|
||||
.range(&range)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
|
||||
|
||||
let body = resp
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|e| format!("failed to read object {} body: {}", key, e))?;
|
||||
Ok(body.into_bytes().to_vec())
|
||||
}
|
||||
|
||||
/// Delete a file from S3.
|
||||
pub async fn delete_file(&self, key: &str) -> Result<(), String> {
|
||||
self.client
|
||||
.delete_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to delete object {}: {}", key, e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn delete_file_blocking(&self, key: &str) -> Result<(), String> {
|
||||
let client = self.client.clone();
|
||||
let bucket = self.bucket.clone();
|
||||
let key = key.to_string();
|
||||
block_on_tier_future(async move {
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to delete object {}: {}", key, e))?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn read_range_blocking(
|
||||
&self,
|
||||
key: &str,
|
||||
offset: u64,
|
||||
size: usize,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let client = self.client.clone();
|
||||
let bucket = self.bucket.clone();
|
||||
let key = key.to_string();
|
||||
block_on_tier_future(async move {
|
||||
let end = offset + (size as u64).saturating_sub(1);
|
||||
let range = format!("bytes={}-{}", offset, end);
|
||||
let resp = client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.range(&range)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("failed to get object {} range {}: {}", key, range, e))?;
|
||||
|
||||
let body = resp
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map_err(|e| format!("failed to read object {} body: {}", key, e))?;
|
||||
Ok(body.into_bytes().to_vec())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a backend name like "s3" or "s3.default" into (backend_type, backend_id).
|
||||
/// Matches Go's `BackendNameToTypeId`.
|
||||
pub fn backend_name_to_type_id(backend_name: &str) -> (String, String) {
|
||||
let parts: Vec<&str> = backend_name.split('.').collect();
|
||||
match parts.len() {
|
||||
1 => (backend_name.to_string(), "default".to_string()),
|
||||
2 => (parts[0].to_string(), parts[1].to_string()),
|
||||
_ => (String::new(), String::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// A registry of configured S3 tier backends, keyed by backend name (e.g., "s3.default").
|
||||
#[derive(Default)]
|
||||
pub struct S3TierRegistry {
|
||||
backends: HashMap<String, Arc<S3TierBackend>>,
|
||||
}
|
||||
|
||||
impl S3TierRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
backends: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a backend with the given name.
|
||||
pub fn register(&mut self, name: String, backend: S3TierBackend) {
|
||||
self.backends.insert(name, Arc::new(backend));
|
||||
}
|
||||
|
||||
/// Look up a backend by name.
|
||||
pub fn get(&self, name: &str) -> Option<Arc<S3TierBackend>> {
|
||||
self.backends.get(name).cloned()
|
||||
}
|
||||
|
||||
/// List all registered backend names.
|
||||
pub fn names(&self) -> Vec<String> {
|
||||
self.backends.keys().cloned().collect()
|
||||
}
|
||||
|
||||
/// Remove a backend by name.
|
||||
pub fn remove(&mut self, name: &str) {
|
||||
self.backends.remove(name);
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.backends.clear();
|
||||
}
|
||||
}
|
||||
|
||||
static GLOBAL_S3_TIER_REGISTRY: OnceLock<RwLock<S3TierRegistry>> = OnceLock::new();
|
||||
|
||||
pub fn global_s3_tier_registry() -> &'static RwLock<S3TierRegistry> {
|
||||
GLOBAL_S3_TIER_REGISTRY.get_or_init(|| RwLock::new(S3TierRegistry::new()))
|
||||
}
|
||||
|
||||
fn block_on_tier_future<F, T>(future: F) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|e| format!("failed to build tokio runtime: {}", e))?;
|
||||
runtime.block_on(future)
|
||||
})
|
||||
.join()
|
||||
.map_err(|_| "tier runtime thread panicked".to_string())?
|
||||
}
|
||||
@@ -1,481 +0,0 @@
|
||||
//! Security: JWT validation and IP whitelist checking.
|
||||
//!
|
||||
//! Matches Go's security/guard.go and security/jwt.go.
|
||||
//! - Guard: combines whitelist IP checking with JWT token validation
|
||||
//! - JWT: HS256 HMAC signing with file-id claims
|
||||
|
||||
pub mod tls;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::net::IpAddr;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use jsonwebtoken::{decode, encode, Algorithm, DecodingKey, EncodingKey, Header, Validation};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
// ============================================================================
|
||||
// JWT Claims
|
||||
// ============================================================================
|
||||
|
||||
/// Claims for volume server file access tokens.
|
||||
/// Matches Go's `SeaweedFileIdClaims`.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct FileIdClaims {
|
||||
/// File ID this token grants access to (e.g., "3,01637037d6").
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub fid: Option<String>,
|
||||
|
||||
/// Expiration time (Unix timestamp).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub exp: Option<u64>,
|
||||
|
||||
/// Not before (Unix timestamp).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nbf: Option<u64>,
|
||||
}
|
||||
|
||||
/// Signing key wrapper (empty = security disabled).
|
||||
#[derive(Clone)]
|
||||
pub struct SigningKey(pub Vec<u8>);
|
||||
|
||||
impl SigningKey {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
pub fn from_string(s: &str) -> Self {
|
||||
SigningKey(s.as_bytes().to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a JWT token for file access.
|
||||
pub fn gen_jwt(
|
||||
signing_key: &SigningKey,
|
||||
expires_after_sec: i64,
|
||||
file_id: &str,
|
||||
) -> Result<String, JwtError> {
|
||||
if signing_key.is_empty() {
|
||||
return Err(JwtError::NoSigningKey);
|
||||
}
|
||||
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
let claims = FileIdClaims {
|
||||
fid: Some(file_id.to_string()),
|
||||
exp: if expires_after_sec > 0 {
|
||||
Some(now + expires_after_sec as u64)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
nbf: None,
|
||||
};
|
||||
|
||||
let token = encode(
|
||||
&Header::new(Algorithm::HS256),
|
||||
&claims,
|
||||
&EncodingKey::from_secret(&signing_key.0),
|
||||
)?;
|
||||
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// Decode and validate a JWT token.
|
||||
pub fn decode_jwt(signing_key: &SigningKey, token: &str) -> Result<FileIdClaims, JwtError> {
|
||||
if signing_key.is_empty() {
|
||||
return Err(JwtError::NoSigningKey);
|
||||
}
|
||||
|
||||
let mut validation = Validation::new(Algorithm::HS256);
|
||||
// Match Go behavior: tokens without exp are accepted (Go's jwt-go does not require exp)
|
||||
// But if exp IS present, it must be valid (not expired).
|
||||
validation.required_spec_claims.clear();
|
||||
validation.validate_exp = true;
|
||||
// Go's jwt-go/v5 validates nbf when present
|
||||
validation.validate_nbf = true;
|
||||
validation.leeway = 0;
|
||||
|
||||
let data = decode::<FileIdClaims>(
|
||||
token,
|
||||
&DecodingKey::from_secret(&signing_key.0),
|
||||
&validation,
|
||||
)?;
|
||||
|
||||
Ok(data.claims)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Guard
|
||||
// ============================================================================
|
||||
|
||||
/// Security guard: IP whitelist + JWT token validation.
|
||||
pub struct Guard {
|
||||
whitelist_ips: HashSet<String>,
|
||||
whitelist_cidrs: Vec<(IpAddr, u8)>, // (network, prefix_len)
|
||||
pub signing_key: SigningKey,
|
||||
pub expires_after_sec: i64,
|
||||
pub read_signing_key: SigningKey,
|
||||
pub read_expires_after_sec: i64,
|
||||
/// Combined flag: true when whitelist is non-empty OR signing key is present.
|
||||
/// Matches Go's `isWriteActive = !isEmptyWhiteList || len(SigningKey) != 0`.
|
||||
is_write_active: bool,
|
||||
}
|
||||
|
||||
impl Guard {
|
||||
pub fn new(
|
||||
whitelist: &[String],
|
||||
signing_key: SigningKey,
|
||||
expires_after_sec: i64,
|
||||
read_signing_key: SigningKey,
|
||||
read_expires_after_sec: i64,
|
||||
) -> Self {
|
||||
let mut guard = Guard {
|
||||
whitelist_ips: HashSet::new(),
|
||||
whitelist_cidrs: Vec::new(),
|
||||
signing_key,
|
||||
expires_after_sec,
|
||||
read_signing_key,
|
||||
read_expires_after_sec,
|
||||
is_write_active: false,
|
||||
};
|
||||
guard.update_whitelist(whitelist);
|
||||
guard
|
||||
}
|
||||
|
||||
/// Update the IP whitelist.
|
||||
pub fn update_whitelist(&mut self, entries: &[String]) {
|
||||
self.whitelist_ips.clear();
|
||||
self.whitelist_cidrs.clear();
|
||||
|
||||
for entry in entries {
|
||||
let entry = entry.trim();
|
||||
if entry.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if entry.contains('/') {
|
||||
// CIDR range
|
||||
if let Some((ip, prefix)) = parse_cidr(entry) {
|
||||
self.whitelist_cidrs.push((ip, prefix));
|
||||
} else {
|
||||
tracing::error!("Parse CIDR {} in whitelist failed", entry);
|
||||
}
|
||||
} else {
|
||||
// Exact IP/hostname
|
||||
self.whitelist_ips.insert(entry.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Match Go: isWriteActive = !isEmptyWhiteList || len(SigningKey) != 0
|
||||
let is_empty_whitelist = self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty();
|
||||
self.is_write_active = !is_empty_whitelist || !self.signing_key.is_empty();
|
||||
}
|
||||
|
||||
/// Check if a remote IP is in the whitelist.
|
||||
/// Returns true if write security is inactive (no whitelist and no signing key),
|
||||
/// if the whitelist is empty, or if the IP matches.
|
||||
pub fn check_whitelist(&self, remote_addr: &str) -> bool {
|
||||
if !self.is_write_active {
|
||||
return true;
|
||||
}
|
||||
if self.whitelist_ips.is_empty() && self.whitelist_cidrs.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let host = extract_host(remote_addr);
|
||||
|
||||
// Check exact match
|
||||
if self.whitelist_ips.contains(&host) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check CIDR ranges
|
||||
if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
for &(ref network, prefix_len) in &self.whitelist_cidrs {
|
||||
if ip_in_cidr(&ip, network, prefix_len) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if a read signing key is configured.
|
||||
pub fn has_read_signing_key(&self) -> bool {
|
||||
!self.read_signing_key.is_empty()
|
||||
}
|
||||
|
||||
/// Validate a request's JWT token.
|
||||
/// `is_write` determines which signing key to use.
|
||||
/// Returns Ok(()) if valid, or if security is disabled.
|
||||
pub fn check_jwt(&self, token: Option<&str>, is_write: bool) -> Result<(), JwtError> {
|
||||
let key = if is_write {
|
||||
&self.signing_key
|
||||
} else {
|
||||
&self.read_signing_key
|
||||
};
|
||||
|
||||
if key.is_empty() {
|
||||
return Ok(()); // Security disabled for this operation type
|
||||
}
|
||||
|
||||
let token = token.ok_or(JwtError::MissingToken)?;
|
||||
decode_jwt(key, token)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check JWT and validate the file ID claim matches.
|
||||
pub fn check_jwt_for_file(
|
||||
&self,
|
||||
token: Option<&str>,
|
||||
expected_fid: &str,
|
||||
is_write: bool,
|
||||
) -> Result<(), JwtError> {
|
||||
let key = if is_write {
|
||||
&self.signing_key
|
||||
} else {
|
||||
&self.read_signing_key
|
||||
};
|
||||
|
||||
if key.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let token = token.ok_or(JwtError::MissingToken)?;
|
||||
let claims = decode_jwt(key, token)?;
|
||||
|
||||
match claims.fid {
|
||||
None => {
|
||||
return Err(JwtError::MissingFileIdClaim);
|
||||
}
|
||||
Some(ref fid) if fid != expected_fid => {
|
||||
return Err(JwtError::FileIdMismatch {
|
||||
expected: expected_fid.to_string(),
|
||||
got: fid.to_string(),
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helpers
|
||||
// ============================================================================
|
||||
|
||||
/// Extract host from "host:port" or "[::1]:port" format.
|
||||
fn extract_host(addr: &str) -> String {
|
||||
// Handle IPv6 with brackets
|
||||
if addr.starts_with('[') {
|
||||
if let Some(end) = addr.find(']') {
|
||||
return addr[1..end].to_string();
|
||||
}
|
||||
}
|
||||
// Handle host:port
|
||||
if let Some(pos) = addr.rfind(':') {
|
||||
return addr[..pos].to_string();
|
||||
}
|
||||
addr.to_string()
|
||||
}
|
||||
|
||||
/// Parse CIDR notation "192.168.1.0/24" into (IpAddr, prefix_len).
|
||||
fn parse_cidr(cidr: &str) -> Option<(IpAddr, u8)> {
|
||||
let parts: Vec<&str> = cidr.split('/').collect();
|
||||
if parts.len() != 2 {
|
||||
return None;
|
||||
}
|
||||
let ip: IpAddr = parts[0].parse().ok()?;
|
||||
let prefix: u8 = parts[1].parse().ok()?;
|
||||
Some((ip, prefix))
|
||||
}
|
||||
|
||||
/// Check if an IP is within a CIDR range.
|
||||
fn ip_in_cidr(ip: &IpAddr, network: &IpAddr, prefix_len: u8) -> bool {
|
||||
match (ip, network) {
|
||||
(IpAddr::V4(ip), IpAddr::V4(net)) => {
|
||||
let ip_bits = u32::from(*ip);
|
||||
let net_bits = u32::from(*net);
|
||||
let mask = if prefix_len == 0 {
|
||||
0
|
||||
} else if prefix_len >= 32 {
|
||||
u32::MAX
|
||||
} else {
|
||||
u32::MAX << (32 - prefix_len)
|
||||
};
|
||||
(ip_bits & mask) == (net_bits & mask)
|
||||
}
|
||||
(IpAddr::V6(ip), IpAddr::V6(net)) => {
|
||||
let ip_bits = u128::from(*ip);
|
||||
let net_bits = u128::from(*net);
|
||||
let mask = if prefix_len == 0 {
|
||||
0
|
||||
} else if prefix_len >= 128 {
|
||||
u128::MAX
|
||||
} else {
|
||||
u128::MAX << (128 - prefix_len)
|
||||
};
|
||||
(ip_bits & mask) == (net_bits & mask)
|
||||
}
|
||||
_ => false, // V4/V6 mismatch
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Errors
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum JwtError {
|
||||
#[error("no signing key configured")]
|
||||
NoSigningKey,
|
||||
|
||||
#[error("missing JWT token")]
|
||||
MissingToken,
|
||||
|
||||
#[error("JWT error: {0}")]
|
||||
Jwt(#[from] jsonwebtoken::errors::Error),
|
||||
|
||||
#[error("JWT token missing required fid claim")]
|
||||
MissingFileIdClaim,
|
||||
|
||||
#[error("file ID mismatch: expected {expected}, got {got}")]
|
||||
FileIdMismatch { expected: String, got: String },
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_jwt_round_trip() {
|
||||
let key = SigningKey::from_string("test-secret-key");
|
||||
let token = gen_jwt(&key, 3600, "3,01637037d6").unwrap();
|
||||
let claims = decode_jwt(&key, &token).unwrap();
|
||||
assert_eq!(claims.fid, Some("3,01637037d6".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jwt_no_signing_key() {
|
||||
let key = SigningKey(vec![]);
|
||||
assert!(gen_jwt(&key, 3600, "1,abc").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jwt_invalid_token() {
|
||||
let key = SigningKey::from_string("secret");
|
||||
let result = decode_jwt(&key, "invalid.token.here");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jwt_wrong_key() {
|
||||
let key1 = SigningKey::from_string("secret1");
|
||||
let key2 = SigningKey::from_string("secret2");
|
||||
let token = gen_jwt(&key1, 3600, "1,abc").unwrap();
|
||||
assert!(decode_jwt(&key2, &token).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_empty_whitelist() {
|
||||
let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
assert!(guard.check_whitelist("192.168.1.1:8080"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_whitelist_exact() {
|
||||
let guard = Guard::new(
|
||||
&["192.168.1.1".to_string(), "10.0.0.1".to_string()],
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
);
|
||||
assert!(guard.check_whitelist("192.168.1.1:8080"));
|
||||
assert!(guard.check_whitelist("10.0.0.1:1234"));
|
||||
assert!(!guard.check_whitelist("172.16.0.1:8080"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_whitelist_cidr() {
|
||||
let guard = Guard::new(
|
||||
&["10.0.0.0/8".to_string()],
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
);
|
||||
assert!(guard.check_whitelist("10.1.2.3:8080"));
|
||||
assert!(guard.check_whitelist("10.255.255.255:80"));
|
||||
assert!(!guard.check_whitelist("11.0.0.1:80"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_check_jwt_disabled() {
|
||||
let guard = Guard::new(&[], SigningKey(vec![]), 0, SigningKey(vec![]), 0);
|
||||
// No signing key = security disabled
|
||||
assert!(guard.check_jwt(None, true).is_ok());
|
||||
assert!(guard.check_jwt(None, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_check_jwt_enabled() {
|
||||
let key = SigningKey::from_string("write-secret");
|
||||
let read_key = SigningKey::from_string("read-secret");
|
||||
let guard = Guard::new(&[], key.clone(), 3600, read_key.clone(), 3600);
|
||||
|
||||
// Missing token
|
||||
assert!(guard.check_jwt(None, true).is_err());
|
||||
|
||||
// Valid write token
|
||||
let token = gen_jwt(&key, 3600, "1,abc").unwrap();
|
||||
assert!(guard.check_jwt(Some(&token), true).is_ok());
|
||||
|
||||
// Write token for read should fail (different key)
|
||||
assert!(guard.check_jwt(Some(&token), false).is_err());
|
||||
|
||||
// Valid read token
|
||||
let read_token = gen_jwt(&read_key, 3600, "1,abc").unwrap();
|
||||
assert!(guard.check_jwt(Some(&read_token), false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_guard_check_jwt_file_id() {
|
||||
let key = SigningKey::from_string("secret");
|
||||
let guard = Guard::new(&[], key.clone(), 3600, SigningKey(vec![]), 0);
|
||||
|
||||
let token = gen_jwt(&key, 3600, "3,01637037d6").unwrap();
|
||||
|
||||
// Correct file ID
|
||||
assert!(guard
|
||||
.check_jwt_for_file(Some(&token), "3,01637037d6", true)
|
||||
.is_ok());
|
||||
|
||||
// Wrong file ID
|
||||
let err = guard.check_jwt_for_file(Some(&token), "4,deadbeef", true);
|
||||
assert!(matches!(err, Err(JwtError::FileIdMismatch { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_host() {
|
||||
assert_eq!(extract_host("192.168.1.1:8080"), "192.168.1.1");
|
||||
assert_eq!(extract_host("[::1]:8080"), "::1");
|
||||
assert_eq!(extract_host("localhost"), "localhost");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ip_in_cidr() {
|
||||
let net: IpAddr = "10.0.0.0".parse().unwrap();
|
||||
let ip1: IpAddr = "10.1.2.3".parse().unwrap();
|
||||
let ip2: IpAddr = "11.0.0.1".parse().unwrap();
|
||||
assert!(ip_in_cidr(&ip1, &net, 8));
|
||||
assert!(!ip_in_cidr(&ip2, &net, 8));
|
||||
}
|
||||
}
|
||||
@@ -1,437 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustls::client::danger::HandshakeSignatureValid;
|
||||
use rustls::crypto::aws_lc_rs;
|
||||
use rustls::crypto::CryptoProvider;
|
||||
use rustls::pki_types::UnixTime;
|
||||
use rustls::pki_types::{CertificateDer, PrivateKeyDer};
|
||||
use rustls::server::danger::{ClientCertVerified, ClientCertVerifier};
|
||||
use rustls::server::WebPkiClientVerifier;
|
||||
use rustls::{
|
||||
CipherSuite, DigitallySignedStruct, DistinguishedName, RootCertStore, ServerConfig,
|
||||
SignatureScheme, SupportedCipherSuite, SupportedProtocolVersion,
|
||||
};
|
||||
use x509_parser::prelude::{FromDer, X509Certificate};
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct TlsPolicy {
|
||||
pub min_version: String,
|
||||
pub max_version: String,
|
||||
pub cipher_suites: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct GrpcClientAuthPolicy {
|
||||
pub allowed_common_names: Vec<String>,
|
||||
pub allowed_wildcard_domain: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TlsPolicyError(String);
|
||||
|
||||
impl fmt::Display for TlsPolicyError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TlsPolicyError {}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum GoTlsVersion {
|
||||
Ssl3,
|
||||
Tls10,
|
||||
Tls11,
|
||||
Tls12,
|
||||
Tls13,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CommonNameVerifier {
|
||||
inner: Arc<dyn ClientCertVerifier>,
|
||||
allowed_common_names: HashSet<String>,
|
||||
allowed_wildcard_domain: String,
|
||||
}
|
||||
|
||||
impl ClientCertVerifier for CommonNameVerifier {
|
||||
fn offer_client_auth(&self) -> bool {
|
||||
self.inner.offer_client_auth()
|
||||
}
|
||||
|
||||
fn client_auth_mandatory(&self) -> bool {
|
||||
self.inner.client_auth_mandatory()
|
||||
}
|
||||
|
||||
fn root_hint_subjects(&self) -> &[DistinguishedName] {
|
||||
self.inner.root_hint_subjects()
|
||||
}
|
||||
|
||||
fn verify_client_cert(
|
||||
&self,
|
||||
end_entity: &CertificateDer<'_>,
|
||||
intermediates: &[CertificateDer<'_>],
|
||||
now: UnixTime,
|
||||
) -> Result<ClientCertVerified, rustls::Error> {
|
||||
self.inner
|
||||
.verify_client_cert(end_entity, intermediates, now)?;
|
||||
let common_name = parse_common_name(end_entity).map_err(|e| {
|
||||
rustls::Error::General(format!(
|
||||
"parse client certificate common name failed: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
if common_name_is_allowed(
|
||||
&common_name,
|
||||
&self.allowed_common_names,
|
||||
&self.allowed_wildcard_domain,
|
||||
) {
|
||||
return Ok(ClientCertVerified::assertion());
|
||||
}
|
||||
Err(rustls::Error::General(format!(
|
||||
"Authenticate: invalid subject client common name: {}",
|
||||
common_name
|
||||
)))
|
||||
}
|
||||
|
||||
fn verify_tls12_signature(
|
||||
&self,
|
||||
message: &[u8],
|
||||
cert: &CertificateDer<'_>,
|
||||
dss: &DigitallySignedStruct,
|
||||
) -> Result<HandshakeSignatureValid, rustls::Error> {
|
||||
self.inner.verify_tls12_signature(message, cert, dss)
|
||||
}
|
||||
|
||||
fn verify_tls13_signature(
|
||||
&self,
|
||||
message: &[u8],
|
||||
cert: &CertificateDer<'_>,
|
||||
dss: &DigitallySignedStruct,
|
||||
) -> Result<HandshakeSignatureValid, rustls::Error> {
|
||||
self.inner.verify_tls13_signature(message, cert, dss)
|
||||
}
|
||||
|
||||
fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
|
||||
self.inner.supported_verify_schemes()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_rustls_server_config(
|
||||
cert_path: &str,
|
||||
key_path: &str,
|
||||
ca_path: &str,
|
||||
policy: &TlsPolicy,
|
||||
) -> Result<ServerConfig, TlsPolicyError> {
|
||||
build_rustls_server_config_with_client_auth(cert_path, key_path, ca_path, policy, None)
|
||||
}
|
||||
|
||||
pub fn build_rustls_server_config_with_grpc_client_auth(
|
||||
cert_path: &str,
|
||||
key_path: &str,
|
||||
ca_path: &str,
|
||||
policy: &TlsPolicy,
|
||||
client_auth_policy: &GrpcClientAuthPolicy,
|
||||
) -> Result<ServerConfig, TlsPolicyError> {
|
||||
build_rustls_server_config_with_client_auth(
|
||||
cert_path,
|
||||
key_path,
|
||||
ca_path,
|
||||
policy,
|
||||
Some(client_auth_policy),
|
||||
)
|
||||
}
|
||||
|
||||
fn build_rustls_server_config_with_client_auth(
|
||||
cert_path: &str,
|
||||
key_path: &str,
|
||||
ca_path: &str,
|
||||
policy: &TlsPolicy,
|
||||
client_auth_policy: Option<&GrpcClientAuthPolicy>,
|
||||
) -> Result<ServerConfig, TlsPolicyError> {
|
||||
let cert_chain = read_cert_chain(cert_path)?;
|
||||
let private_key = read_private_key(key_path)?;
|
||||
let provider = build_crypto_provider(policy)?;
|
||||
let versions = build_supported_versions(policy)?;
|
||||
|
||||
let builder = ServerConfig::builder_with_provider(provider.clone())
|
||||
.with_protocol_versions(&versions)
|
||||
.map_err(|e| TlsPolicyError(format!("invalid TLS version policy: {}", e)))?;
|
||||
|
||||
let builder = if ca_path.is_empty() {
|
||||
builder.with_no_client_auth()
|
||||
} else {
|
||||
let roots = read_root_store(ca_path)?;
|
||||
let verifier =
|
||||
WebPkiClientVerifier::builder_with_provider(Arc::new(roots), provider.clone())
|
||||
.build()
|
||||
.map_err(|e| TlsPolicyError(format!("build client verifier failed: {}", e)))?;
|
||||
let verifier: Arc<dyn ClientCertVerifier> = if let Some(client_auth_policy) =
|
||||
client_auth_policy.filter(|policy| {
|
||||
!policy.allowed_common_names.is_empty()
|
||||
|| !policy.allowed_wildcard_domain.is_empty()
|
||||
}) {
|
||||
Arc::new(CommonNameVerifier {
|
||||
inner: verifier,
|
||||
allowed_common_names: client_auth_policy
|
||||
.allowed_common_names
|
||||
.iter()
|
||||
.cloned()
|
||||
.collect(),
|
||||
allowed_wildcard_domain: client_auth_policy.allowed_wildcard_domain.clone(),
|
||||
})
|
||||
} else {
|
||||
verifier
|
||||
};
|
||||
builder.with_client_cert_verifier(verifier)
|
||||
};
|
||||
|
||||
builder
|
||||
.with_single_cert(cert_chain, private_key)
|
||||
.map_err(|e| TlsPolicyError(format!("build rustls server config failed: {}", e)))
|
||||
}
|
||||
|
||||
fn read_cert_chain(cert_path: &str) -> Result<Vec<CertificateDer<'static>>, TlsPolicyError> {
|
||||
let cert_pem = std::fs::read(cert_path).map_err(|e| {
|
||||
TlsPolicyError(format!(
|
||||
"Failed to read TLS cert file '{}': {}",
|
||||
cert_path, e
|
||||
))
|
||||
})?;
|
||||
rustls_pemfile::certs(&mut &cert_pem[..])
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| {
|
||||
TlsPolicyError(format!(
|
||||
"Failed to parse TLS cert PEM '{}': {}",
|
||||
cert_path, e
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn read_private_key(key_path: &str) -> Result<PrivateKeyDer<'static>, TlsPolicyError> {
|
||||
let key_pem = std::fs::read(key_path).map_err(|e| {
|
||||
TlsPolicyError(format!("Failed to read TLS key file '{}': {}", key_path, e))
|
||||
})?;
|
||||
rustls_pemfile::private_key(&mut &key_pem[..])
|
||||
.map_err(|e| TlsPolicyError(format!("Failed to parse TLS key PEM '{}': {}", key_path, e)))?
|
||||
.ok_or_else(|| TlsPolicyError(format!("No private key found in '{}'", key_path)))
|
||||
}
|
||||
|
||||
fn read_root_store(ca_path: &str) -> Result<RootCertStore, TlsPolicyError> {
|
||||
let ca_pem = std::fs::read(ca_path)
|
||||
.map_err(|e| TlsPolicyError(format!("Failed to read TLS CA file '{}': {}", ca_path, e)))?;
|
||||
let ca_certs = rustls_pemfile::certs(&mut &ca_pem[..])
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| TlsPolicyError(format!("Failed to parse TLS CA PEM '{}': {}", ca_path, e)))?;
|
||||
let mut roots = RootCertStore::empty();
|
||||
for cert in ca_certs {
|
||||
roots
|
||||
.add(cert)
|
||||
.map_err(|e| TlsPolicyError(format!("Failed to add CA cert '{}': {}", ca_path, e)))?;
|
||||
}
|
||||
Ok(roots)
|
||||
}
|
||||
|
||||
fn build_crypto_provider(policy: &TlsPolicy) -> Result<Arc<CryptoProvider>, TlsPolicyError> {
|
||||
let mut provider = aws_lc_rs::default_provider();
|
||||
let cipher_suites = parse_cipher_suites(&provider.cipher_suites, &policy.cipher_suites)?;
|
||||
if !cipher_suites.is_empty() {
|
||||
provider.cipher_suites = cipher_suites;
|
||||
}
|
||||
Ok(Arc::new(provider))
|
||||
}
|
||||
|
||||
pub fn build_supported_versions(
|
||||
policy: &TlsPolicy,
|
||||
) -> Result<Vec<&'static SupportedProtocolVersion>, TlsPolicyError> {
|
||||
let min_version = parse_go_tls_version(&policy.min_version)?;
|
||||
let max_version = parse_go_tls_version(&policy.max_version)?;
|
||||
let versions = [&rustls::version::TLS13, &rustls::version::TLS12]
|
||||
.into_iter()
|
||||
.filter(|version| {
|
||||
let current = go_tls_version_for_supported(version);
|
||||
min_version.map(|min| current >= min).unwrap_or(true)
|
||||
&& max_version.map(|max| current <= max).unwrap_or(true)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if versions.is_empty() {
|
||||
return Err(TlsPolicyError(format!(
|
||||
"TLS version range min='{}' max='{}' is unsupported by rustls",
|
||||
policy.min_version, policy.max_version
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(versions)
|
||||
}
|
||||
|
||||
fn parse_go_tls_version(value: &str) -> Result<Option<GoTlsVersion>, TlsPolicyError> {
|
||||
match value.trim() {
|
||||
"" => Ok(None),
|
||||
"SSLv3" => Ok(Some(GoTlsVersion::Ssl3)),
|
||||
"TLS 1.0" => Ok(Some(GoTlsVersion::Tls10)),
|
||||
"TLS 1.1" => Ok(Some(GoTlsVersion::Tls11)),
|
||||
"TLS 1.2" => Ok(Some(GoTlsVersion::Tls12)),
|
||||
"TLS 1.3" => Ok(Some(GoTlsVersion::Tls13)),
|
||||
other => Err(TlsPolicyError(format!("invalid TLS version {}", other))),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_cipher_suites(
|
||||
available: &[SupportedCipherSuite],
|
||||
value: &str,
|
||||
) -> Result<Vec<SupportedCipherSuite>, TlsPolicyError> {
|
||||
let trimmed = value.trim();
|
||||
if trimmed.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
trimmed
|
||||
.split(',')
|
||||
.map(|name| {
|
||||
let suite = parse_cipher_suite_name(name.trim())?;
|
||||
available
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|candidate| candidate.suite() == suite)
|
||||
.ok_or_else(|| {
|
||||
TlsPolicyError(format!(
|
||||
"TLS cipher suite '{}' is unsupported by the Rust implementation",
|
||||
name.trim()
|
||||
))
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse_cipher_suite_name(value: &str) -> Result<CipherSuite, TlsPolicyError> {
|
||||
match value {
|
||||
"TLS_AES_128_GCM_SHA256" => Ok(CipherSuite::TLS13_AES_128_GCM_SHA256),
|
||||
"TLS_AES_256_GCM_SHA384" => Ok(CipherSuite::TLS13_AES_256_GCM_SHA384),
|
||||
"TLS_CHACHA20_POLY1305_SHA256" => Ok(CipherSuite::TLS13_CHACHA20_POLY1305_SHA256),
|
||||
"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" => {
|
||||
Ok(CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256)
|
||||
}
|
||||
"TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" => {
|
||||
Ok(CipherSuite::TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
|
||||
}
|
||||
"TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" => {
|
||||
Ok(CipherSuite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256)
|
||||
}
|
||||
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" => {
|
||||
Ok(CipherSuite::TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256)
|
||||
}
|
||||
"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" => {
|
||||
Ok(CipherSuite::TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
|
||||
}
|
||||
"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" => {
|
||||
Ok(CipherSuite::TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256)
|
||||
}
|
||||
other => Err(TlsPolicyError(format!(
|
||||
"TLS cipher suite '{}' is unsupported by the Rust implementation",
|
||||
other
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_common_name(cert: &CertificateDer<'_>) -> Result<String, TlsPolicyError> {
|
||||
let (_, certificate) = X509Certificate::from_der(cert.as_ref())
|
||||
.map_err(|e| TlsPolicyError(format!("parse X.509 certificate failed: {}", e)))?;
|
||||
let common_name = certificate
|
||||
.subject()
|
||||
.iter_common_name()
|
||||
.next()
|
||||
.and_then(|common_name| common_name.as_str().ok())
|
||||
.map(str::to_string);
|
||||
match common_name {
|
||||
Some(common_name) => Ok(common_name),
|
||||
None => Ok(String::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn common_name_is_allowed(
|
||||
common_name: &str,
|
||||
allowed_common_names: &HashSet<String>,
|
||||
allowed_wildcard_domain: &str,
|
||||
) -> bool {
|
||||
(!allowed_wildcard_domain.is_empty() && common_name.ends_with(allowed_wildcard_domain))
|
||||
|| allowed_common_names.contains(common_name)
|
||||
}
|
||||
|
||||
fn go_tls_version_for_supported(version: &SupportedProtocolVersion) -> GoTlsVersion {
|
||||
match version.version {
|
||||
rustls::ProtocolVersion::TLSv1_2 => GoTlsVersion::Tls12,
|
||||
rustls::ProtocolVersion::TLSv1_3 => GoTlsVersion::Tls13,
|
||||
_ => unreachable!("rustls only exposes TLS 1.2 and 1.3"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_supported_versions, common_name_is_allowed, parse_cipher_suites, TlsPolicy};
|
||||
use rustls::crypto::aws_lc_rs;
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn test_build_supported_versions_defaults_to_tls12_and_tls13() {
|
||||
let versions = build_supported_versions(&TlsPolicy::default()).unwrap();
|
||||
assert_eq!(
|
||||
versions,
|
||||
vec![&rustls::version::TLS13, &rustls::version::TLS12]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_supported_versions_filters_to_tls13() {
|
||||
let versions = build_supported_versions(&TlsPolicy {
|
||||
min_version: "TLS 1.3".to_string(),
|
||||
max_version: "TLS 1.3".to_string(),
|
||||
cipher_suites: String::new(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(versions, vec![&rustls::version::TLS13]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_supported_versions_rejects_unsupported_legacy_range() {
|
||||
let err = build_supported_versions(&TlsPolicy {
|
||||
min_version: "TLS 1.0".to_string(),
|
||||
max_version: "TLS 1.1".to_string(),
|
||||
cipher_suites: String::new(),
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("unsupported by rustls"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_cipher_suites_accepts_go_names() {
|
||||
let cipher_suites = parse_cipher_suites(
|
||||
&aws_lc_rs::default_provider().cipher_suites,
|
||||
"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_AES_128_GCM_SHA256",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(cipher_suites.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_common_name_is_allowed_matches_exact_and_wildcard() {
|
||||
let allowed_common_names =
|
||||
HashSet::from([String::from("volume-a.internal"), String::from("worker-7")]);
|
||||
assert!(common_name_is_allowed(
|
||||
"volume-a.internal",
|
||||
&allowed_common_names,
|
||||
"",
|
||||
));
|
||||
assert!(common_name_is_allowed(
|
||||
"node.prod.example.com",
|
||||
&allowed_common_names,
|
||||
".example.com",
|
||||
));
|
||||
assert!(!common_name_is_allowed(
|
||||
"node.prod.other.net",
|
||||
&allowed_common_names,
|
||||
".example.com",
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
use axum::body::Body;
|
||||
use axum::extract::Query;
|
||||
use axum::http::{header, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::{any, get};
|
||||
use axum::Router;
|
||||
use pprof::protos::Message;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct ProfileQuery {
|
||||
seconds: Option<u64>,
|
||||
}
|
||||
|
||||
pub fn build_debug_router() -> Router {
|
||||
Router::new()
|
||||
.route("/debug/pprof/", get(pprof_index_handler))
|
||||
.route("/debug/pprof/cmdline", get(pprof_cmdline_handler))
|
||||
.route("/debug/pprof/profile", get(pprof_profile_handler))
|
||||
.route("/debug/pprof/symbol", any(pprof_symbol_handler))
|
||||
.route("/debug/pprof/trace", get(pprof_trace_handler))
|
||||
}
|
||||
|
||||
async fn pprof_index_handler() -> Response {
|
||||
let body = concat!(
|
||||
"<html><head><title>/debug/pprof/</title></head><body>",
|
||||
"<a href=\"cmdline\">cmdline</a><br>",
|
||||
"<a href=\"profile\">profile</a><br>",
|
||||
"<a href=\"symbol\">symbol</a><br>",
|
||||
"<a href=\"trace\">trace</a><br>",
|
||||
"</body></html>",
|
||||
);
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "text/html; charset=utf-8")],
|
||||
body,
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn pprof_cmdline_handler() -> Response {
|
||||
let body = std::env::args().collect::<Vec<_>>().join("\0");
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
|
||||
body,
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn pprof_profile_handler(Query(query): Query<ProfileQuery>) -> Response {
|
||||
let seconds = query.seconds.unwrap_or(30).clamp(1, 300);
|
||||
let guard = match pprof::ProfilerGuard::new(100) {
|
||||
Ok(guard) => guard,
|
||||
Err(e) => {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("failed to start profiler: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_secs(seconds)).await;
|
||||
|
||||
let report = match guard.report().build() {
|
||||
Ok(report) => report,
|
||||
Err(e) => {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("failed to build profile report: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
let profile = match report.pprof() {
|
||||
Ok(profile) => profile,
|
||||
Err(e) => {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("failed to encode profile: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
if let Err(e) = profile.encode(&mut bytes) {
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("failed to serialize profile: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "application/octet-stream")],
|
||||
bytes,
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn pprof_symbol_handler() -> Response {
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
|
||||
"num_symbols: 0\n",
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn pprof_trace_handler(Query(query): Query<ProfileQuery>) -> Response {
|
||||
let seconds = query.seconds.unwrap_or(1).clamp(1, 30);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(seconds)).await;
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, "application/octet-stream")
|
||||
.body(Body::from(Vec::<u8>::new()))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::http::Request;
|
||||
use tower::ServiceExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_debug_index_route() {
|
||||
let app = build_debug_router();
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/debug/pprof/")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_debug_cmdline_route() {
|
||||
let app = build_debug_router();
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/debug/pprof/cmdline")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.6 KiB |
@@ -1,206 +0,0 @@
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
use std::time::Duration;
|
||||
|
||||
use hyper::http::Uri;
|
||||
use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity};
|
||||
|
||||
use crate::config::VolumeServerConfig;
|
||||
|
||||
pub const GRPC_MAX_MESSAGE_SIZE: usize = 1 << 30;
|
||||
const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
const GRPC_INITIAL_WINDOW_SIZE: u32 = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct OutgoingGrpcTlsConfig {
|
||||
cert_pem: String,
|
||||
key_pem: String,
|
||||
ca_pem: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct GrpcClientError(String);
|
||||
|
||||
impl fmt::Display for GrpcClientError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for GrpcClientError {}
|
||||
|
||||
pub fn load_outgoing_grpc_tls(
|
||||
config: &VolumeServerConfig,
|
||||
) -> Result<Option<OutgoingGrpcTlsConfig>, GrpcClientError> {
|
||||
if config.grpc_cert_file.is_empty()
|
||||
|| config.grpc_key_file.is_empty()
|
||||
|| config.grpc_ca_file.is_empty()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let cert_pem = std::fs::read_to_string(&config.grpc_cert_file).map_err(|e| {
|
||||
GrpcClientError(format!(
|
||||
"Failed to read outgoing gRPC cert '{}': {}",
|
||||
config.grpc_cert_file, e
|
||||
))
|
||||
})?;
|
||||
let key_pem = std::fs::read_to_string(&config.grpc_key_file).map_err(|e| {
|
||||
GrpcClientError(format!(
|
||||
"Failed to read outgoing gRPC key '{}': {}",
|
||||
config.grpc_key_file, e
|
||||
))
|
||||
})?;
|
||||
let ca_pem = std::fs::read_to_string(&config.grpc_ca_file).map_err(|e| {
|
||||
GrpcClientError(format!(
|
||||
"Failed to read outgoing gRPC CA '{}': {}",
|
||||
config.grpc_ca_file, e
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok(Some(OutgoingGrpcTlsConfig {
|
||||
cert_pem,
|
||||
key_pem,
|
||||
ca_pem,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn grpc_endpoint_uri(grpc_host_port: &str, tls: Option<&OutgoingGrpcTlsConfig>) -> String {
|
||||
let scheme = if tls.is_some() { "https" } else { "http" };
|
||||
format!("{}://{}", scheme, grpc_host_port)
|
||||
}
|
||||
|
||||
pub fn build_grpc_endpoint(
|
||||
grpc_host_port: &str,
|
||||
tls: Option<&OutgoingGrpcTlsConfig>,
|
||||
) -> Result<Endpoint, GrpcClientError> {
|
||||
let uri = grpc_endpoint_uri(grpc_host_port, tls);
|
||||
let mut endpoint = Channel::from_shared(uri.clone())
|
||||
.map_err(|e| GrpcClientError(format!("invalid gRPC endpoint {}: {}", uri, e)))?
|
||||
.http2_keep_alive_interval(GRPC_KEEPALIVE_INTERVAL)
|
||||
.keep_alive_timeout(GRPC_KEEPALIVE_TIMEOUT)
|
||||
.keep_alive_while_idle(false)
|
||||
.initial_stream_window_size(Some(GRPC_INITIAL_WINDOW_SIZE))
|
||||
.initial_connection_window_size(Some(GRPC_INITIAL_WINDOW_SIZE))
|
||||
.http2_adaptive_window(false);
|
||||
|
||||
if let Some(tls) = tls {
|
||||
let parsed = uri
|
||||
.parse::<Uri>()
|
||||
.map_err(|e| GrpcClientError(format!("invalid gRPC endpoint {}: {}", uri, e)))?;
|
||||
let host = parsed
|
||||
.host()
|
||||
.ok_or_else(|| GrpcClientError(format!("missing host in gRPC endpoint {}", uri)))?;
|
||||
let tls_config = ClientTlsConfig::new()
|
||||
.identity(Identity::from_pem(
|
||||
tls.cert_pem.clone(),
|
||||
tls.key_pem.clone(),
|
||||
))
|
||||
.ca_certificate(Certificate::from_pem(tls.ca_pem.clone()))
|
||||
.domain_name(host.to_string());
|
||||
endpoint = endpoint.tls_config(tls_config).map_err(|e| {
|
||||
GrpcClientError(format!("configure gRPC TLS for {} failed: {}", uri, e))
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(endpoint)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_grpc_endpoint, grpc_endpoint_uri, load_outgoing_grpc_tls};
|
||||
use crate::config::{NeedleMapKind, ReadMode, VolumeServerConfig};
|
||||
use crate::security::tls::TlsPolicy;
|
||||
|
||||
fn sample_config() -> VolumeServerConfig {
|
||||
VolumeServerConfig {
|
||||
port: 8080,
|
||||
grpc_port: 18080,
|
||||
public_port: 8080,
|
||||
ip: "127.0.0.1".to_string(),
|
||||
bind_ip: String::new(),
|
||||
public_url: "127.0.0.1:8080".to_string(),
|
||||
id: String::new(),
|
||||
masters: vec![],
|
||||
pre_stop_seconds: 0,
|
||||
idle_timeout: 0,
|
||||
data_center: String::new(),
|
||||
rack: String::new(),
|
||||
index_type: NeedleMapKind::InMemory,
|
||||
disk_type: String::new(),
|
||||
folders: vec![],
|
||||
folder_max_limits: vec![],
|
||||
folder_tags: vec![],
|
||||
min_free_spaces: vec![],
|
||||
disk_types: vec![],
|
||||
idx_folder: String::new(),
|
||||
white_list: vec![],
|
||||
fix_jpg_orientation: false,
|
||||
read_mode: ReadMode::Local,
|
||||
cpu_profile: String::new(),
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
file_size_limit_bytes: 0,
|
||||
concurrent_upload_limit: 0,
|
||||
concurrent_download_limit: 0,
|
||||
inflight_upload_data_timeout: std::time::Duration::from_secs(0),
|
||||
inflight_download_data_timeout: std::time::Duration::from_secs(0),
|
||||
has_slow_read: false,
|
||||
read_buffer_size_mb: 0,
|
||||
ldb_timeout: 0,
|
||||
pprof: false,
|
||||
metrics_port: 0,
|
||||
metrics_ip: String::new(),
|
||||
debug: false,
|
||||
debug_port: 0,
|
||||
ui_enabled: false,
|
||||
jwt_signing_key: vec![],
|
||||
jwt_signing_expires_seconds: 0,
|
||||
jwt_read_signing_key: vec![],
|
||||
jwt_read_signing_expires_seconds: 0,
|
||||
https_cert_file: String::new(),
|
||||
https_key_file: String::new(),
|
||||
https_ca_file: String::new(),
|
||||
https_client_enabled: false,
|
||||
https_client_cert_file: String::new(),
|
||||
https_client_key_file: String::new(),
|
||||
https_client_ca_file: String::new(),
|
||||
grpc_cert_file: String::new(),
|
||||
grpc_key_file: String::new(),
|
||||
grpc_ca_file: String::new(),
|
||||
grpc_allowed_wildcard_domain: String::new(),
|
||||
grpc_volume_allowed_common_names: vec![],
|
||||
tls_policy: TlsPolicy::default(),
|
||||
enable_write_queue: false,
|
||||
security_file: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_grpc_endpoint_uri_uses_https_when_tls_enabled() {
|
||||
let tls = super::OutgoingGrpcTlsConfig {
|
||||
cert_pem: "cert".to_string(),
|
||||
key_pem: "key".to_string(),
|
||||
ca_pem: "ca".to_string(),
|
||||
};
|
||||
assert_eq!(
|
||||
grpc_endpoint_uri("master.example.com:19333", Some(&tls)),
|
||||
"https://master.example.com:19333"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_outgoing_grpc_tls_requires_cert_key_and_ca() {
|
||||
let mut config = sample_config();
|
||||
config.grpc_cert_file = "/tmp/client.pem".to_string();
|
||||
assert!(load_outgoing_grpc_tls(&config).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_grpc_endpoint_without_tls_uses_http_scheme() {
|
||||
let endpoint = build_grpc_endpoint("127.0.0.1:19333", None).unwrap();
|
||||
assert_eq!(endpoint.uri().scheme_str(), Some("http"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,102 +0,0 @@
|
||||
use crate::pb::volume_server_pb;
|
||||
|
||||
pub fn collect_mem_status() -> volume_server_pb::MemStatus {
|
||||
#[allow(unused_mut)]
|
||||
let mut mem = volume_server_pb::MemStatus {
|
||||
goroutines: 1,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if let Some((all, free)) = get_system_memory_linux() {
|
||||
mem.all = all;
|
||||
mem.free = free;
|
||||
mem.used = all.saturating_sub(free);
|
||||
}
|
||||
|
||||
if let Some(status) = read_process_status_linux() {
|
||||
if status.threads > 0 {
|
||||
mem.goroutines = status.threads as i32;
|
||||
}
|
||||
if let Some(rss) = status.rss {
|
||||
mem.self_ = rss;
|
||||
}
|
||||
if let Some(heap) = status.data.or(status.rss) {
|
||||
mem.heap = heap;
|
||||
}
|
||||
if let Some(stack) = status.stack {
|
||||
mem.stack = stack;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mem
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn get_system_memory_linux() -> Option<(u64, u64)> {
|
||||
unsafe {
|
||||
let mut info: libc::sysinfo = std::mem::zeroed();
|
||||
if libc::sysinfo(&mut info) == 0 {
|
||||
let unit = info.mem_unit as u64;
|
||||
let total = info.totalram as u64 * unit;
|
||||
let free = info.freeram as u64 * unit;
|
||||
return Some((total, free));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[derive(Default)]
|
||||
struct ProcessStatus {
|
||||
threads: u64,
|
||||
rss: Option<u64>,
|
||||
data: Option<u64>,
|
||||
stack: Option<u64>,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn read_process_status_linux() -> Option<ProcessStatus> {
|
||||
let status = std::fs::read_to_string("/proc/self/status").ok()?;
|
||||
let mut out = ProcessStatus::default();
|
||||
|
||||
for line in status.lines() {
|
||||
if let Some(value) = line.strip_prefix("Threads:") {
|
||||
out.threads = value.trim().parse().ok()?;
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = parse_proc_status_kib_field(line, "VmRSS:") {
|
||||
out.rss = Some(value);
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = parse_proc_status_kib_field(line, "VmData:") {
|
||||
out.data = Some(value);
|
||||
continue;
|
||||
}
|
||||
if let Some(value) = parse_proc_status_kib_field(line, "VmStk:") {
|
||||
out.stack = Some(value);
|
||||
}
|
||||
}
|
||||
|
||||
Some(out)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn parse_proc_status_kib_field(line: &str, prefix: &str) -> Option<u64> {
|
||||
let raw = line.strip_prefix(prefix)?.trim();
|
||||
let value = raw.strip_suffix(" kB").unwrap_or(raw).trim();
|
||||
value.parse::<u64>().ok().map(|kib| kib * 1024)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::collect_mem_status;
|
||||
|
||||
#[test]
|
||||
fn test_collect_mem_status_reports_live_process_state() {
|
||||
let mem = collect_mem_status();
|
||||
assert!(mem.goroutines > 0);
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
#[cfg(unix)]
|
||||
pub mod debug;
|
||||
pub mod grpc_client;
|
||||
pub mod grpc_server;
|
||||
pub mod handlers;
|
||||
pub mod heartbeat;
|
||||
pub mod memory_status;
|
||||
#[cfg(unix)]
|
||||
pub mod profiling;
|
||||
pub mod request_id;
|
||||
pub mod server_stats;
|
||||
pub mod ui;
|
||||
pub mod volume_server;
|
||||
pub mod write_queue;
|
||||
@@ -1,187 +0,0 @@
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use pprof::protos::Message;
|
||||
|
||||
use crate::config::VolumeServerConfig;
|
||||
|
||||
const GO_CPU_PROFILE_FREQUENCY: i32 = 100;
|
||||
const GO_PPROF_BLOCKLIST: [&str; 4] = ["libc", "libgcc", "pthread", "vdso"];
|
||||
|
||||
pub struct CpuProfileSession {
|
||||
output_path: PathBuf,
|
||||
guard: pprof::ProfilerGuard<'static>,
|
||||
}
|
||||
|
||||
impl CpuProfileSession {
|
||||
pub fn start(config: &VolumeServerConfig) -> Result<Option<Self>, String> {
|
||||
if config.cpu_profile.is_empty() {
|
||||
if !config.mem_profile.is_empty() && !config.pprof {
|
||||
tracing::warn!(
|
||||
"--memprofile is not yet supported in the Rust volume server; ignoring '{}'",
|
||||
config.mem_profile
|
||||
);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if config.pprof {
|
||||
tracing::info!(
|
||||
"--pprof is enabled; ignoring --cpuprofile '{}' and --memprofile '{}'",
|
||||
config.cpu_profile,
|
||||
config.mem_profile
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if !config.mem_profile.is_empty() {
|
||||
tracing::warn!(
|
||||
"--memprofile is not yet supported in the Rust volume server; only --cpuprofile '{}' will be written",
|
||||
config.cpu_profile
|
||||
);
|
||||
}
|
||||
|
||||
let guard = pprof::ProfilerGuardBuilder::default()
|
||||
.frequency(GO_CPU_PROFILE_FREQUENCY)
|
||||
.blocklist(&GO_PPROF_BLOCKLIST)
|
||||
.build()
|
||||
.map_err(|e| {
|
||||
format!(
|
||||
"Failed to start CPU profiler '{}': {}",
|
||||
config.cpu_profile, e
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(Some(Self {
|
||||
output_path: PathBuf::from(&config.cpu_profile),
|
||||
guard,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn finish(self) -> Result<(), String> {
|
||||
let report = self
|
||||
.guard
|
||||
.report()
|
||||
.build()
|
||||
.map_err(|e| format!("Failed to build CPU profile report: {}", e))?;
|
||||
let profile = report
|
||||
.pprof()
|
||||
.map_err(|e| format!("Failed to encode CPU profile report: {}", e))?;
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
profile
|
||||
.encode(&mut bytes)
|
||||
.map_err(|e| format!("Failed to serialize CPU profile report: {}", e))?;
|
||||
|
||||
let mut file = File::create(&self.output_path).map_err(|e| {
|
||||
format!(
|
||||
"Failed to create CPU profile '{}': {}",
|
||||
self.output_path.display(),
|
||||
e
|
||||
)
|
||||
})?;
|
||||
file.write_all(&bytes).map_err(|e| {
|
||||
format!(
|
||||
"Failed to write CPU profile '{}': {}",
|
||||
self.output_path.display(),
|
||||
e
|
||||
)
|
||||
})?;
|
||||
file.flush().map_err(|e| {
|
||||
format!(
|
||||
"Failed to flush CPU profile '{}': {}",
|
||||
self.output_path.display(),
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::CpuProfileSession;
|
||||
use crate::config::{NeedleMapKind, ReadMode, VolumeServerConfig};
|
||||
use crate::security::tls::TlsPolicy;
|
||||
|
||||
fn sample_config() -> VolumeServerConfig {
|
||||
VolumeServerConfig {
|
||||
port: 8080,
|
||||
grpc_port: 18080,
|
||||
public_port: 8080,
|
||||
ip: "127.0.0.1".to_string(),
|
||||
bind_ip: "127.0.0.1".to_string(),
|
||||
public_url: "127.0.0.1:8080".to_string(),
|
||||
id: "127.0.0.1:8080".to_string(),
|
||||
masters: vec![],
|
||||
pre_stop_seconds: 0,
|
||||
idle_timeout: 0,
|
||||
data_center: String::new(),
|
||||
rack: String::new(),
|
||||
index_type: NeedleMapKind::InMemory,
|
||||
disk_type: String::new(),
|
||||
folders: vec!["/tmp".to_string()],
|
||||
folder_max_limits: vec![8],
|
||||
folder_tags: vec![vec![]],
|
||||
min_free_spaces: vec![],
|
||||
disk_types: vec![String::new()],
|
||||
idx_folder: String::new(),
|
||||
white_list: vec![],
|
||||
fix_jpg_orientation: false,
|
||||
read_mode: ReadMode::Local,
|
||||
cpu_profile: String::new(),
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
file_size_limit_bytes: 0,
|
||||
concurrent_upload_limit: 0,
|
||||
concurrent_download_limit: 0,
|
||||
inflight_upload_data_timeout: std::time::Duration::from_secs(0),
|
||||
inflight_download_data_timeout: std::time::Duration::from_secs(0),
|
||||
has_slow_read: false,
|
||||
read_buffer_size_mb: 4,
|
||||
ldb_timeout: 0,
|
||||
pprof: false,
|
||||
metrics_port: 0,
|
||||
metrics_ip: String::new(),
|
||||
debug: false,
|
||||
debug_port: 0,
|
||||
ui_enabled: false,
|
||||
jwt_signing_key: vec![],
|
||||
jwt_signing_expires_seconds: 0,
|
||||
jwt_read_signing_key: vec![],
|
||||
jwt_read_signing_expires_seconds: 0,
|
||||
https_cert_file: String::new(),
|
||||
https_key_file: String::new(),
|
||||
https_ca_file: String::new(),
|
||||
https_client_enabled: false,
|
||||
https_client_cert_file: String::new(),
|
||||
https_client_key_file: String::new(),
|
||||
https_client_ca_file: String::new(),
|
||||
grpc_cert_file: String::new(),
|
||||
grpc_key_file: String::new(),
|
||||
grpc_ca_file: String::new(),
|
||||
grpc_allowed_wildcard_domain: String::new(),
|
||||
grpc_volume_allowed_common_names: vec![],
|
||||
tls_policy: TlsPolicy::default(),
|
||||
enable_write_queue: false,
|
||||
security_file: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cpu_profile_session_skips_when_disabled() {
|
||||
let config = sample_config();
|
||||
assert!(CpuProfileSession::start(&config).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cpu_profile_session_skips_when_pprof_enabled() {
|
||||
let mut config = sample_config();
|
||||
config.cpu_profile = "/tmp/cpu.pb".to_string();
|
||||
config.pprof = true;
|
||||
assert!(CpuProfileSession::start(&config).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use hyper::http::{self, HeaderValue};
|
||||
use tonic::metadata::MetadataValue;
|
||||
use tonic::{Request, Status};
|
||||
use tower::{Layer, Service};
|
||||
|
||||
tokio::task_local! {
|
||||
static CURRENT_REQUEST_ID: String;
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct GrpcRequestIdLayer;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GrpcRequestIdService<S> {
|
||||
inner: S,
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for GrpcRequestIdLayer {
|
||||
type Service = GrpcRequestIdService<S>;
|
||||
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
GrpcRequestIdService { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, B> Service<http::Request<B>> for GrpcRequestIdService<S>
|
||||
where
|
||||
S: Service<http::Request<B>, Response = http::Response<tonic::body::BoxBody>> + Send + 'static,
|
||||
S::Future: Send + 'static,
|
||||
B: Send + 'static,
|
||||
{
|
||||
type Response = http::Response<tonic::body::BoxBody>;
|
||||
type Error = S::Error;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: http::Request<B>) -> Self::Future {
|
||||
let request_id = match request.headers().get("x-amz-request-id") {
|
||||
Some(value) => match value.to_str() {
|
||||
Ok(value) if !value.is_empty() => value.to_owned(),
|
||||
_ => generate_grpc_request_id(),
|
||||
},
|
||||
None => generate_grpc_request_id(),
|
||||
};
|
||||
|
||||
if let Ok(value) = HeaderValue::from_str(&request_id) {
|
||||
request.headers_mut().insert("x-amz-request-id", value);
|
||||
}
|
||||
|
||||
let future = self.inner.call(request);
|
||||
|
||||
Box::pin(async move {
|
||||
let mut response: http::Response<tonic::body::BoxBody> =
|
||||
scope_request_id(request_id.clone(), future).await?;
|
||||
if let Ok(value) = HeaderValue::from_str(&request_id) {
|
||||
response.headers_mut().insert("x-amz-request-id", value);
|
||||
}
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn scope_request_id<F, T>(request_id: String, future: F) -> T
|
||||
where
|
||||
F: Future<Output = T>,
|
||||
{
|
||||
CURRENT_REQUEST_ID.scope(request_id, future).await
|
||||
}
|
||||
|
||||
pub fn current_request_id() -> Option<String> {
|
||||
CURRENT_REQUEST_ID.try_with(Clone::clone).ok()
|
||||
}
|
||||
|
||||
pub fn outgoing_request_id_interceptor(mut request: Request<()>) -> Result<Request<()>, Status> {
|
||||
if let Some(request_id) = current_request_id() {
|
||||
let value = MetadataValue::try_from(request_id.as_str())
|
||||
.map_err(|_| Status::internal("invalid scoped request id"))?;
|
||||
request.metadata_mut().insert("x-amz-request-id", value);
|
||||
}
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
pub fn generate_http_request_id() -> String {
|
||||
use rand::Rng;
|
||||
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos() as u64;
|
||||
let rand_val: u32 = rand::thread_rng().gen();
|
||||
format!("{:X}{:08X}", nanos, rand_val)
|
||||
}
|
||||
|
||||
fn generate_grpc_request_id() -> String {
|
||||
uuid::Uuid::new_v4().to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{current_request_id, outgoing_request_id_interceptor, scope_request_id};
|
||||
use tonic::Request;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scope_request_id_exposes_current_value() {
|
||||
let request_id = "req-123".to_string();
|
||||
let current = scope_request_id(
|
||||
request_id.clone(),
|
||||
async move { current_request_id().unwrap() },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(current, request_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_outgoing_request_id_interceptor_propagates_scope() {
|
||||
let request = scope_request_id("req-456".to_string(), async move {
|
||||
outgoing_request_id_interceptor(Request::new(())).unwrap()
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-amz-request-id")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap(),
|
||||
"req-456"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
use chrono::{Datelike, Local, Timelike};
|
||||
use serde::Serialize;
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
static START_TIME: LazyLock<Instant> = LazyLock::new(Instant::now);
|
||||
static SERVER_STATS: LazyLock<ServerStats> = LazyLock::new(ServerStats::default);
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ServerStats {
|
||||
inner: Mutex<ServerStatsInner>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ServerStatsInner {
|
||||
requests: DurationCounter,
|
||||
connections: DurationCounter,
|
||||
assign_requests: DurationCounter,
|
||||
read_requests: DurationCounter,
|
||||
write_requests: DurationCounter,
|
||||
delete_requests: DurationCounter,
|
||||
bytes_in: DurationCounter,
|
||||
bytes_out: DurationCounter,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
pub struct ServerStatsSnapshot {
|
||||
pub requests: DurationCounterSnapshot,
|
||||
pub connections: DurationCounterSnapshot,
|
||||
pub assign_requests: DurationCounterSnapshot,
|
||||
pub read_requests: DurationCounterSnapshot,
|
||||
pub write_requests: DurationCounterSnapshot,
|
||||
pub delete_requests: DurationCounterSnapshot,
|
||||
pub bytes_in: DurationCounterSnapshot,
|
||||
pub bytes_out: DurationCounterSnapshot,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
pub struct DurationCounterSnapshot {
|
||||
pub minute_counter: RoundRobinCounterSnapshot,
|
||||
pub hour_counter: RoundRobinCounterSnapshot,
|
||||
pub day_counter: RoundRobinCounterSnapshot,
|
||||
pub week_counter: RoundRobinCounterSnapshot,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
pub struct RoundRobinCounterSnapshot {
|
||||
pub last_index: i32,
|
||||
pub values: Vec<i64>,
|
||||
pub counts: Vec<i64>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct DurationCounter {
|
||||
minute_counter: RoundRobinCounter,
|
||||
hour_counter: RoundRobinCounter,
|
||||
day_counter: RoundRobinCounter,
|
||||
week_counter: RoundRobinCounter,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RoundRobinCounter {
|
||||
last_index: i32,
|
||||
values: Vec<i64>,
|
||||
counts: Vec<i64>,
|
||||
}
|
||||
|
||||
impl Default for DurationCounter {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
minute_counter: RoundRobinCounter::new(60),
|
||||
hour_counter: RoundRobinCounter::new(60),
|
||||
day_counter: RoundRobinCounter::new(24),
|
||||
week_counter: RoundRobinCounter::new(7),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RoundRobinCounter {
|
||||
fn new(slots: usize) -> Self {
|
||||
Self {
|
||||
last_index: -1,
|
||||
values: vec![0; slots],
|
||||
counts: vec![0; slots],
|
||||
}
|
||||
}
|
||||
|
||||
fn add(&mut self, index: usize, val: i64) {
|
||||
if index >= self.values.len() {
|
||||
return;
|
||||
}
|
||||
while self.last_index != index as i32 {
|
||||
self.last_index = (self.last_index + 1).rem_euclid(self.values.len() as i32);
|
||||
self.values[self.last_index as usize] = 0;
|
||||
self.counts[self.last_index as usize] = 0;
|
||||
}
|
||||
self.values[index] += val;
|
||||
self.counts[index] += 1;
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> RoundRobinCounterSnapshot {
|
||||
RoundRobinCounterSnapshot {
|
||||
last_index: self.last_index,
|
||||
values: self.values.clone(),
|
||||
counts: self.counts.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DurationCounter {
|
||||
fn add_now(&mut self, val: i64) {
|
||||
let now = Local::now();
|
||||
self.minute_counter.add(now.second() as usize, val);
|
||||
self.hour_counter.add(now.minute() as usize, val);
|
||||
self.day_counter.add(now.hour() as usize, val);
|
||||
self.week_counter
|
||||
.add(now.weekday().num_days_from_sunday() as usize, val);
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> DurationCounterSnapshot {
|
||||
DurationCounterSnapshot {
|
||||
minute_counter: self.minute_counter.snapshot(),
|
||||
hour_counter: self.hour_counter.snapshot(),
|
||||
day_counter: self.day_counter.snapshot(),
|
||||
week_counter: self.week_counter.snapshot(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ServerStatsInner {
|
||||
fn snapshot(&self) -> ServerStatsSnapshot {
|
||||
ServerStatsSnapshot {
|
||||
requests: self.requests.snapshot(),
|
||||
connections: self.connections.snapshot(),
|
||||
assign_requests: self.assign_requests.snapshot(),
|
||||
read_requests: self.read_requests.snapshot(),
|
||||
write_requests: self.write_requests.snapshot(),
|
||||
delete_requests: self.delete_requests.snapshot(),
|
||||
bytes_in: self.bytes_in.snapshot(),
|
||||
bytes_out: self.bytes_out.snapshot(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ServerStats {
|
||||
fn update<F>(&self, update: F)
|
||||
where
|
||||
F: FnOnce(&mut ServerStatsInner),
|
||||
{
|
||||
let mut inner = self.inner.lock().unwrap();
|
||||
update(&mut inner);
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> ServerStatsSnapshot {
|
||||
self.inner.lock().unwrap().snapshot()
|
||||
}
|
||||
}
|
||||
|
||||
impl RoundRobinCounterSnapshot {
|
||||
pub fn to_list(&self) -> Vec<i64> {
|
||||
if self.values.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut ret = Vec::with_capacity(self.values.len());
|
||||
let mut index = self.last_index;
|
||||
let mut step = self.values.len();
|
||||
while step > 0 {
|
||||
step -= 1;
|
||||
index += 1;
|
||||
if index >= self.values.len() as i32 {
|
||||
index = 0;
|
||||
}
|
||||
ret.push(self.values[index as usize]);
|
||||
}
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub fn init_process_start() {
|
||||
LazyLock::force(&START_TIME);
|
||||
LazyLock::force(&SERVER_STATS);
|
||||
}
|
||||
|
||||
pub fn uptime_string() -> String {
|
||||
let secs = START_TIME.elapsed().as_secs();
|
||||
let hours = secs / 3600;
|
||||
let minutes = (secs % 3600) / 60;
|
||||
let seconds = secs % 60;
|
||||
let mut out = String::new();
|
||||
if hours > 0 {
|
||||
out.push_str(&format!("{}h", hours));
|
||||
}
|
||||
if hours > 0 || minutes > 0 {
|
||||
out.push_str(&format!("{}m", minutes));
|
||||
}
|
||||
out.push_str(&format!("{}s", seconds));
|
||||
out
|
||||
}
|
||||
|
||||
pub fn snapshot() -> ServerStatsSnapshot {
|
||||
SERVER_STATS.snapshot()
|
||||
}
|
||||
|
||||
pub fn record_request_open() {
|
||||
SERVER_STATS.update(|inner| inner.requests.add_now(1));
|
||||
}
|
||||
|
||||
pub fn record_request_close() {
|
||||
SERVER_STATS.update(|inner| inner.requests.add_now(-1));
|
||||
}
|
||||
|
||||
pub fn record_connection_open() {
|
||||
SERVER_STATS.update(|inner| inner.connections.add_now(1));
|
||||
}
|
||||
|
||||
pub fn record_connection_close() {
|
||||
SERVER_STATS.update(|inner| inner.connections.add_now(-1));
|
||||
}
|
||||
|
||||
pub fn record_read_request() {
|
||||
SERVER_STATS.update(|inner| inner.read_requests.add_now(1));
|
||||
}
|
||||
|
||||
pub fn record_write_request() {
|
||||
SERVER_STATS.update(|inner| inner.write_requests.add_now(1));
|
||||
}
|
||||
|
||||
pub fn record_delete_request() {
|
||||
SERVER_STATS.update(|inner| inner.delete_requests.add_now(1));
|
||||
}
|
||||
|
||||
pub fn record_bytes_in(bytes: i64) {
|
||||
SERVER_STATS.update(|inner| inner.bytes_in.add_now(bytes));
|
||||
}
|
||||
|
||||
pub fn record_bytes_out(bytes: i64) {
|
||||
SERVER_STATS.update(|inner| inner.bytes_out.add_now(bytes));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn reset_for_tests() {
|
||||
LazyLock::force(&START_TIME);
|
||||
let mut inner = SERVER_STATS.inner.lock().unwrap();
|
||||
*inner = ServerStatsInner::default();
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user