Compare commits

..
Author SHA1 Message Date
Chris Lu 08dbfce493 shell: use tabwriter for aligned output, remove fragile string error checks
- Use text/tabwriter in runS3UserList and runS3UserAccessKeyList for
  properly aligned column output
- Remove fragile strings.Contains "not found" and "already exists"
  fallbacks in s3ShellIsNotFound and s3ShellIsAlreadyExists; rely on
  typed errors and gRPC status codes only
- Update test assertions to match tabwriter-formatted output
2026-04-07 11:24:42 -07:00
Chris Lu d4261c2271 Add workflow-oriented S3 shell commands 2026-04-06 18:27:22 -07:00
1097 changed files with 10640 additions and 112318 deletions
+1
View File
@@ -0,0 +1 @@
{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}
-5
View File
@@ -4,10 +4,6 @@ on:
push:
branches: [ master ]
concurrency:
group: binaries-dev-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
@@ -22,7 +18,6 @@ jobs:
- name: Delete old release assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
+7 -8
View File
@@ -1,10 +1,9 @@
name: "docker: build latest container"
# Manual fallback only. On tag push, container_release_unified.yml already
# re-tags the released versioned image as `latest` / `latest_large_disk`,
# so a full rebuild here is unnecessary. Run this manually if you need to
# rebuild `latest` from an arbitrary ref.
on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
source_ref:
@@ -338,7 +337,7 @@ jobs:
- name: Trivy report (published image)
if: needs.setup.outputs.publish == 'true'
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
scan-type: image
# Scan amd64 only - OS packages are identical across architectures
@@ -355,7 +354,7 @@ jobs:
exit-code: '0'
- name: Trivy report (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
@@ -375,7 +374,7 @@ jobs:
if: needs.setup.outputs.publish == 'true'
# Gate only on fixable high/critical vulnerabilities. Non-fixable
# findings are still visible in the SARIF upload above.
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
scan-type: image
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
@@ -388,7 +387,7 @@ jobs:
skip-setup-trivy: true
- name: Trivy gate (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
@@ -4,19 +4,11 @@ on:
push:
tags:
- '*'
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag to publish (e.g. 3.93)'
required: true
default: ''
workflow_dispatch: {}
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
jobs:
build-large-release-container_foundationdb:
@@ -34,7 +26,7 @@ jobs:
images: |
chrislusf/seaweedfs
tags: |
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
type=ref,event=tag,suffix=_large_disk_foundationdb
flavor: |
latest=false
labels: |
+1 -123
View File
@@ -29,7 +29,6 @@ on:
permissions:
contents: read
security-events: write
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
@@ -358,128 +357,7 @@ jobs:
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
# Security for visibility, but never blocks the release. Releases (including
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
# we sometimes need to publish through known findings (e.g. unfixed upstream
# CVE, base-image lag).
trivy-scan:
runs-on: ubuntu-latest
needs: [build]
if: github.event_name == 'push'
continue-on-error: true
strategy:
fail-fast: false
matrix:
include:
- source_suffix: ""
variant: normal
- source_suffix: _large_disk
variant: large_disk
steps:
- name: Login to GHCR
uses: docker/login-action@v4.1.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Trivy report (${{ matrix.variant }})
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
scan-type: image
# Scan the multi-arch tag on GHCR (already pushed by the build job).
# Trivy scans the runner's native platform; OS packages are identical
# across architectures since they all share the same alpine base.
image-ref: ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Upload Trivy scan results to GitHub Security
if: always()
uses: github/codeql-action/upload-sarif@v4
with:
sarif_file: trivy-results.sarif
category: trivy-${{ matrix.variant }}
# Point `latest` (and `latest_large_disk`) at the just-released versioned
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
# no QEMU, no separate workflow. Replaces the old container_latest.yml
# rebuild that often failed or lagged behind the release. Independent of
# trivy-scan: vuln findings are reported but do not block `latest`.
tag-latest:
runs-on: ubuntu-latest
needs: [copy-to-dockerhub]
if: github.event_name == 'push'
strategy:
matrix:
include:
- source_suffix: ""
latest_tag: latest
- source_suffix: _large_disk
latest_tag: latest_large_disk
steps:
- name: Login to Docker Hub
uses: docker/login-action@v4.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4.1.0
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Install crane
run: |
cd $(mktemp -d)
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
sudo mv crane /usr/local/bin/
crane version
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
run: |
retry_with_backoff() {
local max_attempts=5
local timeout=1
local attempt=1
local exit_code=0
while [ $attempt -le $max_attempts ]; do
if "$@"; then
return 0
else
exit_code=$?
fi
if [ $attempt -lt $max_attempts ]; then
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
sleep $timeout
timeout=$((timeout * 2))
fi
attempt=$((attempt + 1))
done
echo "Command failed after $max_attempts attempts" >&2
return $exit_code
}
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
DST_TAG="${{ matrix.latest_tag }}"
echo "Tagging ghcr.io/chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
retry_with_backoff crane tag "ghcr.io/chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
helm-release:
runs-on: ubuntu-latest
@@ -94,7 +94,7 @@ jobs:
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push image
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
with:
context: ./docker
push: true
+1 -1
View File
@@ -11,4 +11,4 @@ jobs:
- name: 'Checkout Repository'
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- name: 'Dependency Review'
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
+3 -11
View File
@@ -57,12 +57,12 @@ jobs:
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
- name: Start SeaweedFS
timeout-minutes: 15
timeout-minutes: 10
run: |
# Enable Docker buildkit for better caching
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
# Build with retry logic
for i in {1..3}; do
echo "Build attempt $i/3"
@@ -77,18 +77,10 @@ jobs:
sleep 30
fi
done
# Start services with wait
docker compose -f ./compose/e2e-mount.yml up --wait
- name: Rotate buildx cache
if: always()
run: |
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
rm -rf /tmp/.buildx-cache
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
- name: Run FIO 4k
timeout-minutes: 15
run: |
@@ -1,63 +0,0 @@
name: "FUSE DLM Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
- '.github/workflows/fuse-dlm-integration.yml'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/cluster/**'
- 'test/fuse_dlm/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-dlm-integration:
name: FUSE DLM Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run DLM integration tests
timeout-minutes: 25
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-dlm-test-logs
path: /tmp/seaweedfs-fuse-dlm-logs/
retention-days: 3
@@ -1,69 +0,0 @@
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
on:
pull_request:
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
- '.github/workflows/fuse-p2p-integration.yml'
push:
branches: [master]
paths:
- 'weed/command/mount*.go'
- 'weed/mount/**'
- 'weed/filer/mount_peer_registry*.go'
- 'weed/server/filer_grpc_server_mount_peer.go'
- 'weed/pb/mount_peer.proto'
- 'weed/pb/filer.proto'
- 'test/fuse_p2p/**'
concurrency:
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
cancel-in-progress: true
permissions:
contents: read
jobs:
fuse-p2p-integration:
name: FUSE P2P Peer Chunk Sharing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install FUSE dependencies
run: |
sudo apt-get update
sudo apt-get install -y libfuse3-dev
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
sudo chmod 644 /etc/fuse.conf
- name: Build SeaweedFS
run: go build -o weed/weed -buildvcs=false ./weed
- name: Run P2P integration tests
timeout-minutes: 15
env:
WEED_BINARY: ${{ github.workspace }}/weed/weed
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: fuse-p2p-test-logs
path: /tmp/seaweedfs-fuse-p2p-logs/
retention-days: 3
-100
View File
@@ -217,106 +217,6 @@ jobs:
print("✓ No blank lines in security+S3 command blocks")
PYEOF
echo ""
echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
# Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
# the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
# them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
# makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
# HTTP request to an HTTPS server" into the pod log.
#
# When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
# together so the opt-in HTTPS listener actually has credentials.
python3 - "$CHART_DIR" <<'PYEOF'
import subprocess, sys, yaml
chart = sys.argv[1]
def render(values):
args = ["helm", "template", "test", chart]
for k, v in values.items():
args += ["--set", f"{k}={v}"]
return subprocess.check_output(args, text=True)
def script_of(manifest, kind_name):
for doc in yaml.safe_load_all(manifest):
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
continue
if doc["metadata"]["name"] != kind_name:
continue
for c in doc["spec"]["template"]["spec"]["containers"]:
cmd = c.get("command", [])
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
return cmd[2]
raise AssertionError(f"no container script for {kind_name}")
cases = [
# (values, workload-name, httpsPort-set?, arg-prefix)
({"global.seaweedfs.enableSecurity": "true",
"s3.enabled": "true"},
"test-seaweedfs-s3", False, ""),
({"global.seaweedfs.enableSecurity": "true",
"s3.enabled": "true",
"s3.httpsPort": "8443"},
"test-seaweedfs-s3", True, ""),
({"global.seaweedfs.enableSecurity": "true",
"filer.s3.enabled": "true"},
"test-seaweedfs-filer", False, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"filer.s3.enabled": "true",
"filer.s3.httpsPort": "8444"},
"test-seaweedfs-filer", True, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"allInOne.enabled": "true",
"allInOne.s3.enabled": "true"},
"test-seaweedfs-all-in-one", False, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"allInOne.enabled": "true",
"allInOne.s3.enabled": "true",
"allInOne.s3.httpsPort": "8445"},
"test-seaweedfs-all-in-one", True, "s3."),
]
failed = False
for values, name, https_on, prefix in cases:
script = script_of(render(values), name)
cert_flag = f"-{prefix}cert.file="
key_flag = f"-{prefix}key.file="
https_flag = f"-{prefix}port.https="
has_cert = cert_flag in script
has_key = key_flag in script
has_https = https_flag in script
label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
if https_on:
if not (has_cert and has_key and has_https):
print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
failed = True
else:
print(f"✓ {label}: cert/key/https args emitted together")
else:
if has_cert or has_key or has_https:
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
f"main S3 -port would silently become HTTPS and break HTTP probes "
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
failed = True
else:
print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
# bash -n: pin down that the rendered script parses. Guards against
# a future helper change that leaves a dangling `\` with nothing
# after it (every current caller already exits cleanly because
# bash treats trailing `\<newline><EOF>` as line-continuation to
# an empty line — but keep the contract explicit).
parse = subprocess.run(["bash", "-n"], input=script, text=True,
capture_output=True)
if parse.returncode != 0:
print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
file=sys.stderr)
failed = True
sys.exit(1 if failed else 0)
PYEOF
echo "✅ All template rendering tests passed!"
- name: Create kind cluster
+1 -4
View File
@@ -1,9 +1,6 @@
name: "helm: release"
name: "helm: manual release"
on:
push:
tags:
- '*'
workflow_dispatch:
permissions:
+1 -1
View File
@@ -445,7 +445,7 @@ jobs:
# Test consumer group functionality with explicit timeout
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
ulimit -u 100 || echo "Warning: Could not set process limit"
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
env:
GOMAXPROCS: 1
SEAWEEDFS_MASTERS: 127.0.0.1:9333
-77
View File
@@ -1,77 +0,0 @@
name: "Multi-Master Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/server/master_*.go'
- 'weed/server/raft_*.go'
- 'weed/topology/**'
- 'test/multi_master/**'
- 'test/testutil/**'
- '.github/workflows/multi-master-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/multi-master-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
multi-master-failover-tests:
name: Multi-Master Failover Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Run multi-master failover tests
# The tests in test/multi_master spin up their own 3-node master raft
# cluster (using the freshly-installed `weed` binary) and exercise
# leader-election, failover and recovery scenarios. The shared
# test/testutil port-allocator regression test runs alongside since it
# is a prerequisite for the cluster fixtures.
run: |
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
- name: Collect server logs on failure
if: failure()
run: |
# test/multi_master/cluster.go creates per-test dirs via
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
# node's log into <baseDir>/logs/master*.log.
echo "Collecting per-node master logs from temp directories..."
mkdir -p /tmp/multi-master-logs
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
echo "Found test directory: $dir"
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
done
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: multi-master-test-logs
path: /tmp/multi-master-logs/
retention-days: 7
-137
View File
@@ -1,137 +0,0 @@
name: "NFS Integration Tests"
on:
push:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/server/nfs/**'
- 'weed/command/nfs.go'
- 'weed/filer/filer_inode.go'
- 'weed/filer/filer_inode_index.go'
- 'weed/filer/filerstore_wrapper.go'
- 'weed/server/filer_grpc_server_rename.go'
- 'test/nfs/**'
- '.github/workflows/nfs-tests.yml'
concurrency:
group: ${{ github.head_ref }}/nfs-tests
cancel-in-progress: true
permissions:
contents: read
env:
TEST_TIMEOUT: '15m'
jobs:
nfs-integration:
name: NFS Integration Testing
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Build SeaweedFS
run: |
cd weed
go build -o weed .
chmod +x weed
./weed version
- name: Run NFS Integration Tests
run: |
cd test/nfs
echo "Running NFS integration tests..."
echo "============================================"
# Install test dependencies
go mod download
# Run the protocol-layer tests. The kernel-mount tests require root
# for mount(2) and are exercised in their own privileged step below;
# skip them here so a "skipped because not root" line doesn't show
# up as noise on every CI run.
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
echo "============================================"
echo "NFS integration tests completed"
- name: Install kernel NFS client
run: |
# nfs-common provides mount.nfs; netbase provides /etc/protocols
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
sudo apt-get update
sudo apt-get install -y nfs-common netbase
- name: Run kernel-mount E2E tests
run: |
cd test/nfs
echo "Running kernel-mount end-to-end tests..."
echo "These mount the running 'weed nfs' subprocess via the actual"
echo "Linux NFS client to catch protocol regressions invisible to"
echo "the go-nfs-client-based tests above."
echo "============================================"
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
# binary that test/nfs/framework.go locates via $PATH) resolve
# correctly under sudo, and pass through the Go module/cache dirs
# so we don't redownload modules under root.
sudo env "PATH=$PATH" \
GOMODCACHE="$(go env GOMODCACHE)" \
GOCACHE="$(go env GOCACHE)" \
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
echo "============================================"
echo "Kernel-mount E2E tests completed"
- name: Test Summary
if: always()
run: |
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Harness" >> $GITHUB_STEP_SUMMARY
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
-118
View File
@@ -1,118 +0,0 @@
name: "pjdfstest POSIX Compliance"
on:
push:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
pull_request:
branches: [ master, main ]
paths:
- 'weed/mount/**'
- 'weed/filer/**'
- 'test/pjdfstest/**'
- '.github/workflows/pjdfstest.yml'
workflow_dispatch:
concurrency:
group: pjdfstest/${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
pjdfstest:
name: pjdfstest
runs-on: ubuntu-22.04
timeout-minutes: 60
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Start local Docker registry
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver-opts: network=host
- name: Build weed race binary
run: |
cd docker
make binary_race
- name: Build SeaweedFS e2e image
uses: docker/build-push-action@v7
with:
context: docker
file: docker/Dockerfile.e2e
tags: localhost:5000/chrislusf/seaweedfs:e2e
push: true
cache-from: type=gha,scope=pjdfstest-e2e
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
- name: Tag e2e image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:e2e
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
- name: Build pjdfstest image
uses: docker/build-push-action@v7
with:
context: test/pjdfstest
build-contexts: |
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
push: true
cache-from: type=gha,scope=pjdfstest-harness
cache-to: type=gha,mode=max,scope=pjdfstest-harness
- name: Tag pjdfstest image for docker compose
run: |
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
- name: Start SeaweedFS cluster
run: |
docker compose -f test/pjdfstest/docker-compose.yml up --wait
- name: Run pjdfstest
run: |
set -o pipefail
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/pjdfstest-docker-logs
for svc in master volume filer mount; do
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
done
- name: Tear down
if: always()
run: |
docker compose -f test/pjdfstest/docker-compose.yml down -v
- name: Upload logs
if: always()
uses: actions/upload-artifact@v7
with:
name: pjdfstest-results
path: |
/tmp/pjdfstest-output.log
/tmp/pjdfstest-docker-logs/
retention-days: 7
+2 -3
View File
@@ -19,7 +19,6 @@ jobs:
steps:
- name: Delete old Rust volume dev assets
uses: mknejp/delete-release-assets@v1
continue-on-error: true
with:
token: ${{ github.token }}
tag: dev
@@ -84,7 +83,7 @@ jobs:
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
tag_name: dev
prerelease: true
@@ -155,7 +154,7 @@ jobs:
rm weed-volume-normal
- name: Upload dev release assets
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
tag_name: dev
prerelease: true
+3 -3
View File
@@ -89,7 +89,7 @@ jobs:
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
@@ -166,7 +166,7 @@ jobs:
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
files: |
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
@@ -235,7 +235,7 @@ jobs:
- name: Upload release assets
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v3
uses: softprops/action-gh-release@v2
with:
files: |
weed-volume_large_disk_windows_amd64.zip
-129
View File
@@ -1,129 +0,0 @@
name: "S3 ETag and ACL Tests"
on:
push:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
pull_request:
branches: [ master ]
paths:
- 'weed/s3api/**'
- 'weed/filer/etag*.go'
- 'weed/server/filer_server_handlers_*.go'
- 'test/s3/etag/**'
- 'test/s3/acl/**'
- '.github/workflows/s3-etag-acl-tests.yml'
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
s3-etag-acl-tests:
name: S3 ETag + ACL Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
- name: Install SeaweedFS
run: |
cd weed && go install -buildvcs=false
- name: Start weed mini (S3 on :8333)
run: |
mkdir -p /tmp/seaweedfs-etag-acl
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
{
"identities": [
{
"name": "admin",
"credentials": [
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
],
"actions": ["Admin", "Read", "Write"]
}
]
}
JSON
AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
weed mini \
-dir=/tmp/seaweedfs-etag-acl \
-s3.port=8333 \
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
-ip=127.0.0.1 \
> /tmp/weed-mini.log 2>&1 &
echo $! > /tmp/weed-mini.pid
# Wait for the S3 endpoint to come up (returns 403 unauth before any
# request is signed; that's fine — it means the server is listening).
for i in $(seq 1 30); do
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
echo "weed mini is ready"
exit 0
fi
sleep 1
done
echo "weed mini failed to start within 30s"
tail -50 /tmp/weed-mini.log
exit 1
- name: Run ETag tests
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
# SDK for Java v2 rejects the latter on the PutObject path.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/etag/...
- name: Run ACL versioning tests
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
# PutObjectAcl with and without versionId, modifying ACLs on different
# versions independently.
env:
S3_ENDPOINT: http://127.0.0.1:8333
AWS_ACCESS_KEY_ID: some_access_key1
AWS_SECRET_ACCESS_KEY: some_secret_key1
AWS_REGION: us-east-1
run: go test -v -timeout=5m ./test/s3/acl/...
- name: Stop weed mini
if: always()
run: |
if [ -f /tmp/weed-mini.pid ]; then
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
fi
- name: Show server log on failure
if: failure()
run: |
echo "=== weed mini log (last 200 lines) ==="
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
- name: Archive log
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-etag-acl-server-log
path: /tmp/weed-mini.log
retention-days: 3
-125
View File
@@ -245,131 +245,6 @@ jobs:
path: test/s3/retention/weed-test*.log
retention-days: 3
s3-lifecycle-tests:
name: S3 Lifecycle Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
# One job per test so each gets a fresh `weed mini` server, avoiding
# the cross-test volume-pool exhaustion that surfaced when several
# TTL-pinned bucket collections piled up in a single run.
test:
- TestLifecycleAbortIncompleteMultipartUpload
- TestLifecycleBootstrapWalkOnExistingObjects
- TestLifecycleConfigUpdateBetweenSweeps
- TestLifecycleDeleteBucketLifecycleStopsDispatching
- TestLifecycleDisabledRuleSkipsObject
- TestLifecycleEmptyBucketSweepIsNoOp
- TestLifecycleExpirationDateInThePast
- TestLifecycleExpirationFiresOnBackdatedObject
- TestLifecycleExpiredDeleteMarkerCleanup
- TestLifecycleMultipleBucketsInOneSweep
- TestLifecycleMultipleRulesInOneBucket
- TestLifecycleNewerNoncurrentVersions
- TestLifecycleNoncurrentVersionExpiration
- TestLifecycleSizeFilterGreaterThan
- TestLifecycleSkipsObjectLockedObjects
- TestLifecycleSuspendedVersioningExpiration
- TestLifecycleTagFilter
- TestLifecycleVersionedBucketCreatesDeleteMarker
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run ${{ matrix.test }}
timeout-minutes: 8
working-directory: test/s3/lifecycle
run: |
set -x
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
- name: Show server logs on failure
if: failure()
working-directory: test/s3/lifecycle
run: |
if [ -f weed-test.log ]; then
echo "=== Last 200 lines of server logs ==="
tail -200 weed-test.log
fi
ps aux | grep -E "(weed|test)" || true
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-lifecycle-test-logs-${{ matrix.test }}
path: test/s3/lifecycle/weed-test*.log
retention-days: 3
s3-checksum-tests:
name: S3 Checksum Tests
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Checksum Tests
timeout-minutes: 16
working-directory: test/s3/checksum
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Tests ==="
make test-with-server
- name: Show server logs on failure
if: failure()
working-directory: test/s3/checksum
run: |
echo "=== Server Logs ==="
if [ -f weed-test.log ]; then
echo "Last 100 lines of server logs:"
tail -100 weed-test.log
else
echo "No server log file found"
fi
echo "=== Test Environment ==="
ps aux | grep -E "(weed|test)" || true
netstat -tlnp | grep -E "(8333|9333|8080)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: s3-checksum-test-logs
path: test/s3/checksum/weed-test*.log
retention-days: 3
s3-cors-tests:
name: S3 CORS Tests
runs-on: ubuntu-22.04
+1 -18
View File
@@ -84,7 +84,7 @@ jobs:
timeout-minutes: 25
strategy:
matrix:
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
test-type: ["basic", "advanced", "policy-enforcement", "group"]
steps:
- name: Check out code
@@ -138,23 +138,6 @@ jobs:
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestIAMGroup" ./...
;;
"sts")
echo "Running STS and service account tests..."
make clean setup start-services wait-for-services
# SigV4-signed STS calls need admin credentials matching test_config.json.
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
export STS_TEST_ACCESS_KEY=test-access-key
export STS_TEST_SECRET_KEY=test-secret-key
# The use_service_account_credentials subtest is excluded because
# newly-created service-account access keys are not currently
# persisted to the filer after CreateServiceAccount — a
# pre-existing sync issue tracked separately from the
# GetFederationToken routing fix this PR addresses.
go test -v -timeout 15m \
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
./...
;;
*)
echo "Unknown test type: ${{ matrix.test-type }}"
exit 1
+2 -6
View File
@@ -73,12 +73,8 @@ jobs:
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
else
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
# same change that fixed them.
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
fi
- name: Show server logs on failure
+4 -213
View File
@@ -2,6 +2,10 @@ name: "S3 Tables Integration Tests"
on:
pull_request:
concurrency:
group: ${{ github.head_ref }}/s3-tables-tests
cancel-in-progress: true
permissions:
contents: read
@@ -190,148 +194,6 @@ jobs:
path: test/s3tables/catalog_trino/test-output.log
retention-days: 3
dremio-iceberg-catalog-tests:
name: Dremio Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Dremio image
run: docker pull dremio/dremio-oss:25.2.0
- name: Pre-pull Python image for PyIceberg writer
run: docker pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Dremio Iceberg Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/catalog_dremio
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Dremio Iceberg Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Dremio Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_dremio
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|dremio)" || true
echo "=== Dremio containers ==="
docker ps -a --filter "name=seaweed-dremio" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: dremio-iceberg-catalog-test-logs
path: test/s3tables/catalog_dremio/test-output.log
retention-days: 3
doris-iceberg-catalog-tests:
name: Doris Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 35
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Doris image
run: docker pull apache/doris:doris-all-in-one-2.1.0
- name: Pre-pull Python image for PyIceberg writer
run: docker pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Doris Iceberg Catalog Integration Tests
timeout-minutes: 30
working-directory: test/s3tables/catalog_doris
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Doris Iceberg Catalog Tests ==="
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
echo "Doris Iceberg catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/catalog_doris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|doris)" || true
echo "=== Doris containers ==="
docker ps -a --filter "name=seaweed-doris" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: doris-iceberg-catalog-test-logs
path: test/s3tables/catalog_doris/test-output.log
retention-days: 3
polaris-integration-tests:
name: Polaris Integration Tests
runs-on: ubuntu-22.04
@@ -662,77 +524,6 @@ jobs:
path: test/s3tables/lakekeeper/test-output.log
retention-days: 3
unity-catalog-integration-tests:
name: Unity Catalog Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
- name: Pre-pull Unity Catalog image
run: docker pull unitycatalog/unitycatalog:v0.4.0
- name: Pre-pull Python image for delta-rs writer
run: docker pull python:3.11-slim
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Run Unity Catalog Integration Tests
timeout-minutes: 25
working-directory: test/s3tables/unity_catalog
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
docker info
echo "=== Starting Unity Catalog Tests ==="
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
echo "Unity Catalog integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/unity_catalog
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
echo "=== Unity Catalog containers ==="
docker ps -a --filter "name=seaweed-unity-catalog" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: unity-catalog-integration-test-logs
path: test/s3tables/unity_catalog/test-output.log
retention-days: 3
s3-tables-build-verification:
name: S3 Tables Build Verification
runs-on: ubuntu-22.04
+11 -88
View File
@@ -37,8 +37,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -132,49 +130,7 @@ jobs:
done
echo "✅ S3 server is responding, starting tests..."
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
# something driving deletions. The s3tests build tag rescales one
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
# the upload's mtime; -dispatch / -checkpoint defaults are already
# tightened under the same build tag.
LC_LOG=/tmp/lifecycle-worker.log
# -debug routes glog to stderr so the bootstrap walker's progress
# shows up in $LC_LOG; without it weed shell silences glog.
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
| weed shell -debug -master=localhost:9333 \
> "$LC_LOG" 2>&1 &
lc_pid=$!
# Aliveness check: a bad shell command exits in <1s and the suite
# would otherwise just timeout the expiration tests with no signal.
sleep 2
if ! kill -0 "$lc_pid" 2>/dev/null; then
echo "lifecycle worker died on startup"
tail -50 "$LC_LOG" 2>/dev/null || true
exit 1
fi
echo "lifecycle worker pid=$lc_pid"
# bash -e exits the step on the first tox failure, so move teardown
# into a trap to guarantee the worker log + data dir reach the runner.
cleanup() {
status=$?
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
# bash fallback if it ignores TERM. Reading the log AFTER the
# graceful-stop window catches the bootstrap walker's progress.
kill -TERM "$lc_pid" 2>/dev/null || true
kill -TERM "$pid" 2>/dev/null || true
sleep 1
if [ "$status" -ne 0 ]; then
echo "=== lifecycle worker log (tail) ==="
tail -200 "$LC_LOG" 2>/dev/null || true
fi
kill -9 "$lc_pid" 2>/dev/null || true
kill -9 "$pid" 2>/dev/null || true
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
}
trap cleanup EXIT
tox -- \
s3tests/functional/test_s3.py::test_bucket_list_empty \
s3tests/functional/test_s3.py::test_bucket_list_distinct \
@@ -354,8 +310,11 @@ jobs:
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
# cleanup() trap handles worker/server kill + data dir wipe.
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
versioning-tests:
name: S3 Versioning & Object Lock tests
@@ -380,8 +339,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -550,8 +507,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -775,8 +730,6 @@ jobs:
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
sudo apt-get update -qq
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
pip install -r requirements.txt
pip install tox
pip install -e .
@@ -997,39 +950,6 @@ jobs:
sleep 2
done
# Spawn the lifecycle worker (see basic-tests block for context).
LC_LOG=/tmp/lifecycle-worker-sql.log
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
| weed shell -debug -master=localhost:9337 \
> "$LC_LOG" 2>&1 &
lc_pid=$!
sleep 2
if ! kill -0 "$lc_pid" 2>/dev/null; then
echo "lifecycle worker died on startup"
tail -50 "$LC_LOG" 2>/dev/null || true
exit 1
fi
echo "lifecycle worker pid=$lc_pid"
cleanup() {
status=$?
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
# bash fallback if it ignores TERM. Reading the log AFTER the
# graceful-stop window catches the bootstrap walker's progress.
kill -TERM "$lc_pid" 2>/dev/null || true
kill -TERM "$pid" 2>/dev/null || true
sleep 1
if [ "$status" -ne 0 ]; then
echo "=== lifecycle worker log (tail) ==="
tail -200 "$LC_LOG" 2>/dev/null || true
fi
kill -9 "$lc_pid" 2>/dev/null || true
kill -9 "$pid" 2>/dev/null || true
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
}
trap cleanup EXIT
tox -- \
s3tests/functional/test_s3.py::test_bucket_list_empty \
s3tests/functional/test_s3.py::test_bucket_list_distinct \
@@ -1209,7 +1129,10 @@ jobs:
s3tests/functional/test_s3.py::test_lifecycle_get \
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
s3tests/functional/test_s3.py::test_lifecycle_expiration \
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
# cleanup() trap handles worker/server kill + data dir wipe.
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
-56
View File
@@ -1,56 +0,0 @@
name: "TLS Rotation Integration Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
permissions:
contents: read
jobs:
tls-rotation-tests:
name: TLS Rotation Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run TLS Rotation Integration Tests
working-directory: test/tls_rotation
run: |
go test -v -count=1 -timeout 5m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting master logs from temp directories..."
mkdir -p /tmp/tls-rotation-test-logs
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: tls-rotation-test-logs
path: /tmp/tls-rotation-test-logs/
retention-days: 14
@@ -1,56 +0,0 @@
name: "Vacuum Integration Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
permissions:
contents: read
jobs:
vacuum-integration-tests:
name: Vacuum Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Build weed binary
run: |
cd weed && go build -o weed .
- name: Run Vacuum Integration Tests
working-directory: test/vacuum
run: |
go test -v -timeout 10m
- name: Collect server logs on failure
if: failure()
run: |
echo "Collecting server logs from temp directories..."
mkdir -p /tmp/vacuum-test-logs
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
if [ -d "$dir" ]; then
echo "Found test directory: $dir"
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
fi
done
echo "Collected logs:"
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: vacuum-integration-test-logs
path: /tmp/vacuum-test-logs/
retention-days: 14
-1
View File
@@ -2,7 +2,6 @@
vendor
tags
*.swp
.claude/
### OSX template
.DS_Store
.AppleDouble
+2 -1
View File
@@ -42,7 +42,8 @@ test: admin-generate
# Admin component targets
admin-generate:
@cd $(ADMIN_DIR) && $(MAKE) generate
@echo "Generating admin component templates..."
@cd $(ADMIN_DIR) && templ generate ./view
admin-build: admin-generate
@echo "Building admin component..."
+18 -39
View File
@@ -56,6 +56,7 @@ Table of Contents
* [Quick Start](#quick-start)
* [Quick Start with weed mini](#quick-start-with-weed-mini)
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
* [Quick Start with Single Binary](#quick-start-with-single-binary)
* [Introduction](#introduction)
* [Features](#features)
* [Additional Features](#additional-features)
@@ -79,41 +80,39 @@ Table of Contents
## Quick Start with weed mini ##
The easiest way to get started with SeaweedFS for development and testing:
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file, or run `go install github.com/seaweedfs/seaweedfs/weed@latest`. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
Example:
```bash
AWS_ACCESS_KEY_ID=admin \
AWS_SECRET_ACCESS_KEY=secret \
S3_BUCKET=my-bucket \
# remove quarantine on macOS
# xattr -d com.apple.quarantine ./weed
./weed mini -dir=/data
```
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
The same command starts everything else too:
- **S3 Endpoint**: http://localhost:8333
This single command starts a complete SeaweedFS setup with:
- **Master UI**: http://localhost:9333
- **Volume Server**: http://localhost:9340
- **Filer UI**: http://localhost:8888
- **S3 Endpoint**: http://localhost:8333
- **WebDAV**: http://localhost:7333
- **Admin UI**: http://localhost:23646
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
Perfect for development, testing, learning SeaweedFS, and single-node deployments. To scale out, add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, on another machine, or on thousands of machines.
Perfect for development, testing, learning SeaweedFS, and single node deployments!
## Quick Start for S3 API on Docker ##
```bash
docker run -p 8333:8333 \
-e AWS_ACCESS_KEY_ID=admin \
-e AWS_SECRET_ACCESS_KEY=secret \
-e S3_BUCKET=my-bucket \
chrislusf/seaweedfs
```
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
Same behavior as the `weed mini` command above — the S3 endpoint is at http://localhost:8333 with `my-bucket` pre-created. Drop the env vars to run anonymously for development.
## Quick Start with Single Binary ##
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
# Introduction #
@@ -146,11 +145,6 @@ SeaweedFS can achieve both fast local access time and elastic cloud storage capa
What's more, the cloud storage access API cost is minimized.
Faster and cheaper than direct cloud storage!
SeaweedFS also ships a built-in **Iceberg REST Catalog**, turning the same cluster into a self-contained lakehouse.
Spark, Trino, Dremio, DuckDB, and RisingWave can query Iceberg tables directly — no Hive Metastore, Glue, or
external catalog service required. Storage and table metadata live in one system, simplifying on-prem and
small-team analytics stacks.
[Back to TOC](#table-of-contents)
# Features #
@@ -187,13 +181,6 @@ small-team analytics stacks.
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
## Data Lakehouse Features ##
* [S3 Table Buckets][S3TableBucket] expose a dedicated namespace for Iceberg tables with strict layout validation.
* Built-in [Iceberg REST Catalog][IcebergCatalog] runs alongside the S3 endpoint — no external metastore needed.
* Native integrations with [Apache Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], and [RisingWave][RisingWaveIceberg].
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan removal, manifest rewriting.
* Granular IAM at the bucket, namespace, and table level via standard S3 bucket policies.
## Kubernetes ##
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
@@ -217,14 +204,6 @@ small-team analytics stacks.
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket
[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration
[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration
[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration
[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
[Back to TOC](#table-of-contents)
File diff suppressed because it is too large Load Diff
-24
View File
@@ -1,24 +0,0 @@
# Security Policy
## Reporting a Vulnerability
If you find a security issue in SeaweedFS, please report it privately:
- Email: support@seaweedfs.com
- Do not open a public GitHub issue
Please include:
- A clear description of the issue
- Steps to reproduce (if possible)
- Affected versions
## Response
- We will respond as soon as possible (usually within 1 business day)
- We will investigate and work on a fix
- We may coordinate disclosure with you
## Notes
- Please allow time for a fix before public disclosure
- If you’re unsure whether something is a security issue, feel free to reach out
+3 -5
View File
@@ -2,15 +2,13 @@ FROM ubuntu:22.04
LABEL author="Chris Lu"
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
# Use faster mirrors and optimize package installation
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
# For testing purposes, running as root avoids permission complexities and dependency
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
apt-get -o Acquire::http::Timeout=15 update && \
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
--no-install-recommends \
--no-install-suggests \
curl \
+2 -7
View File
@@ -56,8 +56,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
# To disable: docker run -e GODEBUG=fips140=off ...
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse curl su-exec libgcc && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -89,8 +89,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -6
View File
@@ -7,7 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -40,8 +40,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -6
View File
@@ -34,7 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse snappy gflags curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -68,8 +68,3 @@ WORKDIR /data
# Entrypoint will handle permission fixes and user switching
ENTRYPOINT ["/entrypoint.sh"]
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
# so the image is usable out of the box — including in environments like
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
# Override with any other subcommand at `docker run` / compose time.
CMD ["mini", "-dir=/data"]
+1 -1
View File
@@ -17,7 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache && \
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
+2 -15
View File
@@ -37,21 +37,17 @@ if [ "$(id -u)" = "0" ]; then
fi
isArgPassed() {
# Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9
# library accepts both, and users may pick either form on the CLI.
arg="$1"
argWithEqualSign="$1="
argDouble="-$1"
argDoubleWithEqualSign="-$1="
shift
while [ $# -gt 0 ]; do
passedArg="$1"
shift
case $passedArg in
"$arg"|"$argDouble")
"$arg")
return 0
;;
"$argWithEqualSign"*|"$argDoubleWithEqualSign"*)
"$argWithEqualSign"*)
return 0
;;
esac
@@ -99,15 +95,6 @@ case "$1" in
exec /usr/bin/weed -logtostderr=true server $ARGS $@
;;
'mini')
ARGS="-dir=/data"
if isArgPassed "-dir" "$@"; then
ARGS=""
fi
shift
exec /usr/bin/weed -logtostderr=true mini $ARGS $@
;;
'filer')
ARGS=""
shift
+87 -95
View File
@@ -4,8 +4,8 @@ go 1.25.0
require (
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/pubsub v1.50.2
cloud.google.com/go/storage v1.62.1
cloud.google.com/go/pubsub v1.50.1
cloud.google.com/go/storage v1.60.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.55.8
github.com/beorn7/perks v1.0.1 // indirect
@@ -34,20 +34,20 @@ require (
github.com/google/btree v1.1.3
github.com/google/uuid v1.6.0
github.com/google/wire v0.7.0 // indirect
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
github.com/gorilla/mux v1.8.1
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/jackc/pgx/v5 v5.9.2
github.com/jackc/pgx/v5 v5.8.0
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
github.com/jinzhu/copier v0.4.0
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/klauspost/compress v1.18.6
github.com/klauspost/reedsolomon v1.14.0
github.com/klauspost/compress v1.18.5
github.com/klauspost/reedsolomon v1.13.3
github.com/kurin/blazer v0.5.3
github.com/linxGnu/grocksdb v1.10.7
github.com/mailru/easyjson v0.9.1 // indirect
@@ -61,14 +61,14 @@ require (
github.com/posener/complete v1.2.3
github.com/pquerna/cachecontrol v0.2.0
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/client_model v0.6.2
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.2 // indirect
github.com/prometheus/procfs v0.20.1
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/seaweedfs/goexif v1.0.3
github.com/seaweedfs/raft v1.1.8
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/seaweedfs/raft v1.1.7
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/viper v1.21.0
@@ -92,45 +92,44 @@ require (
gocloud.dev v0.45.0
gocloud.dev/pubsub/natspubsub v0.45.0
gocloud.dev/pubsub/rabbitpubsub v0.45.0
golang.org/x/crypto v0.50.0
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
golang.org/x/image v0.39.0
golang.org/x/net v0.53.0
golang.org/x/crypto v0.49.0
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
golang.org/x/image v0.38.0
golang.org/x/net v0.52.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sys v0.43.0
golang.org/x/text v0.36.0 // indirect
golang.org/x/tools v0.44.0 // indirect
golang.org/x/sys v0.42.0
golang.org/x/text v0.35.0 // indirect
golang.org/x/tools v0.42.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.278.0
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
google.golang.org/grpc v1.80.0
google.golang.org/api v0.274.0
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
google.golang.org/grpc v1.79.3
google.golang.org/protobuf v1.36.11
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.49.1
modernc.org/sqlite v1.46.1
)
require (
cloud.google.com/go/kms v1.31.0
cloud.google.com/go/kms v1.26.0
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
github.com/Jille/raft-grpc-transport v1.6.1
github.com/ThreeDotsLabs/watermill v1.5.1
github.com/a-h/templ v0.3.1001
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
github.com/a-h/templ v0.3.977
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
github.com/apache/iceberg-go v0.5.0
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
github.com/arangodb/go-driver v1.6.9
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.41.6
github.com/aws/aws-sdk-go-v2/config v1.32.14
github.com/aws/aws-sdk-go-v2 v1.41.5
github.com/aws/aws-sdk-go-v2/config v1.32.13
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.44.1
github.com/go-git/go-billy/v5 v5.9.0
github.com/go-ldap/ldap/v3 v3.4.13
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -143,21 +142,19 @@ require (
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.28.0
github.com/pkg/sftp v1.13.10
github.com/rabbitmq/amqp091-go v1.11.0
github.com/rclone/rclone v1.74.1
github.com/rabbitmq/amqp091-go v1.10.0
github.com/rclone/rclone v1.73.1
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.18.0
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.3
github.com/shirou/gopsutil/v4 v4.26.3
github.com/seaweedfs/go-fuse/v2 v2.9.2
github.com/shirou/gopsutil/v4 v4.26.2
github.com/tarantool/go-tarantool/v2 v2.4.2
github.com/testcontainers/testcontainers-go v0.40.0
github.com/tikv/client-go/v2 v2.0.7
github.com/willscott/go-nfs v0.0.4
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
go.etcd.io/etcd/client/pkg/v3 v3.6.10
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.20.0
@@ -171,18 +168,17 @@ require (
atomicgo.dev/cursor v0.2.0 // indirect
atomicgo.dev/keyboard v0.2.9 // indirect
atomicgo.dev/schedule v0.1.0 // indirect
cloud.google.com/go/longrunning v0.9.0 // indirect
cloud.google.com/go/pubsub/v2 v2.4.0 // indirect
cloud.google.com/go/longrunning v0.8.0 // indirect
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 // indirect
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
github.com/adrg/xdg v0.5.3 // indirect
github.com/anchore/go-lzo v0.1.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
github.com/apache/thrift v0.23.0 // indirect
github.com/apache/thrift v0.22.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/bazelbuild/rules_go v0.46.0 // indirect
@@ -191,7 +187,8 @@ require (
github.com/boombuler/barcode v1.1.0 // indirect
github.com/buger/jsonparser v1.1.2 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
github.com/cockroachdb/errors v1.11.3 // indirect
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
@@ -212,6 +209,7 @@ require (
github.com/dromara/dongle v1.0.1 // indirect
github.com/gin-gonic/gin v1.11.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
github.com/go-git/go-billy/v5 v5.6.2 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
github.com/google/go-cmp v0.7.0 // indirect
@@ -224,7 +222,7 @@ require (
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
@@ -257,10 +255,9 @@ require (
github.com/pierrre/geohash v1.0.0 // indirect
github.com/pquerna/otp v1.5.0 // indirect
github.com/pterm/pterm v0.12.82 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
github.com/rclone/go-proton-api v1.0.2 // indirect
github.com/quic-go/quic-go v0.57.0 // indirect
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/sasha-s/go-deadlock v0.3.1 // indirect
@@ -285,40 +282,40 @@ require (
go.uber.org/mock v0.5.2 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.35.0 // indirect
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
golang.org/x/mod v0.33.0 // indirect
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
gonum.org/v1/gonum v0.17.0 // indirect
)
require (
cel.dev/expr v0.25.1 // indirect
cloud.google.com/go/auth v0.20.0 // indirect
cloud.google.com/go/auth v0.18.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.7.0 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
filippo.io/edwards25519 v1.1.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
github.com/Azure/go-ntlmssp v0.1.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
github.com/Azure/go-ntlmssp v0.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
github.com/ProtonMail/go-crypto v1.4.1 // indirect
github.com/ProtonMail/go-crypto v1.3.0 // indirect
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
github.com/ProtonMail/go-srp v0.0.7 // indirect
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
github.com/PuerkitoBio/goquery v1.11.0 // indirect
github.com/PuerkitoBio/goquery v1.10.3 // indirect
github.com/abbot/go-http-auth v0.4.0 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
@@ -326,7 +323,7 @@ require (
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
@@ -340,7 +337,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
github.com/aws/smithy-go v1.25.1
github.com/aws/smithy-go v1.24.2
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -348,13 +345,13 @@ require (
github.com/calebcase/tmpfile v1.0.3 // indirect
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
github.com/cloudflare/circl v1.6.3 // indirect
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/creasty/defaults v1.8.0 // indirect
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
@@ -368,7 +365,7 @@ require (
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
github.com/geoffgarside/ber v1.2.0 // indirect
github.com/go-chi/chi/v5 v5.2.5 // indirect
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
@@ -376,19 +373,19 @@ require (
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/errors v0.22.6 // indirect
github.com/go-openapi/errors v0.22.4 // indirect
github.com/go-openapi/strfmt v0.25.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.1 // indirect
github.com/go-resty/resty/v2 v2.17.2 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/go-playground/validator/v10 v10.28.0 // indirect
github.com/go-resty/resty/v2 v2.16.5 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/gofrs/flock v0.13.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
github.com/gorilla/schema v1.4.1 // indirect
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/sessions v1.4.0
@@ -405,7 +402,7 @@ require (
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
@@ -419,9 +416,9 @@ require (
github.com/lanrat/extsort v1.4.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lpar/date v1.0.0 // indirect
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-runewidth v0.0.22 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
@@ -436,8 +433,8 @@ require (
github.com/oklog/ulid v1.3.1 // indirect
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
github.com/panjf2000/ants/v2 v2.11.5 // indirect
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
github.com/panjf2000/ants/v2 v2.11.3 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
@@ -453,7 +450,7 @@ require (
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
github.com/relvacode/iso8601 v1.7.0 // indirect
github.com/rfjakob/eme v1.2.0 // indirect
github.com/rfjakob/eme v1.1.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
@@ -466,11 +463,11 @@ require (
github.com/spf13/pflag v1.0.10 // indirect
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
github.com/tarantool/go-iproto v1.1.0 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
github.com/tinylib/msgp v1.6.3 // indirect
github.com/tinylib/msgp v1.5.0 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/twmb/murmur3 v1.1.8 // indirect
@@ -479,7 +476,7 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
@@ -490,37 +487,32 @@ require (
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
go.opentelemetry.io/otel v1.43.0 // indirect
go.opentelemetry.io/otel/metric v1.43.0 // indirect
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
go.opentelemetry.io/otel/trace v1.43.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
go.opentelemetry.io/otel v1.42.0 // indirect
go.opentelemetry.io/otel/metric v1.42.0 // indirect
go.opentelemetry.io/otel/sdk v1.42.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect
go.opentelemetry.io/otel/trace v1.42.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
golang.org/x/term v0.42.0
golang.org/x/time v0.15.0
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
golang.org/x/term v0.41.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.72.0 // indirect
modernc.org/libc v1.68.0 // indirect
moul.io/http2curl/v2 v2.3.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
storj.io/infectious v0.0.2 // indirect
storj.io/picobuf v0.0.4 // indirect
storj.io/uplink v1.14.0 // indirect
storj.io/uplink v1.13.1 // indirect
)
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
// apache/thrift v0.23.0 uses math.MaxUint32 as an untyped int constant in
// lib/go/thrift/framed_transport.go, which overflows int on 32-bit GOARCHes
// (e.g. openbsd/arm, linux/arm). Pin to v0.22.0 until upstream fixes it.
replace github.com/apache/thrift => github.com/apache/thrift v0.22.0
+182 -190
View File
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA=
cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q=
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
@@ -283,8 +283,8 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE
cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY=
cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U=
cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY=
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc=
cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A=
cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk=
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
cloud.google.com/go/kms v1.31.0 h1:LS8N92OxFDgOLg5NCo3OmbvjtQAIVT5gUHVLKIDHaFE=
cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U=
cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -315,8 +315,8 @@ cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZC
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY=
cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E=
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE=
cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM=
cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA=
@@ -391,10 +391,10 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp
cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI=
cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0=
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
cloud.google.com/go/pubsub v1.50.2 h1:54Up97HnThdP4H8jjWJSSQ/mnYG2EKon7ZSNETRq0tM=
cloud.google.com/go/pubsub v1.50.2/go.mod h1:jyCWeZdGFqd4mitSsBERnJcpqaHBsxQoPkNvjj4sp0w=
cloud.google.com/go/pubsub/v2 v2.4.0 h1:oMKNiBQpXImRWnHYla9uSU66ZzByZwBSCJOEs/pTKVg=
cloud.google.com/go/pubsub/v2 v2.4.0/go.mod h1:2lS/XQKq5qtOMs6kHBK+WX1ytUC36kLl2ig3zqsGUx8=
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
@@ -485,8 +485,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq
cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8=
cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA=
cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I=
cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4=
@@ -569,16 +569,16 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiIS75iTVAKIUxrST5X2dVHMTptL4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc=
github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M=
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A=
github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
@@ -593,20 +593,20 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0=
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
github.com/Files-com/files-sdk-go/v3 v3.3.82 h1:2RfP0d2QgkFH64BjZSWd59aMsc28IyWsUuHqU0txBtY=
github.com/Files-com/files-sdk-go/v3 v3.3.82/go.mod h1:IPk80dOmc7VFC0DJ85xMTPmre+8xoXX6kGHAkf5jRRw=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 h1:W8S9TrAyZ4//3PXsU6+Bi+fe/6uIL986GyS7PVzIDL4=
github.com/FilenCloudDienste/filen-sdk-go v0.0.37/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
github.com/Files-com/files-sdk-go/v3 v3.2.264 h1:lMHTplAYI9FtmCo/QOcpRxmPA5REVAct1r2riQmDQKw=
github.com/Files-com/files-sdk-go/v3 v3.2.264/go.mod h1:wGqkOzRu/ClJibvDgcfuJNAqI2nLhe8g91tPlDKRCdE=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
github.com/IBM/go-sdk-core/v5 v5.21.2 h1:mJ5QbLPOm4g5qhZiVB6wbSllfpeUExftGoyPek2hk4M=
github.com/IBM/go-sdk-core/v5 v5.21.2/go.mod h1:ngpMgwkjur1VNUjqn11LPk3o5eCyOCRbcfg/0YAY7Hc=
github.com/IBM/go-sdk-core/v5 v5.21.0 h1:DUnYhvC4SoC8T84rx5omnhY3+xcQg/Whyoa3mDPIMkk=
github.com/IBM/go-sdk-core/v5 v5.21.0/go.mod h1:Q3BYO6iDA2zweQPDGbNTtqft5tDcEpm6RTuqMlPcvbw=
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
github.com/Jille/raft-grpc-transport v1.6.1/go.mod h1:HbOjEdu/yzCJ/mjTF6wEOJNbAUpHfU2UOA2hVD4CNFg=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
@@ -633,16 +633,16 @@ github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk=
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo=
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI=
github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk=
github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE=
github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s=
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/Sereal/Sereal/Go/sereal v0.0.0-20231009093132-b9187f1a92c6/go.mod h1:JwrycNnC8+sZPDyzM3MQ86LvaGzSpfxg885KOOwFRW4=
github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
@@ -652,18 +652,16 @@ github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfih
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb h1:wumPkzt4zaxO4rHPBrjDK8iZMR41C1qs7njNqlacwQg=
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb/go.mod h1:QiYsIBRQEO+Z4Rz7GoI+dsHVneZNONvhczuA+llOZNM=
github.com/a-h/templ v0.3.1001 h1:yHDTgexACdJttyiyamcTHXr2QkIeVF1MukLy44EAhMY=
github.com/a-h/templ v0.3.1001/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 h1:iLDOF0rdGTrol/q8OfPIIs5kLD8XvA2q75o6Uq/tgak=
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0/go.mod h1:DrEWcQJjz7t5iF2duaiyhg4jyoF0kxOD6LtECNGkZ/Q=
github.com/aalpar/deheap v1.1.2 h1:MABHLcnjqsffb8GLkUFDigqpBBxOMz0DoKM9QfELeTw=
github.com/aalpar/deheap v1.1.2/go.mod h1:A+nfkD4JbS05sewV0he/MYgR/90vfqyMoNNROgs+rmA=
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 h1:hhdWprfSpFbN7lz3W1gM40vOgvSh1WCSMxYD6gGB4Hs=
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3/go.mod h1:XaUnRxSCYgL3kkgX0QHIV0D+znljPIDImxlv2kbGv0Y=
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -694,10 +692,11 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 h1:r/gg2gzUXiXoy72VU3jnODh8l/5rL0aslnTAbtmai/U=
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87/go.mod h1:IJTMBTlHe7cDOhRh0ioGuEKBl5iTR6xPfl5BN4AgirU=
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq/RrkjCcxCnqUxhwf8=
github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
github.com/apache/cassandra-gocql-driver/v2 v2.0.0 h1:Omnzb1Z/P90Dr2TbVNu54ICQL7TKVIIsJO231w484HU=
github.com/apache/cassandra-gocql-driver/v2 v2.0.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
@@ -715,18 +714,18 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg=
github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ=
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
@@ -743,8 +742,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3x
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
@@ -757,8 +756,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -828,13 +827,15 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
github.com/cloudinary/cloudinary-go/v2 v2.15.0 h1:iLoIwb7BJECHTbNcmIhYDsQhoZiACWGNvEpyqQy97Dk=
github.com/cloudinary/cloudinary-go/v2 v2.15.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
github.com/cloudinary/cloudinary-go/v2 v2.13.0 h1:ugiQwb7DwpWQnete2AZkTh94MonZKmxD7hDGy1qTzDs=
github.com/cloudinary/cloudinary-go/v2 v2.13.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg=
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=
@@ -904,9 +905,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=
github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 h1:tLWBZgPg6TV67oe76W4p+aUQEWIa52wbcuiz8GFd3vo=
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
@@ -1047,8 +1047,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
@@ -1071,8 +1071,8 @@ github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3
github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA=
github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw=
github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM=
github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -1096,8 +1096,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM=
github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
@@ -1116,8 +1116,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
@@ -1126,8 +1126,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -1136,8 +1136,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas=
github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4=
github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY=
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
@@ -1397,14 +1397,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 h1:PV6ipOJN0JIek4dqPqsTtenQmjI1SZntCUgPzhfk9gM=
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160/go.mod h1:yRuPDnHgGmsbvopF0amMqXxr4n32GynzX5GTwFYDHaw=
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c h1:r+KtxPyrhsYeNbsfeqTfEM8xRdwgV6LuNhLZxpXecb4=
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jaegertracing/jaeger v1.47.0 h1:XXxTMO+GxX930gxKWsg90rFr6RswkCRIW0AgWFnTYsg=
@@ -1429,8 +1429,8 @@ github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb h1:6vkM8gO+zFV2m21QzGYyUSq5TP0VQgP2Xz3UQyCN2kI=
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb/go.mod h1:H1+whwD0Qe3YOunlXIWhh3rlvzW5cZfkMDYGQPg+KAM=
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs=
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -1475,15 +1475,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
@@ -1523,8 +1523,8 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
github.com/lpar/date v1.0.0/go.mod h1:KjYe0dDyMQTgpqcUz4LEIeM5VZwhggjVx/V2dtc8NSo=
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM=
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
@@ -1544,8 +1544,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk=
github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -1657,12 +1657,12 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
github.com/oracle/oci-go-sdk/v65 v65.111.0 h1:eDkWg6ZN0uKwWzSekoFcQJhR+C+F/aVdTwr+lGHU9Qk=
github.com/oracle/oci-go-sdk/v65 v65.111.0/go.mod h1:8ZzvzuEG/cFLFZhxg/Mg1w19KqyXBKO3c17QIc5PkGs=
github.com/oracle/oci-go-sdk/v65 v65.104.0 h1:l9awEvzWvxmYhy/97A0hZ87pa7BncYXmcO/S8+rvgK0=
github.com/oracle/oci-go-sdk/v65 v65.104.0/go.mod h1:oB8jFGVc/7/zJ+DbleE8MzGHjhs2ioCz5stRTdZdIcY=
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
@@ -1755,8 +1755,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8=
github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
@@ -1779,18 +1779,16 @@ github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++
github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
github.com/rclone/go-proton-api v1.0.2/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
github.com/rclone/rclone v1.74.1 h1:QQI//tl4O5uooTowdKwNKcv5qCQvJ3LIBG00+TKeXw0=
github.com/rclone/rclone v1.74.1/go.mod h1:44epaKf5w/1ZQounefr76wx4+y9NZmgfMKle9T3pzj0=
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
github.com/rclone/rclone v1.73.1 h1:FCGZUd7sXO8+nIpbuXGPkd+f23o/ZACefrHtRGDAiDM=
github.com/rclone/rclone v1.73.1/go.mod h1:T9HeSXv/SI9qJFkYLqsjY/5ljIFWJTVmTrG7H/4JrPY=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
@@ -1809,8 +1807,8 @@ github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=
github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -1840,12 +1838,12 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
github.com/seaweedfs/go-fuse/v2 v2.9.3 h1:rJufGrHImTx7yoGUmetUi+To4LrmTQJROJnBHWt92ic=
github.com/seaweedfs/go-fuse/v2 v2.9.3/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
github.com/seaweedfs/raft v1.1.8 h1:a5f+wSnriRjINLqLqMY7DDNNfU1+kv6TKBuOZa0KDwU=
github.com/seaweedfs/raft v1.1.8/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
github.com/seaweedfs/raft v1.1.7 h1:3mVJZ2p4rdvBtbbrHROPjYKtH+q5qjMBc56G6VRu1kA=
github.com/seaweedfs/raft v1.1.7/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA=
github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@@ -1856,15 +1854,15 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0=
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
@@ -1887,8 +1885,8 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
@@ -1930,8 +1928,8 @@ github.com/substrait-io/substrait-protobuf/go v0.81.0 h1:/qC1XYKuO4oPdTwLYySuVZ6
github.com/substrait-io/substrait-protobuf/go v0.81.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ=
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c h1:dtcOwRimeiBFrlutmF6K94l0rxYFARNFMA+lSQ41C+M=
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 h1:rrGZv6xYk37hx0tW2sYfgbO0PqStbHqz6Bq6oc9Hurg=
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/tarantool/go-iproto v1.1.0 h1:HULVOIHsiehI+FnHfM7wMDntuzUddO09DKqu2WnFQ5A=
github.com/tarantool/go-iproto v1.1.0/go.mod h1:LNCtdyZxojUed8SbOiYHoc3v9NvaZTB7p96hUySMlIo=
@@ -1961,8 +1959,8 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GR
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA=
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g=
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s=
github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/tinylib/msgp v1.5.0 h1:GWnqAE54wmnlFazjq2+vgr736Akg58iiHImh+kPY2pc=
github.com/tinylib/msgp v1.5.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o=
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
@@ -2028,10 +2026,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
@@ -2064,14 +2058,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 h1:avIdi8eGXjKbn1WLokNR1Ofnz1k8t7tJ88YQLD/iCi8=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2 h1:Bzra5hUQIWrxknQ0oV6HE5q/eZU1VhHMKSZ/XWpqMgM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2/go.mod h1:VYUUkRJkKuQPkIpgtZJj6+58Fa2g8ccAqdmaaK6HP5k=
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3 h1:hTwpF+PdbuR7vcixN+4AC6yu4asaUIAljQCxL51eDII=
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c=
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
@@ -2098,15 +2092,13 @@ github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
github.com/zeebo/mwc v0.0.7 h1:0NerGhCww6ZQx+/xCx5iwznftveokvto1KILpYfENZk=
github.com/zeebo/mwc v0.0.7/go.mod h1:0B32or6moOig1YGuqMoimBpU9QK9uYaGG2bBOuddqtE=
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
@@ -2130,14 +2122,14 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 h1:0tY123n7CdWMem7MOVdKOt0YfshufLCwfE5Bob+hQuM=
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
@@ -2148,18 +2140,18 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4D
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
@@ -2216,8 +2208,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2233,8 +2225,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -2248,8 +2240,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -2283,8 +2275,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2352,8 +2344,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2511,11 +2503,11 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4=
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0=
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2531,8 +2523,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2553,8 +2545,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2632,8 +2624,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2711,8 +2703,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E=
google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ=
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -2846,12 +2838,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -2892,8 +2884,8 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
@@ -2983,19 +2975,19 @@ modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=
modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8=
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU=
modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0=
modernc.org/ccgo/v4 v4.30.2 h1:4yPaaq9dXYXZ2V8s1UgrC3KIj580l2N4ClrLwnbv2so=
modernc.org/ccgo/v4 v4.30.2/go.mod h1:yZMnhWEdW0qw3EtCndG1+ldRrVGS+bIwyWmAWzS0XEw=
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
@@ -3010,8 +3002,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ=
modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0=
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
@@ -3030,8 +3022,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU=
modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
@@ -3054,8 +3046,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aN
sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
storj.io/common v0.0.0-20260225132117-99155641c30a h1:7gSBQY3vhQMIqi3vfaEXR7mreRjcBLfVsYY0rHIN7P0=
storj.io/common v0.0.0-20260225132117-99155641c30a/go.mod h1:kyxwKwlfH4paBZCZt/szvRB770ieAIJF73iDy2DpEHw=
storj.io/common v0.0.0-20251107171817-6221ae45072c h1:UDXSrdeLJe3QFouavSW10fYdpclK0YNu3KvQHzqq2+k=
storj.io/common v0.0.0-20251107171817-6221ae45072c/go.mod h1:XNX7uykja6aco92y2y8RuqaXIDRPpt1YA2OQDKlKEUk=
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 h1:8OE12DvUnB9lfZcHe7IDGsuhjrY9GBAr964PVHmhsro=
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 h1:5MZ0CyMbG6Pi0rRzUWVG6dvpXjbBYEX2oyXuj+tT+sk=
@@ -3064,7 +3056,7 @@ storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q=
storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs=
storj.io/picobuf v0.0.4 h1:qswHDla+YZ2TovGtMnU4astjvrADSIz84FXRn0qgP6o=
storj.io/picobuf v0.0.4/go.mod h1:hSMxmZc58MS/2qSLy1I0idovlO7+6K47wIGUyRZa6mg=
storj.io/uplink v1.14.0 h1:J1yXlt0aRr6kgLTHWXOWosNCFVfbamlcyd+CSxyIczo=
storj.io/uplink v1.14.0/go.mod h1:2ysmjzd/1Xtz4VKoErNcSqBQz3UC9WKTVuLMV1cNu6E=
storj.io/uplink v1.13.1 h1:C8RdW/upALoCyuF16Lod9XGCXEdbJAS+ABQy9JO/0pA=
storj.io/uplink v1.13.1/go.mod h1:x0MQr4UfFsQBwgVWZAtEsLpuwAn6dg7G0Mpne1r516E=
tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc=
tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.25"
appVersion: "4.18"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.25.0
version: 4.18.0
-29
View File
@@ -49,35 +49,6 @@ CREATE TABLE IF NOT EXISTS `filemeta` (
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
#### RocksDB variant
The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
To use this image with the Helm chart, override the image on all three components and disable
the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
override the image's built-in RocksDB configuration:
```yaml
# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
master:
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
volume:
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
filer:
enablePVC: true
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
extraEnvironmentVars:
WEED_LEVELDB2_ENABLED: "false"
```
Notes:
* `master` and `volume` use the same image tag so that all components share a consistent
SeaweedFS build; RocksDB itself is only used by the filer.
* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
### Node Labels
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
@@ -82,7 +82,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.admin.image" . }}
image: {{ template "admin.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
{{- $adminAuthEnabled := or .Values.admin.secret.existingSecret .Values.admin.secret.adminPassword }}
{{- $urlPrefix := .Values.admin.urlPrefix }}
@@ -242,7 +242,7 @@ spec:
securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.admin.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
{{- if eq .Values.admin.data.type "hostPath" }}
@@ -303,7 +303,7 @@ spec:
nodeSelector:
{{ tpl .Values.admin.nodeSelector . | indent 8 | trim }}
{{- end }}
{{- $pvc_exists := include "seaweedfs.admin.pvc_exists" . -}}
{{- $pvc_exists := include "admin.pvc_exists" . -}}
{{- if $pvc_exists }}
volumeClaimTemplates:
{{- if eq .Values.admin.data.type "persistentVolumeClaim" }}
@@ -77,7 +77,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.master.image" . }}
image: {{ template "master.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
env:
{{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
@@ -247,8 +247,8 @@ spec:
{{- $httpsPort := .Values.allInOne.s3.httpsPort | default .Values.s3.httpsPort }}
{{- if $httpsPort }}
-s3.port.https={{ $httpsPort }} \
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
{{- end }}
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
{{- end }}
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
@@ -418,7 +418,7 @@ spec:
{{- omit .Values.allInOne.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.allInOne.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
- name: data
@@ -164,7 +164,7 @@ spec:
securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.cosi.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
- name: socket
@@ -86,7 +86,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.filer.image" . }}
image: {{ template "filer.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
env:
- name: POD_IP
@@ -200,8 +200,8 @@ spec:
{{- if .Values.global.seaweedfs.enableSecurity }}
{{- if .Values.filer.s3.httpsPort }}
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
{{- end }}
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
{{- end }}
{{- if .Values.filer.s3.enableAuth }}
-s3.config=/etc/sw/seaweedfs_s3_config \
@@ -317,7 +317,7 @@ spec:
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.filer.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
{{- if eq .Values.filer.logs.type "hostPath" }}
@@ -413,7 +413,7 @@ spec:
storageClassName: {{ .Values.filer.storageClass }}
{{- end }}
{{- end }}
{{- $pvc_exists := include "seaweedfs.filer.pvc_exists" . -}}
{{- $pvc_exists := include "filer.pvc_exists" . -}}
{{- if $pvc_exists }}
volumeClaimTemplates:
{{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
@@ -80,7 +80,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.master.image" . }}
image: {{ template "master.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
env:
- name: POD_IP
@@ -251,7 +251,7 @@ spec:
securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.master.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
{{- if eq .Values.master.logs.type "hostPath" }}
@@ -312,7 +312,7 @@ spec:
nodeSelector:
{{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
{{- end }}
{{- $pvc_exists := include "seaweedfs.master.pvc_exists" . -}}
{{- $pvc_exists := include "master.pvc_exists" . -}}
{{- if $pvc_exists }}
volumeClaimTemplates:
{{- if eq .Values.master.data.type "persistentVolumeClaim"}}
@@ -74,7 +74,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.s3.image" . }}
image: {{ template "s3.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
env:
- name: POD_IP
@@ -127,8 +127,8 @@ spec:
{{- if .Values.global.seaweedfs.enableSecurity }}
{{- if .Values.s3.httpsPort }}
-port.https={{ .Values.s3.httpsPort }} \
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
{{- end }}
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
{{- end }}
{{- if .Values.s3.domainName }}
-domainName={{ .Values.s3.domainName }} \
@@ -226,7 +226,7 @@ spec:
securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.s3.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
{{- if .Values.s3.enableAuth }}
@@ -15,15 +15,15 @@
{{- $access_key_admin := $adminCreds.accessKey -}}
{{- $secret_key_admin := $adminCreds.secretKey -}}
{{- if not (and $access_key_admin $secret_key_admin) -}}
{{- $access_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- $secret_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- end -}}
{{- $readCreds := $creds.read | default dict -}}
{{- $access_key_read := $readCreds.accessKey -}}
{{- $secret_key_read := $readCreds.secretKey -}}
{{- if not (and $access_key_read $secret_key_read) -}}
{{- $access_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- $secret_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
{{- end -}}
apiVersion: v1
kind: Secret
@@ -1,10 +1,7 @@
{{- include "seaweedfs.compat" . -}}
{{- /*
The seaweedfs-s3 Service only gains a "metrics" port when the standalone S3
gateway is enabled. With only the embedded filer S3 gateway, metrics live on
the filer process and are already scraped by the filer ServiceMonitor.
*/ -}}
{{- if and .Values.s3.enabled .Values.s3.metricsPort .Values.global.seaweedfs.monitoring.enabled }}
{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
{{- if .Values.s3.metricsPort }}
{{- if .Values.global.seaweedfs.monitoring.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
@@ -34,3 +31,5 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: s3
{{- end }}
{{- end }}
{{- end }}
@@ -74,7 +74,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.sftp.image" . }}
image: {{ template "sftp.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
env:
- name: POD_IP
@@ -233,7 +233,7 @@ spec:
securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.sftp.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
{{- if .Values.sftp.enableAuth }}
@@ -1,8 +1,8 @@
{{- if or .Values.sftp.enabled .Values.allInOne.enabled }}
{{- $secretName := printf "%s-sftp-secret" (include "seaweedfs.fullname" .) }}
{{- $admin_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" "length" 20) -}}
{{- $read_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" "length" 20) -}}
{{- $public_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" "length" 20) -}}
{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" 20) -}}
{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" 20) -}}
{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" 20) -}}
apiVersion: v1
kind: Secret
type: Opaque
@@ -72,77 +72,77 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{/* Return the proper filer image */}}
{{- define "seaweedfs.filer.image" -}}
{{- define "filer.image" -}}
{{- if .Values.filer.imageOverride -}}
{{- $imageOverride := .Values.filer.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- include "seaweedfs.image" . }}
{{- include "common.image" . }}
{{- end -}}
{{- end -}}
{{/* Return the proper master image */}}
{{- define "seaweedfs.master.image" -}}
{{- define "master.image" -}}
{{- if .Values.master.imageOverride -}}
{{- $imageOverride := .Values.master.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- include "seaweedfs.image" . }}
{{- include "common.image" . }}
{{- end -}}
{{- end -}}
{{/* Return the proper s3 image */}}
{{- define "seaweedfs.s3.image" -}}
{{- define "s3.image" -}}
{{- if .Values.s3.imageOverride -}}
{{- $imageOverride := .Values.s3.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- include "seaweedfs.image" . }}
{{- include "common.image" . }}
{{- end -}}
{{- end -}}
{{/* Return the proper sftp image */}}
{{- define "seaweedfs.sftp.image" -}}
{{- define "sftp.image" -}}
{{- if .Values.sftp.imageOverride -}}
{{- $imageOverride := .Values.sftp.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- include "seaweedfs.image" . }}
{{- include "common.image" . }}
{{- end -}}
{{- end -}}
{{/* Return the proper admin image */}}
{{- define "seaweedfs.admin.image" -}}
{{- define "admin.image" -}}
{{- if .Values.admin.imageOverride -}}
{{- $imageOverride := .Values.admin.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- include "seaweedfs.image" . }}
{{- include "common.image" . }}
{{- end -}}
{{- end -}}
{{/* Return the proper worker image */}}
{{- define "seaweedfs.worker.image" -}}
{{- define "worker.image" -}}
{{- if .Values.worker.imageOverride -}}
{{- $imageOverride := .Values.worker.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- include "seaweedfs.image" . }}
{{- include "common.image" . }}
{{- end -}}
{{- end -}}
{{/* Return the proper volume image */}}
{{- define "seaweedfs.volume.image" -}}
{{- define "volume.image" -}}
{{- if .Values.volume.imageOverride -}}
{{- $imageOverride := .Values.volume.imageOverride -}}
{{- printf "%s" $imageOverride -}}
{{- else -}}
{{- include "seaweedfs.image" . }}
{{- include "common.image" . }}
{{- end -}}
{{- end -}}
{{/* Computes the container image name for all components (if they are not overridden) */}}
{{- define "seaweedfs.image" -}}
{{- define "common.image" -}}
{{- $registryName := default .Values.image.registry .Values.global.imageRegistry | toString -}}
{{- $repositoryName := default .Values.image.repository .Values.global.seaweedfs.image.repository | toString -}}
{{- $name := .Values.global.seaweedfs.image.name | toString -}}
@@ -160,7 +160,7 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{/* check if any Volume PVC exists */}}
{{- define "seaweedfs.volume.pvc_exists" -}}
{{- define "volume.pvc_exists" -}}
{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
{{- printf "true" -}}
{{- else -}}
@@ -169,7 +169,7 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{/* check if any Filer PVC exists */}}
{{- define "seaweedfs.filer.pvc_exists" -}}
{{- define "filer.pvc_exists" -}}
{{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
{{- printf "true" -}}
{{- else -}}
@@ -178,7 +178,7 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{/* check if any Master PVC exists */}}
{{- define "seaweedfs.master.pvc_exists" -}}
{{- define "master.pvc_exists" -}}
{{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
{{- printf "true" -}}
{{- else -}}
@@ -187,7 +187,7 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{/* check if any Admin PVC exists */}}
{{- define "seaweedfs.admin.pvc_exists" -}}
{{- define "admin.pvc_exists" -}}
{{- if or (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.logs.type "persistentVolumeClaim") -}}
{{- printf "true" -}}
{{- else -}}
@@ -196,7 +196,7 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{/* check if any InitContainers exist for Volumes */}}
{{- define "seaweedfs.volume.initContainers_exists" -}}
{{- define "volume.initContainers_exists" -}}
{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
{{- printf "true" -}}
{{- else -}}
@@ -225,10 +225,10 @@ imagePullSecrets:
{{/*
Renders a value that contains template perhaps with scope if the scope is present.
Usage:
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
*/}}
{{- define "seaweedfs.tplvalues.render" -}}
{{- define "common.tplvalues.render" -}}
{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
{{- if contains "{{" (toJson .value) }}
{{- if .scope }}
@@ -245,9 +245,9 @@ Usage:
Converts a Kubernetes quantity like "256Mi" or "2G" to a float64 in base units,
handling both binary (Ki, Mi, Gi) and decimal (m, k, M) suffixes; numeric inputs
Usage:
{{ include "seaweedfs.resource-quantity" "10Gi" }}
{{ include "common.resource-quantity" "10Gi" }}
*/}}
{{- define "seaweedfs.resource-quantity" -}}
{{- define "common.resource-quantity" -}}
{{- $value := . -}}
{{- $unit := 1.0 -}}
{{- if typeIs "string" . -}}
@@ -267,7 +267,7 @@ Usage:
getOrGeneratePassword will check if a password exists in a secret and return it,
or generate a new random password if it doesn't exist.
*/}}
{{- define "seaweedfs.getOrGeneratePassword" -}}
{{- define "getOrGeneratePassword" -}}
{{- $params := . -}}
{{- $namespace := $params.namespace -}}
{{- $secretName := $params.secretName -}}
@@ -68,7 +68,7 @@ spec:
{{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
containers:
- name: post-install-job
image: {{ template "seaweedfs.master.image" . }}
image: {{ template "master.image" . }}
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
env:
- name: WEED_CLUSTER_DEFAULT
@@ -35,8 +35,8 @@
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
{{- if and $currentPVC }}
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
{{- $oldSize := include "common.resource-quantity" $currentPVC.spec.resources.requests.storage }}
{{- $newSize := include "common.resource-quantity" $desiredSize }}
{{- if gt $newSize $oldSize }}
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
{{- end }}
@@ -71,12 +71,12 @@ spec:
{{- end }}
enableServiceLinks: false
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
{{- $initContainers_exists := include "seaweedfs.volume.initContainers_exists" $ -}}
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
{{- if $initContainers_exists }}
initContainers:
{{- if $volume.idx }}
- name: seaweedfs-vol-move-idx
image: {{ template "seaweedfs.volume.image" $ }}
image: {{ template "volume.image" $ }}
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
command: [ '/bin/sh', '-c' ]
args: [ '{{range $dir := $volume.dataDirs }}if ls /{{$dir.name}}/*.idx >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
@@ -104,7 +104,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.volume.image" $ }}
image: {{ template "volume.image" $ }}
imagePullPolicy: {{ default "IfNotPresent" $.Values.global.seaweedfs.imagePullPolicy }}
env:
- name: POD_NAME
@@ -274,7 +274,7 @@ spec:
securityContext: {{- omit $volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if $volume.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
{{- end }}
volumes:
@@ -77,7 +77,7 @@ spec:
{{- end }}
containers:
- name: seaweedfs
image: {{ template "seaweedfs.worker.image" . }}
image: {{ template "worker.image" . }}
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
env:
- name: POD_IP
@@ -219,7 +219,7 @@ spec:
securityContext: {{- omit .Values.worker.containerSecurityContext "enabled" | toYaml | nindent 12 }}
{{- end }}
{{- if .Values.worker.sidecars }}
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
{{- include "common.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
{{- end }}
volumes:
{{- if eq .Values.worker.data.type "hostPath" }}
+3 -8
View File
@@ -1339,14 +1339,9 @@ worker:
# Admin server to connect to
adminServer: ""
# Worker job types - categories or comma-separated list of names/aliases
# Categories: all, default, heavy
# default: vacuum, volume_balance, ec_balance, admin_script
# heavy: erasure_coding, iceberg_maintenance
# all: every registered job type
# Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
# Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
jobType: "all"
# Worker job types - comma-separated list
# Available: vacuum, volume_balance, erasure_coding
jobType: "vacuum,volume_balance,erasure_coding"
# Maximum number of concurrent detection requests
maxDetect: 1
-31
View File
@@ -1,31 +0,0 @@
MARP ?= npx @marp-team/marp-cli
SRC := seaweedfs-production-setup.md
PDF := $(SRC:.md=.pdf)
HTML := $(SRC:.md=.html)
PPTX := $(SRC:.md=.pptx)
FLAGS := --allow-local-files
.PHONY: all pdf html pptx watch clean
all: pdf
pdf: $(PDF)
html: $(HTML)
pptx: $(PPTX)
$(PDF): $(SRC)
$(MARP) $(SRC) --pdf $(FLAGS) -o $@
$(HTML): $(SRC)
$(MARP) $(SRC) --html $(FLAGS) -o $@
$(PPTX): $(SRC)
$(MARP) $(SRC) --pptx $(FLAGS) -o $@
watch:
$(MARP) $(SRC) --watch --html $(FLAGS)
clean:
rm -f $(PDF) $(HTML) $(PPTX)
Binary file not shown.

Before

Width:  |  Height:  |  Size: 738 KiB

-118
View File
@@ -1,118 +0,0 @@
---
marp: true
theme: default
paginate: true
---
# SeaweedFS Production Setup
A walkthrough of the recommended deployment topology
- Storage layer
- File access layer
- Backend operations
- Erasure coding for durability
---
![bg contain](object-store-layout.png)
---
# Storage Layer
**3 Masters + Volume Servers**
- **Masters (run 3)**
- Form a Raft quorum; tolerate 1 failure
- Track volume locations and cluster topology
- Lightweight; not on the data path for reads/writes after lookup
- **Volume Servers (scale out)**
- Hold the actual needle data on local disks
- Add more servers for capacity and throughput
- Heartbeat to masters; clients talk to them directly for I/O
Rule of thumb: start with 3 masters, then grow volume servers with data.
---
# File Access Layer
**Filer + DB for metadata**
- Filer serves POSIX-like directory and file metadata
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
- Run **3 filers** for HA
- Filers are stateless relative to each other — state lives in the DB
**S3 Servers**
- Expose the S3 API on top of the filer
- Scale the count based on throughput requirements
- Can be co-located with filers or run independently
---
# Backend Operations
**1 Admin Server + a few Workers**
- **Admin server (1 is enough)**
- Coordinates background jobs: balancing, EC encoding, vacuum, replication fixes
- **Not on the data path** — a restart does not affect object store reads/writes
- No HA required for normal operation
- **Workers (a few)**
- Execute long-running tasks issued by the admin server
- Horizontally scalable based on maintenance workload
Keeps housekeeping off the hot path.
---
# Production Topology Summary
| Component | Count | Role |
|----------------|----------------------|-----------------------------------|
| Master | 3 | Raft quorum, topology, volume map |
| Volume Server | N (scale out) | Stores data needles |
| Filer | 3 | Metadata gateway |
| Metadata DB | HA cluster | Persists filer metadata |
| S3 Server | N (by throughput) | S3 API frontend |
| Admin | 1 | Background job coordinator |
| Worker | A few | Executes admin-scheduled tasks |
---
# Erasure Coding for Durability
Prefer **many volume servers** so EC shards spread across failure domains.
**Example: 8 volume servers, EC 5+3**
- Each volume becomes 5 data shards + 3 parity shards
- Shards are placed on 8 distinct volume servers
- **Tolerates up to 3 simultaneous volume server failures**
- Storage overhead: 1.6x vs 3x for full replication
```
Volume -> [D1][D2][D3][D4][D5] + [P1][P2][P3]
| | | | | | | |
VS1 VS2 VS3 VS4 VS5 VS6 VS7 VS8
```
More volume servers = more EC layouts available (e.g. 10+4, 6+3).
---
# Getting Started Checklist
1. Provision 3 master nodes (small boxes are fine)
2. Provision volume servers sized to your data footprint
3. Stand up an HA metadata DB, then 3 filers pointing at it
4. Add S3 servers sized to required throughput
5. Run 1 admin + a few workers for background maintenance
6. Once you have enough volume servers, enable EC (e.g. 5+3 on 8 servers)
Scale each layer independently as usage grows.
@@ -86,13 +86,6 @@ service SeaweedFiler {
}
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
}
// Peer chunk sharing — tier 1: mount-server registry.
// See design-weed-mount-peer-chunk-sharing.md for details.
rpc MountRegister (MountRegisterRequest) returns (MountRegisterResponse) {
}
rpc MountList (MountListRequest) returns (MountListResponse) {
}
}
//////////////////////////////////////////////////
@@ -204,10 +197,6 @@ message FuseAttributes {
bytes md5 = 14;
uint32 rdev = 16;
uint64 inode = 17;
int64 ctime = 18; // unix time in seconds, inode change time
int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
int32 crtime_ns = 21; // nanosecond component of crtime (0-999999999)
}
message CreateEntryRequest {
@@ -305,7 +294,6 @@ message AssignVolumeRequest {
string rack = 7;
string data_node = 9;
string disk_type = 8;
uint64 expected_data_size = 10; // hint for size-aware volume selection
}
message AssignVolumeResponse {
@@ -608,31 +596,3 @@ message StreamMutateEntryResponse {
string error = 7; // human-readable error message when the operation failed
int32 errno = 8; // POSIX errno (e.g. ENOENT=2, ENOTEMPTY=66) for direct FUSE status mapping
}
//////////////////////////////////////////////////
// Peer chunk sharing — mount-server registry
//////////////////////////////////////////////////
message MountRegisterRequest {
string peer_addr = 1; // host:port where this mount serves peer chunk requests
string rack = 2; // locality label (rack); used for peer ranking
int32 ttl_seconds = 3; // how long the filer should keep this entry without a heartbeat
string data_center = 4; // locality label (data center); coarser than rack
}
message MountRegisterResponse {
}
message MountListRequest {
}
message MountListResponse {
repeated MountInfo mounts = 1;
}
message MountInfo {
string peer_addr = 1;
string rack = 2;
int64 last_seen_ns = 3;
string data_center = 4;
}
+2 -2
View File
@@ -3350,9 +3350,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
version = "0.103.13"
version = "0.103.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
dependencies = [
"aws-lc-rs",
"ring",
-6
View File
@@ -145,11 +145,6 @@ message VolumeEcShardInformationMessage {
uint64 expire_at_sec = 5; // used to record the destruction time of ec volume
uint32 disk_id = 6;
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
uint64 file_count = 8; // total needles in the .ecx index (live + tombstoned)
uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
// fields 10-19 reserved for future upstream open-source additions.
// fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
// and must not be used here without coordination.
}
message StorageBackend {
@@ -236,7 +231,6 @@ message AssignRequest {
uint32 memory_map_max_size_mb = 8;
uint32 writable_volume_count = 9;
string disk_type = 10;
uint64 expected_data_size = 11; // hint for size-aware volume selection
}
message VolumeGrowRequest {
-6
View File
@@ -247,9 +247,6 @@ message VolumeUnmountResponse {
message VolumeDeleteRequest {
uint32 volume_id = 1;
bool only_empty = 2;
// when true, do not remove the cloud-tier object backing the volume.
// used for moves where another server is taking over the same .vif.
bool keep_remote_data = 3;
}
message VolumeDeleteResponse {
}
@@ -342,7 +339,6 @@ message ReceiveFileInfo {
bool is_ec_volume = 4;
uint32 shard_id = 5;
uint64 file_size = 6;
uint32 disk_id = 7; // EC shard disk; 0 = auto-select (see VolumeEcShardsCopyRequest.disk_id)
}
message ReceiveFileResponse {
@@ -458,7 +454,6 @@ message VolumeEcShardsMountRequest {
uint32 volume_id = 1;
string collection = 2;
repeated uint32 shard_ids = 3;
string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
}
message VolumeEcShardsMountResponse {
}
@@ -641,7 +636,6 @@ message FetchAndWriteNeedleRequest {
}
repeated Replica replicas = 6;
string auth = 7;
int32 download_concurrency = 8; // multipart download concurrency if supported by the remote storage client; for S3, 0 = default (5)
// remote conf
remote_pb.RemoteConf remote_conf = 15;
remote_pb.RemoteStorageLocation remote_location = 16;
+6 -27
View File
@@ -63,9 +63,7 @@ pub struct Cli {
#[arg(long = "rack", default_value = "")]
pub rack: String,
/// Choose [memory|redb|redbMedium|redbLarge] mode for memory~performance balance.
/// `leveldb`/`leveldbMedium`/`leveldbLarge` are accepted as aliases for the
/// corresponding redb backends (Rust volume server uses redb under the hood).
/// Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.
#[arg(long = "index", default_value = "memory")]
pub index: String,
@@ -703,16 +701,14 @@ fn resolve_config(cli: Cli) -> VolumeServerConfig {
ip.clone()
};
// Parse index type. Accept both `redb*` (preferred — what the volume server
// actually uses) and the legacy `leveldb*` names as aliases.
// Parse index type
let index_type = match cli.index.as_str() {
"memory" => NeedleMapKind::InMemory,
"redb" | "leveldb" => NeedleMapKind::Redb,
"redbMedium" | "leveldbMedium" => NeedleMapKind::RedbMedium,
"redbLarge" | "leveldbLarge" => NeedleMapKind::RedbLarge,
"leveldb" => NeedleMapKind::LevelDb,
"leveldbMedium" => NeedleMapKind::LevelDbMedium,
"leveldbLarge" => NeedleMapKind::LevelDbLarge,
other => panic!(
"Unknown index type: {}. Use memory|redb|redbMedium|redbLarge \
(leveldb/leveldbMedium/leveldbLarge accepted as aliases)",
"Unknown index type: {}. Use memory|leveldb|leveldbMedium|leveldbLarge",
other
),
};
@@ -1378,23 +1374,6 @@ mod tests {
assert_eq!(cfg.folders, vec![default_volume_dir()]);
}
#[test]
fn test_resolve_config_index_accepts_redb_and_leveldb_aliases() {
let pairs = [
("memory", NeedleMapKind::InMemory),
("redb", NeedleMapKind::Redb),
("leveldb", NeedleMapKind::Redb),
("redbMedium", NeedleMapKind::RedbMedium),
("leveldbMedium", NeedleMapKind::RedbMedium),
("redbLarge", NeedleMapKind::RedbLarge),
("leveldbLarge", NeedleMapKind::RedbLarge),
];
for (input, expected) in pairs {
let cfg = resolve_config(Cli::parse_from(["bin", "--index", input]));
assert_eq!(cfg.index_type, expected, "input={}", input);
}
}
#[test]
fn test_parse_security_config_access_ui() {
let _guard = process_state_lock();
+1 -7
View File
@@ -288,10 +288,6 @@ async fn run(
config.jwt_read_signing_expires_seconds,
);
let master_url = config.masters.first().cloned().unwrap_or_default();
// Defensive-copy the configured seed masters before freezing the lookup
// set, so any later mutation of config.masters cannot desync them.
let master_urls: Vec<String> = config.masters.clone();
let seed_master_set = VolumeServerState::build_seed_master_set(&master_urls);
let self_url = format!("{}:{}", config.ip, config.port);
let (http_client, outgoing_http_scheme) = build_outgoing_http_client(&config)?;
let outgoing_grpc_tls = load_outgoing_grpc_tls(&config)?;
@@ -328,9 +324,7 @@ async fn run(
),
read_mode: config.read_mode,
master_url,
master_urls,
seed_master_set,
current_master_url: tokio::sync::RwLock::new(String::new()),
master_urls: config.masters.clone(),
self_url,
http_client,
outgoing_http_scheme,
-32
View File
@@ -142,35 +142,6 @@ lazy_static::lazy_static! {
&["code"],
).expect("metric can be created");
// ---- Scrubbing metrics (Go: VolumeServerScrub*) ----
/// Last scrub execution time, as seconds since UNIX epoch, with label `mode`.
pub static ref SCRUB_LAST_TIME_SECONDS: GaugeVec = GaugeVec::new(
Opts::new(
"SeaweedFS_volumeServer_scrub_last_time_seconds",
"Last scrub execution time, as seconds since UNIX epoch.",
),
&["mode"],
).expect("metric can be created");
/// Counter of overall volumes with issues detected during scrubbing, with label `mode`.
pub static ref SCRUB_VOLUME_FAILURES: IntCounterVec = IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_scrub_volume_failures",
"Counter of overall volumes with issues detected during scrubbing.",
),
&["mode"],
).expect("metric can be created");
/// Counter of overall EC shards with issues detected during scrubbing, with label `mode`.
pub static ref SCRUB_SHARD_FAILURES: IntCounterVec = IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_scrub_shard_failures",
"Counter of overall EC shards with issues detected during scrubbing.",
),
&["mode"],
).expect("metric can be created");
// ---- Legacy aliases for backward compat with existing code ----
/// Total number of volumes on this server (flat gauge).
@@ -280,9 +251,6 @@ pub fn register_metrics() {
Box::new(INFLIGHT_DOWNLOAD_SIZE.clone()),
Box::new(INFLIGHT_UPLOAD_SIZE.clone()),
Box::new(UPLOAD_ERROR_COUNTER.clone()),
Box::new(SCRUB_LAST_TIME_SECONDS.clone()),
Box::new(SCRUB_VOLUME_FAILURES.clone()),
Box::new(SCRUB_SHARD_FAILURES.clone()),
// Legacy metrics
Box::new(VOLUMES_TOTAL.clone()),
Box::new(DISK_SIZE_BYTES.clone()),
@@ -486,11 +486,6 @@ impl S3TierRegistry {
self.backends.keys().cloned().collect()
}
/// Remove a backend by name.
pub fn remove(&mut self, name: &str) {
self.backends.remove(name);
}
pub fn clear(&mut self) {
self.backends.clear();
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.6 KiB

After

Width:  |  Height:  |  Size: 70 B

+68 -462
View File
@@ -16,7 +16,6 @@ use crate::pb::master_pb;
use crate::pb::master_pb::seaweed_client::SeaweedClient;
use crate::pb::volume_server_pb;
use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
use crate::storage::needle::needle::{self, Needle};
use crate::storage::types::*;
@@ -29,42 +28,6 @@ fn volume_is_remote_only(dat_path: &str, has_remote_file: bool) -> bool {
has_remote_file && !std::path::Path::new(dat_path).exists()
}
/// Map a numeric `VolumeScrubMode` to its proto enum name, matching Go's
/// `req.GetMode().String()` used for the Prometheus `mode` label.
fn scrub_mode_label(mode: i32) -> &'static str {
match mode {
0 => "UNKNOWN",
1 => "INDEX",
2 => "FULL",
3 => "LOCAL",
_ => "UNKNOWN",
}
}
fn unix_now_seconds() -> f64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as f64)
.unwrap_or(0.0)
}
/// Record scrub metrics. `broken_shards` is `Some` only for EC scrubs so the
/// shard-failures family stays untouched on regular volume scrubs (matching Go).
fn emit_scrub_metrics(mode: i32, broken_volumes: usize, broken_shards: Option<usize>) {
let mode_label = scrub_mode_label(mode);
crate::metrics::SCRUB_LAST_TIME_SECONDS
.with_label_values(&[mode_label])
.set(unix_now_seconds());
crate::metrics::SCRUB_VOLUME_FAILURES
.with_label_values(&[mode_label])
.inc_by(broken_volumes as u64);
if let Some(n) = broken_shards {
crate::metrics::SCRUB_SHARD_FAILURES
.with_label_values(&[mode_label])
.inc_by(n as u64);
}
}
/// Persist VolumeServerState to a state.pb file (matches Go's State.save).
fn save_state_file(
path: &str,
@@ -145,37 +108,6 @@ pub struct VolumeGrpcService {
}
impl VolumeGrpcService {
/// Verifies the gRPC caller is allowed to invoke a destructive admin
/// operation. Mirrors the Go side's checkGrpcAdminAuth: an empty
/// whitelist accepts everyone (insecure-by-default for tests and
/// upgrades), a populated whitelist accepts only matching peer IPs.
///
/// `remote_addr()` on a real gRPC connection always yields the peer's
/// SocketAddr; if it is somehow None we deny, matching "if we don't
/// know who the caller is, refuse."
fn check_grpc_admin_auth<T>(&self, request: &Request<T>) -> Result<(), Status> {
let remote = match request.remote_addr() {
Some(addr) => addr,
None => {
tracing::warn!("gRPC admin auth failed: no peer info");
return Err(Status::permission_denied("no peer info"));
}
};
let host = remote.ip().to_string();
let guard = self.state.guard.read().unwrap();
if !guard.check_whitelist(&host) {
tracing::warn!(
"gRPC admin auth failed: {} is not whitelisted (remote: {})",
host,
remote,
);
return Err(Status::permission_denied(format!(
"not authorized: {host}"
)));
}
Ok(())
}
async fn notify_master_volume_readonly(
&self,
info: &MasterVolumeInfo,
@@ -490,7 +422,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VacuumVolumeCompactRequest>,
) -> Result<Response<Self::VacuumVolumeCompactStream>, Status> {
self.check_grpc_admin_auth(&request)?;
self.state.check_maintenance()?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
@@ -550,7 +481,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VacuumVolumeCommitRequest>,
) -> Result<Response<volume_server_pb::VacuumVolumeCommitResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
self.state.check_maintenance()?;
let vid = VolumeId(request.into_inner().volume_id);
@@ -586,7 +516,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VacuumVolumeCleanupRequest>,
) -> Result<Response<volume_server_pb::VacuumVolumeCleanupResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
self.state.check_maintenance()?;
let vid = VolumeId(request.into_inner().volume_id);
let mut store = self.state.store.write().unwrap();
@@ -602,7 +531,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::DeleteCollectionRequest>,
) -> Result<Response<volume_server_pb::DeleteCollectionResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
let collection = &request.into_inner().collection;
let mut store = self.state.store.write().unwrap();
store
@@ -615,7 +543,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::AllocateVolumeRequest>,
) -> Result<Response<volume_server_pb::AllocateVolumeResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
self.state.check_maintenance()?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
@@ -764,7 +691,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VolumeMountRequest>,
) -> Result<Response<volume_server_pb::VolumeMountResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
@@ -781,7 +707,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VolumeUnmountRequest>,
) -> Result<Response<volume_server_pb::VolumeUnmountResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
let vid = VolumeId(request.into_inner().volume_id);
let mut store = self.state.store.write().unwrap();
// Go returns nil when volume is not found (idempotent unmount)
@@ -795,7 +720,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VolumeDeleteRequest>,
) -> Result<Response<volume_server_pb::VolumeDeleteResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
self.state.check_maintenance()?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
@@ -809,7 +733,7 @@ impl VolumeServer for VolumeGrpcService {
}
}
store
.delete_volume(vid, req.only_empty, req.keep_remote_data)
.delete_volume(vid, req.only_empty)
.map_err(|e| Status::internal(e.to_string()))?;
self.state.volume_state_notify.notify_one();
Ok(Response::new(volume_server_pb::VolumeDeleteResponse {}))
@@ -819,7 +743,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VolumeMarkReadonlyRequest>,
) -> Result<Response<volume_server_pb::VolumeMarkReadonlyResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
// Go: volume lookup (L239-241) happens before maintenance check (L166 in makeVolumeReadonly)
@@ -839,7 +762,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VolumeMarkWritableRequest>,
) -> Result<Response<volume_server_pb::VolumeMarkWritableResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
let info = {
@@ -889,7 +811,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VolumeConfigureRequest>,
) -> Result<Response<volume_server_pb::VolumeConfigureResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
self.state.check_maintenance()?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
@@ -1043,10 +964,7 @@ impl VolumeServer for VolumeGrpcService {
if store.find_volume(vid).is_some() {
drop(store);
let mut store = self.state.store.write().unwrap();
// keep remote data: the inbound copy carries a .vif that may
// point at the same cloud-tier object the existing volume
// references.
store.delete_volume(vid, false, true).map_err(|e| {
store.delete_volume(vid, false).map_err(|e| {
Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
})?;
self.state.volume_state_notify.notify_one();
@@ -1507,45 +1425,13 @@ impl VolumeServer for VolumeGrpcService {
// Determine file path
let path = if info.is_ec_volume {
let store = self.state.store.read().unwrap();
// std::fs::File::create truncates in place; a mounted
// EcVolume holds fds on the same inodes, so overwriting
// corrupts live readers.
if store.has_ec_volume(VolumeId(info.volume_id)) {
resp_error = Some(format!(
"ec volume {} is mounted; unmount before ReceiveFile",
info.volume_id
));
break;
}
// disk_id=0 means "unset" (protobuf default), so auto-select
// using the same primitive as volume_ec_shards_copy: prefer
// a disk that has the EC volume mounted, then a disk that
// owns the .ecx on disk (volume not yet mounted — relevant
// when shards stream in mid-rebuild before any
// VolumeEcShardsMount has happened; see #9212), then any
// HDD, then any disk. Pass the build's default data-shard
// count for free-slot maths; the helper takes it as a
// parameter so custom-ratio builds can swap it.
let vid = VolumeId(info.volume_id);
let dir = if info.disk_id > 0 {
let count = store.locations.len();
if (info.disk_id as usize) >= count {
resp_error = Some(format!(
"invalid disk_id {}: only have {} disks",
info.disk_id, count
));
break;
}
Some(store.locations[info.disk_id as usize].directory.clone())
} else {
store
.find_ec_shard_target_location(
&info.collection,
vid,
DATA_SHARDS_COUNT as u32,
)
.map(|i| store.locations[i].directory.clone())
};
// Go prefers a HardDriveType location, then falls back to first
let dir = store
.locations
.iter()
.find(|loc| loc.disk_type == DiskType::HardDrive)
.or_else(|| store.locations.first())
.map(|loc| loc.directory.clone());
drop(store);
let dir = match dir {
Some(d) => d,
@@ -2097,9 +1983,7 @@ impl VolumeServer for VolumeGrpcService {
// Check existing .vif for EC shard config (matching Go's MaybeLoadVolumeInfo)
let (data_shards, parity_shards) =
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
&dir, &idx_dir, collection, vid,
);
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(&dir, collection, vid);
if let Err(e) = crate::storage::erasure_coding::ec_encoder::write_ec_files(
&dir,
@@ -2253,7 +2137,6 @@ impl VolumeServer for VolumeGrpcService {
let (data_shards, parity_shards) =
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
&rebuild_dir,
&rebuild_idx_dir,
collection,
vid,
);
@@ -2337,17 +2220,9 @@ impl VolumeServer for VolumeGrpcService {
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
// Select target location:
// When disk_id > 0: use that specific location.
// When disk_id == 0 (unset): auto-select via
// find_ec_shard_target_location, which prefers a disk that
// already has the EC volume mounted, then a disk that owns the
// .ecx on disk (volume not yet mounted — relevant for
// ec.rebuild, where only the first shard carries .ecx and
// subsequent shards must land on the same disk; see #9212),
// then any HDD, then any disk. Pass the build's default
// data-shard count; the helper takes it as a parameter so
// custom-ratio builds can swap it.
// Select target location matching Go's 3-tier fallback:
// When disk_id > 0: use that specific location
// When disk_id == 0 (unset): (1) location with existing EC shards, (2) any HDD, (3) any
let (dest_dir, dest_idx_dir) = {
let store = self.state.store.read().unwrap();
let count = store.locations.len();
@@ -2363,11 +2238,20 @@ impl VolumeServer for VolumeGrpcService {
let loc = &store.locations[req.disk_id as usize];
(loc.directory.clone(), loc.idx_directory.clone())
} else {
match store.find_ec_shard_target_location(
&req.collection,
vid,
DATA_SHARDS_COUNT as u32,
) {
// Auto-select: prefer location with existing EC shards for this volume
let loc_idx = store
.find_free_location_predicate(|loc| loc.has_ec_volume(vid))
.or_else(|| {
// Fall back to any HDD location
store.find_free_location_predicate(|loc| {
loc.disk_type == DiskType::HardDrive
})
})
.or_else(|| {
// Fall back to any location
store.find_free_location_predicate(|_| true)
});
match loc_idx {
Some(i) => {
let loc = &store.locations[i];
(loc.directory.clone(), loc.idx_directory.clone())
@@ -2587,7 +2471,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::VolumeEcShardsDeleteRequest>,
) -> Result<Response<volume_server_pb::VolumeEcShardsDeleteResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
self.state.check_maintenance()?;
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
@@ -2612,7 +2495,7 @@ impl VolumeServer for VolumeGrpcService {
let mut store = self.state.store.write().unwrap();
for &shard_id in &req.shard_ids {
store
.mount_ec_shard(vid, &req.collection, shard_id, &req.source_disk_type)
.mount_ec_shard(vid, &req.collection, shard_id)
.map_err(|e| {
Status::internal(format!("mount {}.{}: {}", req.volume_id, shard_id, e))
})?;
@@ -2656,19 +2539,12 @@ impl VolumeServer for VolumeGrpcService {
let vid = VolumeId(req.volume_id);
let store = self.state.store.read().unwrap();
// Reconciled EC volumes can have their shards split across
// disks (e.g. shards 0/12 on disk 0, shard 1 on disk 1), so
// resolve the EcVolume from the *shard*'s home location
// rather than first-match `find_ec_volume(vid)` which would
// miss shards that live on a sibling. Mirrors Go's findEcShard.
let ec_vol = store
.find_ec_volume_with_shard(vid, req.shard_id)
.ok_or_else(|| {
Status::not_found(format!(
"ec volume {} shard {} not found",
req.volume_id, req.shard_id
))
})?;
let ec_vol = store.find_ec_volume(vid).ok_or_else(|| {
Status::not_found(format!(
"ec volume {} shard {} not found",
req.volume_id, req.shard_id
))
})?;
// Check if the requested needle is deleted (via .ecx index, matching Go)
if req.file_key > 0 {
@@ -2687,8 +2563,7 @@ impl VolumeServer for VolumeGrpcService {
}
}
// Read from the shard. Guaranteed to be present because
// find_ec_volume_with_shard already verified it.
// Read from the shard
let shard = ec_vol
.shards
.get(req.shard_id as usize)
@@ -2778,13 +2653,8 @@ impl VolumeServer for VolumeGrpcService {
let vid = VolumeId(req.volume_id);
let store = self.state.store.read().unwrap();
// Aggregate per-shard data dirs across all locations so the
// shard-presence check + decoder both see the union for
// cross-disk reconciled volumes (#9252). Mirrors Go's
// CollectEcShards.
let max_shard_count = crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT;
let (ec_vol, shard_dirs) = store
.collect_ec_shard_dirs(vid, max_shard_count)
let ec_vol = store
.find_ec_volume(vid)
.ok_or_else(|| Status::not_found(format!("ec volume {} not found", req.volume_id)))?;
if ec_vol.collection != req.collection {
@@ -2798,6 +2668,7 @@ impl VolumeServer for VolumeGrpcService {
let data_shards = ec_vol.data_shards as usize;
// Validate data shard count range (matches Go's VolumeEcShardsToVolume)
let max_shard_count = crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT;
if data_shards == 0 || data_shards > max_shard_count {
return Err(Status::invalid_argument(format!(
"invalid data shard count {} for volume {} (must be 1..{})",
@@ -2805,9 +2676,14 @@ impl VolumeServer for VolumeGrpcService {
)));
}
// Check that all data shards are present somewhere on this server.
// Check that all data shards are present
for shard_id in 0..data_shards {
if shard_dirs[shard_id].is_none() {
if ec_vol
.shards
.get(shard_id)
.map(|s| s.is_none())
.unwrap_or(true)
{
return Err(Status::internal(format!(
"ec volume {} missing shard {}",
req.volume_id, shard_id
@@ -2839,46 +2715,29 @@ impl VolumeServer for VolumeGrpcService {
)));
}
// Reconstruct the volume from EC shards. Use the EcVolume's
// own dir for the produced .dat (matches the volume's home
// disk) and its `ecx_actual_dir` for the .ecx lookup, while
// reading each shard from its real on-disk location.
let dat_dir = ec_vol.dir.clone();
let ecx_dir = ec_vol.ecx_actual_dir().to_string();
// Reconstruct the volume from EC shards
let dir = ec_vol.dir.clone();
let collection = ec_vol.collection.clone();
// shard_dirs[i] is guaranteed Some for i in 0..data_shards by
// the check above; collect concrete dirs for the decoder.
let per_shard_dirs: Vec<String> = shard_dirs[..data_shards]
.iter()
.map(|d| d.clone().unwrap())
.collect();
drop(store);
// Calculate .dat file size from .ecx entries (.ec00 lives on
// its own disk, .ecx on the index disk).
// Calculate .dat file size from .ecx entries
let dat_file_size =
crate::storage::erasure_coding::ec_decoder::find_dat_file_size_with_dirs(
&per_shard_dirs[0],
&ecx_dir,
&collection,
vid,
)
.map_err(|e| Status::internal(format!("FindDatFileSize: {}", e)))?;
crate::storage::erasure_coding::ec_decoder::find_dat_file_size(&dir, &collection, vid)
.map_err(|e| Status::internal(format!("FindDatFileSize: {}", e)))?;
// Write .dat file using block-interleaved reading from shards.
crate::storage::erasure_coding::ec_decoder::write_dat_file_from_shards_with_dirs(
&dat_dir,
// Write .dat file using block-interleaved reading from shards
crate::storage::erasure_coding::ec_decoder::write_dat_file_from_shards(
&dir,
&collection,
vid,
dat_file_size,
data_shards,
&per_shard_dirs,
)
.map_err(|e| Status::internal(format!("WriteDatFile: {}", e)))?;
// Write .idx file from .ecx and .ecj files (lives on idx dir).
// Write .idx file from .ecx and .ecj files
crate::storage::erasure_coding::ec_decoder::write_idx_file_from_ec_index(
&dat_dir,
&dir,
&collection,
vid,
)
@@ -3286,9 +3145,8 @@ impl VolumeServer for VolumeGrpcService {
async fn volume_server_leave(
&self,
request: Request<volume_server_pb::VolumeServerLeaveRequest>,
_request: Request<volume_server_pb::VolumeServerLeaveRequest>,
) -> Result<Response<volume_server_pb::VolumeServerLeaveResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
*self.state.is_stopping.write().unwrap() = true;
self.state.is_heartbeating.store(false, Ordering::Relaxed);
// Wake heartbeat loop to send deregistration.
@@ -3438,7 +3296,6 @@ impl VolumeServer for VolumeGrpcService {
&self,
request: Request<volume_server_pb::ScrubVolumeRequest>,
) -> Result<Response<volume_server_pb::ScrubVolumeResponse>, Status> {
self.check_grpc_admin_auth(&request)?;
let req = request.into_inner();
// Validate mode
@@ -3503,8 +3360,8 @@ impl VolumeServer for VolumeGrpcService {
// Match Go: if mark_broken_volumes_readonly, call makeVolumeReadonly on each broken volume.
// Collect errors via errors.Join semantics (return joined error if any fail).
let mut errs: Vec<String> = Vec::new();
if req.mark_broken_volumes_readonly {
let mut errs: Vec<String> = Vec::new();
for vid in &broken_vids {
match self.make_volume_readonly(*vid, true).await {
Ok(()) => {
@@ -3516,14 +3373,9 @@ impl VolumeServer for VolumeGrpcService {
}
}
}
}
// Record metrics before the post-scrub error check so scrub failures are
// persisted even when a follow-up admin action (mark-readonly) fails.
emit_scrub_metrics(mode, broken_vids.len(), None);
if !errs.is_empty() {
return Err(Status::internal(errs.join("\n")));
if !errs.is_empty() {
return Err(Status::internal(errs.join("\n")));
}
}
Ok(Response::new(volume_server_pb::ScrubVolumeResponse {
@@ -3593,15 +3445,9 @@ impl VolumeServer for VolumeGrpcService {
// LOCAL (2) / FULL (3): verify EC shard data
let files = ecv.walk_ecx_stats().map(|(f, _, _)| f).unwrap_or(0);
// After cross-disk reconciliation, an EcVolume can
// legitimately have ecv.dir != ecv.dir_idx (shards
// on one disk, .ecx / .ecj / .vif on a sibling).
// Use the EcVolume's own dirs rather than collapsing
// both args to find_ec_dir's single answer, otherwise
// read_ec_shard_config falls back to the wrong .vif
// location for split-disk volumes (#9252).
let dir = ecv.dir.clone();
let idx_dir = ecv.dir_idx.clone();
let dir = store
.find_ec_dir(*vid, &collection)
.unwrap_or_else(|| String::from(""));
if dir.is_empty() {
continue;
}
@@ -3611,7 +3457,6 @@ impl VolumeServer for VolumeGrpcService {
let (data_shards, parity_shards) =
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
&dir,
&idx_dir,
&collection,
*vid,
);
@@ -3649,12 +3494,6 @@ impl VolumeServer for VolumeGrpcService {
}
}
emit_scrub_metrics(
mode,
broken_volume_ids.len(),
Some(broken_shard_infos.len()),
);
Ok(Response::new(volume_server_pb::ScrubEcVolumeResponse {
total_volumes,
total_files,
@@ -3898,24 +3737,6 @@ impl VolumeServer for VolumeGrpcService {
let start = now_ns();
// Empty target is a self-liveness probe and stays unauthenticated.
// Otherwise gate the dial on cluster membership: volume servers only
// know masters, so any other target type is refused. Mirrors Go's
// volume_grpc_admin.go Ping admission check. tonic forbids returning
// a body alongside an error, so we surface the InvalidArgument status
// alone — behaviour-identical to Go's status.Errorf return.
if !req.target.is_empty()
&& !self
.state
.is_known_ping_target(&req.target, &req.target_type)
.await
{
return Err(Status::invalid_argument(format!(
"unknown ping target {} of type {}",
req.target, req.target_type
)));
}
// Route ping based on target type (matches Go's volume_grpc_admin.go Ping)
let remote_time_ns = if req.target_type == "volumeServer" {
match ping_volume_server_target(&req.target, self.state.outgoing_grpc_tls.as_ref())
@@ -4371,7 +4192,6 @@ mod tests {
let addr = listener.local_addr().unwrap();
listener.set_nonblocking(true).unwrap();
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
std::thread::spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
@@ -4433,7 +4253,6 @@ mod tests {
}));
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
let _ = ready_tx.send(());
axum::serve(listener, app)
.with_graceful_shutdown(async move {
let _ = shutdown_rx.await;
@@ -4443,8 +4262,6 @@ mod tests {
});
});
// Wait for the server thread to be ready before returning.
ready_rx.recv().unwrap();
(format!("http://{}", addr), shutdown_tx)
}
@@ -4490,9 +4307,7 @@ mod tests {
std::fs::remove_file(&dat_path).unwrap();
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
// Use a test-specific backend_id to avoid racing with other tests
// that share the global registry. Never call clear() — only
// register/remove our own entries.
global_s3_tier_registry().write().unwrap().clear();
let tier_config = S3TierConfig {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
@@ -4504,16 +4319,14 @@ mod tests {
};
{
let mut registry = global_s3_tier_registry().write().unwrap();
registry.register(
"s3.incr_copy_test".to_string(),
S3TierBackend::new(&tier_config),
);
registry.register("s3.default".to_string(), S3TierBackend::new(&tier_config));
registry.register("s3".to_string(), S3TierBackend::new(&tier_config));
}
let vif = crate::storage::volume::VifVolumeInfo {
files: vec![crate::storage::volume::VifRemoteFile {
backend_type: "s3".to_string(),
backend_id: "incr_copy_test".to_string(),
backend_id: "default".to_string(),
key: "remote-key".to_string(),
offset: 0,
file_size: dat_bytes.len() as u64,
@@ -4578,8 +4391,6 @@ mod tests {
read_mode: crate::config::ReadMode::Local,
master_url: String::new(),
master_urls: Vec::new(),
seed_master_set: std::collections::HashSet::new(),
current_master_url: tokio::sync::RwLock::new(String::new()),
self_url: String::new(),
http_client: reqwest::Client::new(),
outgoing_http_scheme: "http".to_string(),
@@ -4682,8 +4493,6 @@ mod tests {
read_mode: crate::config::ReadMode::Local,
master_url: String::new(),
master_urls: Vec::new(),
seed_master_set: std::collections::HashSet::new(),
current_master_url: tokio::sync::RwLock::new(String::new()),
self_url: String::new(),
http_client: reqwest::Client::new(),
outgoing_http_scheme: "http".to_string(),
@@ -4726,210 +4535,7 @@ mod tests {
assert_eq!(copied, dat_bytes[super_block_size as usize..]);
let _ = shutdown_tx.send(());
global_s3_tier_registry()
.write()
.unwrap()
.remove("s3.incr_copy_test");
}
/// Build a bare-bones service with no on-disk store but a configurable
/// seed master set, for Ping admission tests. Matches the structure of
/// `make_local_service_with_volume` minus the volume bits.
fn make_service_with_seed_masters(seeds: &[&str]) -> (VolumeGrpcService, TempDir) {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
store
.add_location(
dir,
dir,
10,
DiskType::HardDrive,
MinFreeSpace::Percent(1.0),
Vec::new(),
)
.unwrap();
let master_urls: Vec<String> = seeds.iter().map(|s| (*s).to_string()).collect();
let seed_master_set =
crate::server::volume_server::VolumeServerState::build_seed_master_set(&master_urls);
let state = Arc::new(VolumeServerState {
store: RwLock::new(store),
guard: RwLock::new(Guard::new(
&[],
SigningKey(vec![]),
0,
SigningKey(vec![]),
0,
)),
is_stopping: RwLock::new(false),
maintenance: std::sync::atomic::AtomicBool::new(false),
state_version: std::sync::atomic::AtomicU32::new(0),
concurrent_upload_limit: 0,
concurrent_download_limit: 0,
inflight_upload_data_timeout: std::time::Duration::from_secs(60),
inflight_download_data_timeout: std::time::Duration::from_secs(60),
inflight_upload_bytes: std::sync::atomic::AtomicI64::new(0),
inflight_download_bytes: std::sync::atomic::AtomicI64::new(0),
upload_notify: tokio::sync::Notify::new(),
download_notify: tokio::sync::Notify::new(),
data_center: String::new(),
rack: String::new(),
file_size_limit_bytes: 0,
maintenance_byte_per_second: 0,
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
has_master: false,
pre_stop_seconds: 0,
volume_state_notify: tokio::sync::Notify::new(),
write_queue: std::sync::OnceLock::new(),
s3_tier_registry: std::sync::RwLock::new(
crate::remote_storage::s3_tier::S3TierRegistry::new(),
),
read_mode: crate::config::ReadMode::Local,
master_url: master_urls.first().cloned().unwrap_or_default(),
master_urls,
seed_master_set,
current_master_url: tokio::sync::RwLock::new(String::new()),
self_url: String::new(),
http_client: reqwest::Client::new(),
outgoing_http_scheme: "http".to_string(),
outgoing_grpc_tls: None,
metrics_runtime: std::sync::RwLock::new(
crate::server::volume_server::RuntimeMetricsConfig::default(),
),
metrics_notify: tokio::sync::Notify::new(),
fix_jpg_orientation: false,
has_slow_read: false,
read_buffer_size_bytes: 1024 * 1024,
security_file: String::new(),
cli_white_list: vec![],
state_file_path: String::new(),
});
(VolumeGrpcService { state }, tmp)
}
#[tokio::test]
async fn test_ping_empty_target_is_self_probe() {
// Empty target stays unauthenticated and returns Ok with timing fields
// populated — it is the local liveness probe path.
let (service, _tmp) = make_service_with_seed_masters(&[]);
let response = service
.ping(Request::new(volume_server_pb::PingRequest {
target: String::new(),
target_type: String::new(),
}))
.await
.expect("empty target ping must succeed");
let inner = response.into_inner();
assert!(inner.start_time_ns > 0);
assert!(inner.stop_time_ns >= inner.start_time_ns);
}
#[tokio::test]
async fn test_ping_seed_master_target_passes_admission() {
// A target that matches a configured seed master clears admission.
// The dial itself may or may not succeed depending on what's listening
// on the loopback; either way, the response must not be the
// InvalidArgument the gate would surface.
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
let result = service
.ping(Request::new(volume_server_pb::PingRequest {
target: "localhost:9333".to_string(),
target_type: "master".to_string(),
}))
.await;
if let Err(err) = result {
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
assert!(
!err.message().contains("unknown ping target"),
"admission gate should have allowed this target: {}",
err.message()
);
}
}
#[tokio::test]
async fn test_ping_unknown_master_target_rejected() {
// A master-type target not in the seed list and not the current
// master is refused with InvalidArgument.
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
let err = service
.ping(Request::new(volume_server_pb::PingRequest {
target: "localhost:9999".to_string(),
target_type: "master".to_string(),
}))
.await
.expect_err("unknown master target must be rejected");
assert_eq!(err.code(), tonic::Code::InvalidArgument);
assert_eq!(
err.message(),
"unknown ping target localhost:9999 of type master"
);
}
#[tokio::test]
async fn test_ping_volume_server_target_always_rejected() {
// Volume servers do not maintain a peer-volume list, so volumeServer
// pings are refused regardless of address.
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
let err = service
.ping(Request::new(volume_server_pb::PingRequest {
target: "localhost:8080".to_string(),
target_type: "volumeServer".to_string(),
}))
.await
.expect_err("volumeServer target must be rejected");
assert_eq!(err.code(), tonic::Code::InvalidArgument);
assert_eq!(
err.message(),
"unknown ping target localhost:8080 of type volumeServer"
);
}
#[tokio::test]
async fn test_ping_current_master_target_passes_admission() {
// A target that matches the current (post-leader-change) master also
// clears admission, even if it is not in the seed list. Pick a port
// that is extremely unlikely to be live so the test does not flake on
// a developer machine that happens to be running a real master.
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
// Simulate the heartbeat goroutine having moved to a new leader.
*service.state.current_master_url.write().await = "127.0.0.1:1".to_string();
let result = service
.ping(Request::new(volume_server_pb::PingRequest {
target: "127.0.0.1:1".to_string(),
target_type: "master".to_string(),
}))
.await;
if let Err(err) = result {
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
assert!(
!err.message().contains("unknown ping target"),
"leader-change master should be admitted: {}",
err.message()
);
}
}
#[tokio::test]
async fn test_ping_target_with_grpc_port_suffix_is_normalised() {
// Seed masters in pb.ServerAddress form (`host:port.grpcPort`) must
// match a Ping target sent in plain `host:port` form, since the gate
// normalises both sides through to_http_address.
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333.19333"]);
let result = service
.ping(Request::new(volume_server_pb::PingRequest {
target: "localhost:9333".to_string(),
target_type: "master".to_string(),
}))
.await;
if let Err(err) = result {
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
}
global_s3_tier_registry().write().unwrap().clear();
}
#[tokio::test]
+6 -93
View File
@@ -16,7 +16,7 @@ use axum::response::{IntoResponse, Response};
use serde::{Deserialize, Serialize};
use super::grpc_client::{build_grpc_endpoint, GRPC_MAX_MESSAGE_SIZE};
use super::volume_server::{normalize_outgoing_http_url, to_http_address, VolumeServerState};
use super::volume_server::{normalize_outgoing_http_url, VolumeServerState};
use crate::config::ReadMode;
use crate::metrics;
use crate::pb::volume_server_pb;
@@ -357,12 +357,9 @@ async fn lookup_volume(
master_url: &str,
volume_id: u32,
) -> Result<Vec<VolumeLocation>, String> {
// master_url may be in SeaweedFS's "host:httpPort.grpcPort" form (mirrors
// Go's pb.ServerAddress); strip the gRPC port before building the HTTP URL.
let master_http = to_http_address(master_url);
let url = normalize_outgoing_http_url(
scheme,
&format!("{}/dir/lookup?volumeId={}", master_http, volume_id),
&format!("{}/dir/lookup?volumeId={}", master_url, volume_id),
)?;
let resp = client
.get(&url)
@@ -531,12 +528,9 @@ async fn do_replicated_request(
let mut futures = Vec::new();
for loc in remote_locations {
// Defensively strip a `.grpcPort` suffix in case loc.url carries it
// (e.g. from an older master); see lookup_volume for the same fix.
let peer_http = to_http_address(&loc.url);
let url = normalize_outgoing_http_url(
&state.outgoing_http_scheme,
&format!("{}{}?{}", peer_http, path, new_query),
&format!("{}{}?{}", loc.url, path, new_query),
)?;
let client = state.http_client.clone();
@@ -704,15 +698,13 @@ async fn proxy_request(
) -> Response {
// Build target URL, adding proxied=true query param
let path = info.path.trim_start_matches('/');
// Defensively strip a `.grpcPort` suffix in case target.url carries it.
let target_http = to_http_address(&target.url);
let raw_target = if info.original_query.is_empty() {
format!("{}/{}?proxied=true", target_http, path)
format!("{}/{}?proxied=true", target.url, path)
} else {
format!(
"{}/{}?{}&proxied=true",
target_http, path, info.original_query
target.url, path, info.original_query
)
};
let target_url = match normalize_outgoing_http_url(&state.outgoing_http_scheme, &raw_target) {
@@ -779,11 +771,9 @@ fn redirect_request(info: &ProxyRequestInfo, target: &VolumeLocation, scheme: &s
query_params.push("proxied=true".to_string());
let query = query_params.join("&");
// Defensively strip a `.grpcPort` suffix in case target.url carries it.
let target_http = to_http_address(&target.url);
let raw_target = format!(
"{}/{},{}?{}",
target_http, &info.vid_str, &info.fid_str, query
target.url, &info.vid_str, &info.fid_str, query
);
let location = match normalize_outgoing_http_url(scheme, &raw_target) {
Ok(url) => url,
@@ -3930,81 +3920,4 @@ mod tests {
"https://volume.internal:8080/3,01637037d6?collection=photos&proxied=true"
);
}
/// Defensive coverage for peer-to-peer redirects: if a master ever returns
/// `Location.url` already encoded in `host:httpPort.grpcPort` form,
/// `redirect_request` must still emit a valid HTTP Location header.
#[test]
fn test_redirect_request_strips_grpc_port_from_target_url() {
let info = ProxyRequestInfo {
original_headers: HeaderMap::new(),
original_query: String::new(),
path: "/3,01637037d6".to_string(),
vid_str: "3".to_string(),
fid_str: "01637037d6".to_string(),
};
let target = VolumeLocation {
url: "volume.internal:8080.18080".to_string(),
public_url: "volume.public:8080.18080".to_string(),
grpc_port: 18080,
};
let response = redirect_request(&info, &target, "http");
assert_eq!(response.status(), StatusCode::MOVED_PERMANENTLY);
assert_eq!(
response.headers().get(header::LOCATION).unwrap(),
"http://volume.internal:8080/3,01637037d6?proxied=true"
);
}
/// Regression test for issue #9274.
///
/// SeaweedFS encodes a server's gRPC port by appending `.grpcPort` to
/// the HTTP port: e.g. `master.local:9333.19333` (mirrors Go's
/// `pb.ServerAddress`). When the volume server's `--mserver` flag is
/// configured this way, replication writes look up the peer locations
/// from the master over HTTP. Prior to the fix, the lookup URL became
/// `http://host:9333.19333/dir/lookup?volumeId=...` — an invalid port
/// that reqwest rejects with a "builder error", causing every
/// replicated write to fail.
#[tokio::test]
async fn test_lookup_volume_strips_grpc_port_from_master_url() {
use axum::{routing::get, Router};
let app = Router::new().route(
"/dir/lookup",
get(|axum::extract::Query(params): axum::extract::Query<std::collections::HashMap<String, String>>| async move {
assert_eq!(params.get("volumeId").map(String::as_str), Some("31"));
axum::Json(serde_json::json!({
"volumeOrFileId": "31",
"locations": [
{"url": "10.0.0.2:5301", "publicUrl": "10.0.0.2:5301", "grpcPort": 5311}
]
}))
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
// Mirrors how main.rs receives masters: a `host:httpPort.grpcPort` token.
// The grpc-port part is fictitious here (the master is HTTP-only in
// this test) — what matters is that the volume server must not pass
// it through to reqwest.
let master_url = format!("127.0.0.1:{}.{}", addr.port(), 19999);
let client = reqwest::Client::new();
let locations = super::lookup_volume(&client, "http", &master_url, 31)
.await
.expect("lookup_volume should succeed even when master_url carries a .grpcPort suffix");
assert_eq!(locations.len(), 1);
assert_eq!(locations[0].url, "10.0.0.2:5301");
assert_eq!(locations[0].grpc_port, 5311);
server.abort();
}
}
+15 -57
View File
@@ -169,20 +169,10 @@ pub async fn run_heartbeat_with_state(
}
}
/// Convert a master address to a gRPC `host:port` target.
///
/// Mirrors Go's `pb.ServerToGrpcAddress()`:
/// - `host:port.grpcPort` returns `host:grpcPort` (explicit gRPC port).
/// - `host:port` returns `host:(port+10000)` (Go's default offset).
/// - Anything that fails to parse is returned unchanged.
/// Convert a master address "host:port" to a gRPC host:port target.
/// The Go master uses port + 10000 for gRPC by default.
pub fn to_grpc_address(master_addr: &str) -> String {
if let Some((host, port_str)) = master_addr.rsplit_once(':') {
// "host:port.grpcPort" — the part after the last '.' is the gRPC port.
if let Some((_, grpc_port)) = port_str.rsplit_once('.') {
if grpc_port.parse::<u16>().is_ok() {
return format!("{}:{}", host, grpc_port);
}
}
if let Ok(port) = port_str.parse::<u16>() {
let grpc_port = port + 10000;
return format!("{}:{}", host, grpc_port);
@@ -397,15 +387,6 @@ async fn do_heartbeat(
let mut response_stream = client.send_heartbeat(stream).await?.into_inner();
info!("Heartbeat stream established with {}", grpc_addr);
// Publish the master we're now talking to in canonical http host:port
// form so Ping admission can recognise it once a leader change moves us
// off the seed list. Mirrors Go's vs.setCurrentMaster(masterAddress).
{
let normalised =
super::volume_server::to_http_address(current_master).into_owned();
let mut guard = state.current_master_url.write().await;
*guard = normalised;
}
if is_stopping(state) {
state.is_heartbeating.store(false, Ordering::Relaxed);
send_deregister_heartbeat(config, state, &tx).await;
@@ -869,7 +850,7 @@ fn build_heartbeat_with_ec_status(
}
for vid in delete_vids {
let _ = loc.delete_volume(vid, false, false);
let _ = loc.delete_volume(vid, false);
}
}
@@ -1042,8 +1023,6 @@ mod tests {
read_mode: ReadMode::Local,
master_url: String::new(),
master_urls: Vec::new(),
seed_master_set: std::collections::HashSet::new(),
current_master_url: tokio::sync::RwLock::new(String::new()),
self_url: String::new(),
http_client: reqwest::Client::new(),
outgoing_http_scheme: "http".to_string(),
@@ -1059,26 +1038,6 @@ mod tests {
})
}
#[test]
fn test_to_grpc_address_default_offset() {
assert_eq!(to_grpc_address("10.0.0.1:9333"), "10.0.0.1:19333");
assert_eq!(to_grpc_address("localhost:9333"), "localhost:19333");
}
#[test]
fn test_to_grpc_address_explicit_grpc_port() {
// host:port.grpcPort form — gRPC port is what's after the dot.
assert_eq!(to_grpc_address("10.85.183.6:5300.6300"), "10.85.183.6:6300");
assert_eq!(to_grpc_address("master.local:9333.19333"), "master.local:19333");
}
#[test]
fn test_to_grpc_address_returns_input_when_unparseable() {
assert_eq!(to_grpc_address(""), "");
assert_eq!(to_grpc_address("no-port"), "no-port");
assert_eq!(to_grpc_address("host:not-a-port"), "host:not-a-port");
}
#[test]
fn test_build_heartbeat_includes_store_identity_and_disk_metadata() {
let temp_dir = tempfile::tempdir().unwrap();
@@ -1252,7 +1211,7 @@ mod tests {
let shard_path = format!("{}/ec_metrics_case_27.ec00", dir);
std::fs::write(&shard_path, b"ec-shard").unwrap();
store.locations[0]
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0], "")
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0])
.unwrap();
let state = test_state_with_store(store);
@@ -1294,7 +1253,7 @@ mod tests {
std::fs::write(format!("{}/expired_heartbeat_ec_31.ec00", dir), b"expired").unwrap();
store.locations[0]
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0], "")
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0])
.unwrap();
store
.find_ec_volume_mut(VolumeId(31))
@@ -1449,8 +1408,10 @@ mod tests {
#[test]
fn test_apply_storage_backends_registers_s3_default_aliases() {
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
// Do not call clear() on the global registry — other tests may be
// running concurrently. Just register our entries and verify them.
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.clear();
apply_storage_backends(
&state,
@@ -1482,8 +1443,10 @@ mod tests {
#[test]
fn test_apply_storage_backends_ignores_unsupported_types() {
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
// Do not call clear() on the global registry — other tests may be
// running concurrently.
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.clear();
apply_storage_backends(
&state,
@@ -1494,17 +1457,12 @@ mod tests {
}],
);
// The per-state registry is freshly created and should have no entries
// since "rclone" is unsupported.
let registry = state.s3_tier_registry.read().unwrap();
assert!(registry.names().is_empty());
// Only check that the unsupported type was not added to the global
// registry. Other tests may have their own entries present.
let global_registry = crate::remote_storage::s3_tier::global_s3_tier_registry()
.read()
.unwrap();
assert!(global_registry.get("rclone.default").is_none());
assert!(global_registry.get("rclone").is_none());
assert!(global_registry.names().is_empty());
}
#[test]
@@ -1597,7 +1555,7 @@ mod tests {
std::fs::write(format!("{}/ec_delta_case_81.ec00", dir), b"delta").unwrap();
store.locations[0]
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0], "")
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0])
.unwrap();
let current = collect_ec_shard_delta_messages(&store);
let (new_ec_shards, deleted_ec_shards) =
-146
View File
@@ -81,16 +81,6 @@ pub struct VolumeServerState {
pub master_url: String,
/// Seed master addresses for UI rendering.
pub master_urls: Vec<String>,
/// Canonical http `host:port` form of every configured seed master.
/// Built once at construction so Ping admission stays O(1). Mirrors
/// Go's `seedMasterSet` on `VolumeServer`.
pub seed_master_set: std::collections::HashSet<String>,
/// Current master this server is heartbeating with, in canonical http
/// `host:port` form. Empty when no heartbeat connection is active. The
/// heartbeat goroutine writes; admission reads — the lock keeps them
/// from racing on a leader change. Mirrors Go's `currentMaster` plus
/// `currentMasterLock`.
pub current_master_url: tokio::sync::RwLock<String>,
/// This server's own address (ip:port) for filtering self from lookup results.
pub self_url: String,
/// HTTP client for proxy requests and master lookups.
@@ -127,35 +117,6 @@ impl VolumeServerState {
}
Ok(())
}
/// Build the seed master set from a list of raw `host:port[.grpcPort]`
/// addresses, normalised the same way Go's `pb.ServerAddress.ToHttpAddress`
/// does (drop the `.grpcPort` suffix, preserve everything else).
pub fn build_seed_master_set(master_urls: &[String]) -> std::collections::HashSet<String> {
master_urls
.iter()
.map(|m| to_http_address(m).into_owned())
.collect()
}
/// Returns true iff `target` (normalised to canonical http `host:port`)
/// is a master this server already knows about. Volume servers do not
/// keep a peer-volume or peer-filer list, so Ping is scoped to masters.
/// Mirrors Go's `VolumeServer.isKnownPingTarget`.
pub async fn is_known_ping_target(&self, target: &str, target_type: &str) -> bool {
if target_type != "master" {
return false;
}
let key = to_http_address(target).into_owned();
if key.is_empty() {
return false;
}
let current = self.current_master_url.read().await.clone();
if !current.is_empty() && current == key {
return true;
}
self.seed_master_set.contains(&key)
}
}
pub fn build_metrics_router() -> Router {
@@ -173,38 +134,6 @@ pub fn normalize_outgoing_http_url(scheme: &str, raw_target: &str) -> Result<Str
Ok(format!("{}://{}", scheme, raw_target))
}
/// Convert a SeaweedFS server address to its HTTP `host:port` form.
///
/// Mirrors Go's `pb.ServerAddress.ToHttpAddress()`: SeaweedFS encodes a
/// server's gRPC port by appending `.grpcPort` to the HTTP port (e.g.
/// `host:9333.19333`). For HTTP requests we want only the HTTP `host:port`.
/// - `host:port.grpcPort` -> `host:port`
/// - `host:port` -> `host:port` (unchanged)
/// - Anything that does not look like `host:port[.grpcPort]` is returned unchanged.
///
/// Returns a `Cow<str>` so the common (no-suffix) case borrows from `addr`
/// without allocating; only the rewrite branch produces a new `String`.
pub fn to_http_address(addr: &str) -> std::borrow::Cow<'_, str> {
let Some(ports_sep_index) = addr.rfind(':') else {
return std::borrow::Cow::Borrowed(addr);
};
let ports = &addr[ports_sep_index + 1..];
if let Some(dot_idx) = ports.rfind('.') {
let http_port = &ports[..dot_idx];
let grpc_port = &ports[dot_idx + 1..];
// Only strip the suffix when both parts parse as real ports — leave
// anything else (e.g. "host:abc.def") untouched so bad config surfaces
// rather than being silently rewritten. Mirrors the validation already
// done in `to_grpc_address` for the inverse direction.
if let (Ok(_), Ok(_)) = (http_port.parse::<u16>(), grpc_port.parse::<u16>()) {
return std::borrow::Cow::Owned(
addr[..ports_sep_index + 1 + dot_idx].to_string(),
);
}
}
std::borrow::Cow::Borrowed(addr)
}
fn request_remote_addr(request: &Request) -> Option<SocketAddr> {
request
.extensions()
@@ -463,78 +392,3 @@ pub fn build_public_router(state: Arc<VolumeServerState>) -> Router {
.layer(middleware::from_fn(common_headers_middleware))
.with_state(state)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_to_http_address_strips_grpc_port_suffix() {
assert_eq!(to_http_address("10.0.0.1:9333.19333"), "10.0.0.1:9333");
assert_eq!(
to_http_address("master.local:9333.19333"),
"master.local:9333"
);
assert_eq!(to_http_address("10.85.183.6:5300.6300"), "10.85.183.6:5300");
}
#[test]
fn test_to_http_address_passthrough_without_grpc_suffix() {
assert_eq!(to_http_address("10.0.0.1:9333"), "10.0.0.1:9333");
assert_eq!(to_http_address("master.local:9333"), "master.local:9333");
}
#[test]
fn test_to_http_address_returns_input_when_unparseable() {
assert_eq!(to_http_address(""), "");
assert_eq!(to_http_address("no-port"), "no-port");
// Trailing colon: nothing after the separator, treat as unparseable.
assert_eq!(to_http_address("host:"), "host:");
}
#[test]
fn test_to_http_address_borrows_when_unchanged_and_owns_when_stripped() {
// The common case (no suffix) must not allocate.
let result = to_http_address("10.0.0.1:9333");
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
// Stripping requires a new string.
let result = to_http_address("10.0.0.1:9333.19333");
assert!(matches!(result, std::borrow::Cow::Owned(_)));
assert_eq!(result, "10.0.0.1:9333");
// Unparseable / passthrough also borrows.
let result = to_http_address("host:abc.def");
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
}
#[test]
fn test_to_http_address_keeps_non_numeric_dotted_suffix() {
// The dotted form is only valid when both sides are real port numbers.
// Otherwise the address is malformed config (e.g. a hostname like
// "host:abc.def"), and silently rewriting it would just hide the bug.
assert_eq!(to_http_address("host:abc.def"), "host:abc.def");
assert_eq!(to_http_address("host:9333.notaport"), "host:9333.notaport");
assert_eq!(to_http_address("host:notaport.19333"), "host:notaport.19333");
// Out-of-range ports must not be silently truncated either.
assert_eq!(to_http_address("host:99999.19333"), "host:99999.19333");
}
#[test]
fn test_to_http_address_handles_bracketed_ipv6_literals() {
// The function uses `rfind(':')`, so for bracketed IPv6 the port
// separator is correctly identified as the colon AFTER the closing
// bracket — making IPv4 and IPv6 behave the same.
assert_eq!(to_http_address("[::1]:9333.19333"), "[::1]:9333");
assert_eq!(
to_http_address("[2001:db8::10]:5300.6300"),
"[2001:db8::10]:5300"
);
// Plain bracketed IPv6 without a dotted suffix is borrowed unchanged.
let result = to_http_address("[2001:db8::1]:9333");
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
assert_eq!(result, "[2001:db8::1]:9333");
// Non-numeric / out-of-range / missing suffix all preserve the input.
assert_eq!(to_http_address("[::1]:"), "[::1]:");
assert_eq!(to_http_address("[::1]:abc.def"), "[::1]:abc.def");
assert_eq!(to_http_address("[::1]:99999.19333"), "[::1]:99999.19333");
}
}
-2
View File
@@ -207,8 +207,6 @@ mod tests {
read_mode: crate::config::ReadMode::Local,
master_url: String::new(),
master_urls: Vec::new(),
seed_master_set: std::collections::HashSet::new(),
current_master_url: tokio::sync::RwLock::new(String::new()),
self_url: String::new(),
http_client: reqwest::Client::new(),
outgoing_http_scheme: "http".to_string(),
+12 -565
View File
@@ -202,15 +202,6 @@ impl DiskLocation {
}
}
// After regular volumes, auto-discover EC shards on disk so a
// fresh restart picks up shards without an explicit
// VolumeEcShardsMount RPC. Mirrors Go's loadExistingVolumes
// calling loadAllEcShards. Failures here only log — they
// shouldn't fail the whole disk's startup.
if let Err(e) = self.load_all_ec_shards() {
warn!(directory = %self.directory, error = %e, "load_all_ec_shards failed");
}
Ok(())
}
@@ -304,11 +295,7 @@ impl DiskLocation {
}
/// Remove all EC-related files for a volume.
/// `pub(crate)` so the Store-level cross-disk passes in
/// `store_ec_reconcile.rs` can call it to scrub partial EC artefacts
/// when a healthy `.dat` for the same vid lives on a sibling disk
/// (seaweedfs/seaweedfs#9478).
pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
let base = volume_file_name(&self.directory, collection, vid);
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
const MAX_SHARD_COUNT: usize = 32;
@@ -390,20 +377,13 @@ impl DiskLocation {
}
}
/// Remove, close, and delete all files for a volume. When keep_remote_data
/// is true the cloud-tier object backing the volume is left intact — used
/// by moves where another server is taking over the same .vif.
pub fn delete_volume(
&mut self,
vid: VolumeId,
only_empty: bool,
keep_remote_data: bool,
) -> Result<(), VolumeError> {
/// Remove, close, and delete all files for a volume.
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
if let Some(mut v) = self.volumes.remove(&vid) {
crate::metrics::VOLUME_GAUGE
.with_label_values(&[&v.collection, "volume"])
.dec();
v.destroy(only_empty, keep_remote_data)?;
v.destroy(only_empty)?;
Ok(())
} else {
Err(VolumeError::NotFound)
@@ -424,7 +404,7 @@ impl DiskLocation {
crate::metrics::VOLUME_GAUGE
.with_label_values(&[&v.collection, "volume"])
.dec();
if let Err(e) = v.destroy(false, false) {
if let Err(e) = v.destroy(false) {
warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
}
}
@@ -564,114 +544,28 @@ impl DiskLocation {
self.ec_volumes.contains_key(&vid)
}
/// Reports whether this disk has a sealed `.ecx` index file for the
/// given (collection, vid). Unlike [`Self::has_ec_volume`] this does
/// not require the EC volume to be mounted in memory, which makes it
/// the right primitive for placement decisions during `ec.balance` /
/// `ec.rebuild` flows where shards may arrive before any
/// `VolumeEcShardsMount` has happened on the receiving disk. Without
/// checking the on-disk state, auto-select can split shards from the
/// `.ecx` that travels with the first shard, which is the source of
/// the orphan-shard layout reported in seaweedfs/seaweedfs#9212.
///
/// Mirrors `DiskLocation.HasEcxFileOnDisk` in
/// `weed/storage/disk_location_ec.go`. Skips entries that are
/// directories so a stray dir named `<collection>_<vid>.ecx` doesn't
/// register as a present index file.
pub fn has_ecx_file_on_disk(&self, collection: &str, vid: VolumeId) -> bool {
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
let idx_path = format!("{}.ecx", idx_base);
if let Ok(meta) = fs::metadata(&idx_path) {
if !meta.is_dir() {
return true;
}
}
if self.idx_directory != self.directory {
let data_base = volume_file_name(&self.directory, collection, vid);
let data_path = format!("{}.ecx", data_base);
if let Ok(meta) = fs::metadata(&data_path) {
if !meta.is_dir() {
return true;
}
}
}
false
}
/// Remove an EC volume, returning it.
pub fn remove_ec_volume(&mut self, vid: VolumeId) -> Option<EcVolume> {
self.ec_volumes.remove(&vid)
}
/// Mount EC shards for a volume on this location.
///
/// `source_disk_type` is the source volume's disk type carried on the
/// `VolumeEcShardsMount` RPC. When non-empty it overrides the in-memory
/// EC volume's reported disk type so heartbeats keep reporting under
/// the source's disk type after EC encoding (#9423). Empty means "use
/// this location's disk type" — used by disk-scan reload paths that
/// have no orchestrator context.
pub fn mount_ec_shards(
&mut self,
vid: VolumeId,
collection: &str,
shard_ids: &[u32],
source_disk_type: &str,
) -> Result<(), VolumeError> {
let idx_dir = self.idx_directory.clone();
self.mount_ec_shards_with_idx_dir(vid, collection, shard_ids, &idx_dir, source_disk_type)
}
/// Mount EC shards but explicitly specify the idx directory the
/// EcVolume should pull `.ecx` / `.ecj` / `.vif` from.
///
/// Cross-disk reconcile (mirrors `loadEcShardsWithIdxDir` in
/// `weed/storage/disk_location_ec.go`): when a volume's shards live
/// on this disk but the index files were left on a sibling disk
/// (the seaweedfs/seaweedfs#9212 orphan-shard layout), the
/// reconciler creates the EcVolume here while pointing it at the
/// sibling's idx dir. Each shard still ends up registered in this
/// disk's `ec_volumes` map so heartbeat reporting carries the right
/// disk_id per shard.
pub fn mount_ec_shards_with_idx_dir(
&mut self,
vid: VolumeId,
collection: &str,
shard_ids: &[u32],
idx_dir: &str,
source_disk_type: &str,
) -> Result<(), VolumeError> {
let dir = self.directory.clone();
// Avoid the entry().or_insert_with() pattern here: that closure
// can't return a Result, so any EcVolume::new failure (e.g.
// .ecx open error, .ecj create error, malformed .vif) would
// have to panic via unwrap(). Build the EcVolume up front and
// propagate the error to the caller.
if !self.ec_volumes.contains_key(&vid) {
let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid)
.map_err(VolumeError::Io)?;
self.ec_volumes.insert(vid, ec_vol);
}
let idx_dir = self.idx_directory.clone();
let ec_vol = self
.ec_volumes
.get_mut(&vid)
.expect("just inserted above");
// When the orchestrator supplied a source disk type on the Mount
// RPC, override the EC volume's disk type so heartbeats report
// under the source volume's disk type (#9423). When the caller
// passed "" (disk-scan reload, orphan-shard reconcile, restart)
// we only default to this location's disk type for a fresh EC
// volume; otherwise we leave whatever a prior RPC mount set in
// place, so empty-source reloads don't reintroduce the drift.
if !source_disk_type.is_empty() {
ec_vol.set_disk_type(DiskType::from_string(source_disk_type));
} else if ec_vol.shard_count() == 0 {
ec_vol.set_disk_type(self.disk_type.clone());
}
.entry(vid)
.or_insert_with(|| EcVolume::new(&dir, &idx_dir, collection, vid).unwrap());
ec_vol.disk_type = self.disk_type.clone();
for &shard_id in shard_ids {
let mut shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
shard.disk_type = ec_vol.disk_type.clone();
let shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
ec_vol.add_shard(shard).map_err(VolumeError::Io)?;
crate::metrics::VOLUME_GAUGE
.with_label_values(&[collection, "ec_shards"])
@@ -681,19 +575,10 @@ impl DiskLocation {
}
/// Unmount EC shards for a volume on this location.
///
/// Only shards that are actually mounted decrement the per-shard
/// metric — without this guard the gauge would underflow when the
/// caller passes a shard that lives on a sibling disk
/// (cross-disk reconcile makes that the common case for the same
/// `vid` after reconciliation).
pub fn unmount_ec_shards(&mut self, vid: VolumeId, shard_ids: &[u32]) {
if let Some(ec_vol) = self.ec_volumes.get_mut(&vid) {
let collection = ec_vol.collection.clone();
for &shard_id in shard_ids {
if !ec_vol.has_shard(shard_id as u8) {
continue;
}
ec_vol.remove_shard(shard_id as u8);
crate::metrics::VOLUME_GAUGE
.with_label_values(&[&collection, "ec_shards"])
@@ -706,183 +591,6 @@ impl DiskLocation {
}
}
/// Auto-discover .ec?? shard files on disk and mount them.
///
/// Mirrors `loadAllEcShards` in `weed/storage/disk_location_ec.go`.
/// Reads the data dir (and idx dir, if separate), groups shards by
/// (collection, vid) using sorted iteration so a volume's shards
/// land contiguously next to their `.ecx`. When the matching `.ecx`
/// is encountered, validates and mounts the group.
///
/// Without this, an EC shard on disk is invisible after a Rust
/// volume-server restart until something explicitly issues
/// VolumeEcShardsMount — strict superset of seaweedfs/seaweedfs#9212.
pub fn load_all_ec_shards(&mut self) -> io::Result<()> {
// Use a HashSet during accumulation so a name appearing in
// both data dir and idx dir (idempotent legacy layouts can
// place .ecx in either) is processed exactly once. Without
// dedup, mount_ec_shards' per-shard metric increment would
// double-count for those filenames.
let mut seen: HashSet<String> = HashSet::new();
let mut entries: Vec<String> = Vec::new();
for ent in fs::read_dir(&self.directory)? {
let ent = ent?;
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
continue;
}
let name = ent.file_name().to_string_lossy().into_owned();
if seen.insert(name.clone()) {
entries.push(name);
}
}
if self.idx_directory != self.directory {
for ent in fs::read_dir(&self.idx_directory)? {
let ent = ent?;
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
continue;
}
let name = ent.file_name().to_string_lossy().into_owned();
if seen.insert(name.clone()) {
entries.push(name);
}
}
}
entries.sort();
let mut same_volume_shards: Vec<(String, u32)> = Vec::new(); // (filename, shard_id)
let mut prev_vid: Option<VolumeId> = None;
let mut prev_collection: String = String::new();
for name in entries {
let Some(dot) = name.rfind('.') else {
continue;
};
let (base, ext) = name.split_at(dot); // ext includes leading '.'
let Some((collection, vid)) = parse_collection_volume_id(base) else {
continue;
};
if let Some(shard_id) = parse_ec_shard_extension(ext) {
// Ignore zero-byte shards — same as Go.
let path = format!("{}/{}", self.directory, name);
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
if size == 0 {
continue;
}
let same_group = match prev_vid {
None => true,
Some(p) => p == vid && prev_collection == collection,
};
if same_group {
same_volume_shards.push((name, shard_id));
} else {
self.check_orphaned_shards(
&same_volume_shards,
&prev_collection,
prev_vid.unwrap_or(VolumeId(0)),
);
same_volume_shards = vec![(name, shard_id)];
}
prev_vid = Some(vid);
prev_collection = collection;
continue;
}
if ext == ".ecx"
&& prev_vid == Some(vid)
&& prev_collection == collection
&& !same_volume_shards.is_empty()
{
let shards = std::mem::take(&mut same_volume_shards);
self.handle_found_ecx_file(&shards, &collection, vid);
prev_vid = None;
prev_collection.clear();
}
}
// Handle any trailing group that never saw a matching .ecx.
self.check_orphaned_shards(
&same_volume_shards,
&prev_collection,
prev_vid.unwrap_or(VolumeId(0)),
);
Ok(())
}
/// Validate + mount a (collection, vid) group when its `.ecx` is
/// found. Mirrors `handleFoundEcxFile` in
/// `weed/storage/disk_location_ec.go`.
fn handle_found_ecx_file(&mut self, shards: &[(String, u32)], collection: &str, vid: VolumeId) {
let base = volume_file_name(&self.directory, collection, vid);
let dat_path = format!("{}.dat", base);
let dat_exists = check_dat_file_exists(&dat_path);
if dat_exists && !self.validate_ec_volume(collection, vid) {
warn!(
volume_id = vid.0,
"Incomplete or invalid EC volume: .dat exists but validation failed, cleaning up EC files",
);
self.remove_ec_volume_files(collection, vid);
return;
}
let shard_ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids, "") {
// mount_ec_shards adds shards one at a time and increments
// the per-shard metric for each. If it fails halfway, plain
// ec_volumes.remove(vid) would leak metric increments for
// the shards that did mount. Drive cleanup through
// unmount_ec_shards which mirror-decrements the metric, then
// the empty EcVolume drops itself.
if dat_exists {
warn!(
volume_id = vid.0,
"Failed to load EC shards and .dat exists ({}), cleaning up EC files to use .dat",
e,
);
self.unmount_ec_shards(vid, &shard_ids);
self.remove_ec_volume_files(collection, vid);
} else {
warn!(
volume_id = vid.0,
"Failed to load EC shards: {} (this may be normal for distributed EC volumes)",
e,
);
self.unmount_ec_shards(vid, &shard_ids);
}
}
}
/// Mirrors `checkOrphanedShards`: if shards exist on disk without a
/// matching `.ecx` AND the legacy `.dat` is still present, treat
/// the encoding as interrupted before `.ecx` was written and clean
/// up. If `.dat` is gone, leave files alone — they may be
/// distributed-EC shards waiting for cross-disk reconciliation.
fn check_orphaned_shards(
&self,
shards: &[(String, u32)],
collection: &str,
vid: VolumeId,
) -> bool {
if shards.is_empty() || vid.0 == 0 {
return false;
}
let base = volume_file_name(&self.directory, collection, vid);
let dat_path = format!("{}.dat", base);
if check_dat_file_exists(&dat_path) {
warn!(
volume_id = vid.0,
"Found {} EC shards without .ecx file (incomplete encoding interrupted before .ecx), cleaning up",
shards.len(),
);
self.remove_ec_volume_files(collection, vid);
return true;
}
false
}
/// Total number of EC shards on this location.
pub fn ec_shard_count(&self) -> usize {
self.ec_volumes
@@ -955,69 +663,6 @@ fn calculate_expected_shard_size(dat_file_size: i64) -> i64 {
}
/// Parse a volume filename like "collection_42.dat" or "42.dat" into (collection, VolumeId).
/// Parse a `<collection>_<vid>` or `<vid>` base name into its parts.
/// Mirrors `parseCollectionVolumeId` in
/// `weed/storage/disk_location.go`. Used when iterating raw filenames
/// where the extension has already been stripped.
///
/// `pub(crate)` companion `parse_collection_volume_id_pub` is exposed
/// so the cross-disk reconcile in `store_ec_reconcile.rs` can call it
/// without re-implementing the parser.
pub(crate) fn parse_collection_volume_id_pub(base: &str) -> Option<(String, VolumeId)> {
parse_collection_volume_id(base)
}
fn parse_collection_volume_id(base: &str) -> Option<(String, VolumeId)> {
if let Some(pos) = base.rfind('_') {
let collection = &base[..pos];
let id_str = &base[pos + 1..];
let id: u32 = id_str.parse().ok()?;
Some((collection.to_string(), VolumeId(id)))
} else {
let id: u32 = base.parse().ok()?;
Some((String::new(), VolumeId(id)))
}
}
/// `pub(crate)` re-export of [`parse_ec_shard_extension`] for the
/// cross-disk reconcile in `store_ec_reconcile.rs`.
pub(crate) fn is_ec_shard_extension(ext: &str) -> Option<u32> {
parse_ec_shard_extension(ext)
}
/// Recognise EC shard extensions `.ec00`–`.ec255` (the ShardId u8
/// range — see `EcVolumeShard`'s typed shard id) and return the
/// shard id. Returns `None` for any other extension.
///
/// Matches the regex `\.ec\d{2,3}` from the Go side, with the same
/// per-shard `id <= 255` clamp Go's loader applies via
/// `strconv.ParseInt(... 10, 64)` followed by the `if shardId < 0 ||
/// shardId > 255` guard. The 3-digit form (`.ec100`–`.ec255`) is
/// retained so the parser can still recognise shards from custom
/// 32+ ratios that fit in a u8 even though OSS only ships 10+4.
fn parse_ec_shard_extension(ext: &str) -> Option<u32> {
let rest = ext.strip_prefix(".ec")?;
if rest.len() < 2 || rest.len() > 3 {
return None;
}
let id: u32 = rest.parse().ok()?;
if id > 255 {
return None;
}
Some(id)
}
/// Robust `.dat` existence check: any unexpected stat error (permission,
/// I/O) is treated as "exists" so we don't misclassify local EC as
/// distributed EC. Mirrors `checkDatFileExists` in Go.
fn check_dat_file_exists(path: &str) -> bool {
match fs::metadata(path) {
Ok(_) => true,
Err(e) if e.kind() == io::ErrorKind::NotFound => false,
Err(_) => true,
}
}
fn parse_volume_filename(filename: &str) -> Option<(String, VolumeId)> {
let stem = filename
.strip_suffix(".dat")
@@ -1189,7 +834,7 @@ mod tests {
.unwrap();
assert_eq!(loc.volumes_len(), 2);
loc.delete_volume(VolumeId(1), false, false).unwrap();
loc.delete_volume(VolumeId(1), false).unwrap();
assert_eq!(loc.volumes_len(), 1);
assert!(loc.find_volume(VolumeId(1)).is_none());
}
@@ -1262,7 +907,7 @@ mod tests {
let shard_path = format!("{}/pics_7.ec00", dir);
std::fs::write(&shard_path, b"ec-shard").unwrap();
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap();
loc.mount_ec_shards(VolumeId(7), "pics", &[0]).unwrap();
assert!(loc.has_ec_volume(VolumeId(7)));
assert!(std::path::Path::new(&shard_path).exists());
assert!(std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
@@ -1274,56 +919,6 @@ mod tests {
assert!(!std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
}
/// #9423: after an RPC `VolumeEcShardsMount` records the source
/// volume's disk type on the EcVolume, a later empty-source mount
/// call (disk-scan reload, orphan-shard reconcile, restart) must
/// not clobber that override back to the physical location's disk
/// type — otherwise heartbeat reporting drifts back to "hdd" on
/// every reload.
#[test]
fn test_mount_ec_shards_empty_source_preserves_existing_disk_type() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut loc = DiskLocation::new(
dir,
dir,
10,
DiskType::HardDrive,
MinFreeSpace::Percent(1.0),
Vec::new(),
)
.unwrap();
// Plant the first shard so the EC volume can mount, then call
// mount_ec_shards with source_disk_type="ssd" — simulating the
// VolumeEcShardsMount RPC path.
std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap();
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd").unwrap();
{
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume mounted");
assert_eq!(
ec_vol.disk_type,
DiskType::Ssd,
"first RPC mount should have set disk type to source's value",
);
}
// Plant another shard and re-mount with an empty source —
// matching what a disk-scan reload or orphan-shard reconcile
// would pass. The previously-recorded "ssd" override must
// survive.
std::fs::write(format!("{}/pics_7.ec01", dir), b"ec-shard").unwrap();
loc.mount_ec_shards(VolumeId(7), "pics", &[1], "").unwrap();
{
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume still mounted");
assert_eq!(
ec_vol.disk_type,
DiskType::Ssd,
"empty-source remount must not reset disk type to the physical location's",
);
}
}
#[test]
fn test_disk_location_persists_directory_uuid_and_tags() {
let tmp = TempDir::new().unwrap();
@@ -1353,152 +948,4 @@ mod tests {
.unwrap();
assert_eq!(reloaded.directory_uuid, directory_uuid);
}
/// Sanity: `parse_ec_shard_extension` accepts `.ec00` through
/// `.ec999` and rejects anything else.
#[test]
fn test_parse_ec_shard_extension() {
assert_eq!(parse_ec_shard_extension(".ec00"), Some(0));
assert_eq!(parse_ec_shard_extension(".ec13"), Some(13));
assert_eq!(parse_ec_shard_extension(".ec999"), None); // > 255
assert_eq!(parse_ec_shard_extension(".ec255"), Some(255));
assert_eq!(parse_ec_shard_extension(".ec0"), None); // need 2 digits
assert_eq!(parse_ec_shard_extension(".ecx"), None);
assert_eq!(parse_ec_shard_extension(".ecj"), None);
assert_eq!(parse_ec_shard_extension(".dat"), None);
assert_eq!(parse_ec_shard_extension("ec00"), None); // missing dot
}
/// Auto-load: shards + .ecx on the same disk get mounted into
/// `ec_volumes` after `load_existing_volumes` finishes. Without
/// this, EC shards on disk are invisible until something issues
/// `VolumeEcShardsMount`.
#[test]
fn test_load_all_ec_shards_mounts_pairs_with_ecx() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let collection = "grafana-loki";
let vid = VolumeId(1234);
// Two shards + .ecx + .ecj + .vif (the post-encode steady state).
for &sid in &[0u8, 4u8] {
let path = format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid);
std::fs::write(&path, b"shard data nonempty").unwrap();
}
std::fs::write(format!("{}/{}_{}.ecx", dir, collection, vid.0), vec![0u8; 20])
.unwrap();
std::fs::write(format!("{}/{}_{}.ecj", dir, collection, vid.0), b"").unwrap();
std::fs::write(
format!("{}/{}_{}.vif", dir, collection, vid.0),
br#"{"version":3}"#,
)
.unwrap();
let mut loc = DiskLocation::new(
dir,
dir,
10,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
loc.load_existing_volumes(NeedleMapKind::InMemory).unwrap();
let ec_vol = loc.find_ec_volume(vid).expect("EcVolume should be mounted");
assert_eq!(ec_vol.shard_count(), 2, "both shards should be loaded");
}
/// Orphan shards with no `.ecx` and no `.dat` must stay on disk
/// untouched (distributed-EC scenario, where the .ecx lives on
/// another server). Mirrors Go's `checkOrphanedShards` no-op
/// branch.
#[test]
fn test_load_all_ec_shards_keeps_orphan_shards_when_no_dat() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let collection = "grafana-loki";
let vid = VolumeId(2345);
// Shards on disk, but no .ecx and no .dat — distributed-EC
// with the .ecx on a sibling/server.
let shard_paths: Vec<String> = [0u8, 12u8]
.iter()
.map(|sid| format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid))
.collect();
for p in &shard_paths {
std::fs::write(p, b"shard data nonempty").unwrap();
}
let mut loc = DiskLocation::new(
dir,
dir,
10,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
loc.load_existing_volumes(NeedleMapKind::InMemory).unwrap();
// No EcVolume should be mounted (no .ecx).
assert!(loc.find_ec_volume(vid).is_none());
// But the shard files must still be on disk for cross-disk
// reconciliation / operator recovery.
for p in &shard_paths {
assert!(
std::path::Path::new(p).exists(),
"orphan shard {} was destroyed",
p,
);
}
}
/// Orphan shards WITH a stale `.dat` are treated as an
/// interrupted-encoding remnant and cleaned up. Mirrors Go's
/// `checkOrphanedShards` cleanup branch. Guards a subtle case
/// where a partial encoding wrote shards before .ecx and a crash
/// left the .dat behind.
#[test]
fn test_load_all_ec_shards_cleans_orphan_shards_when_dat_exists() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let collection = "grafana-loki";
let vid = VolumeId(3456);
let shard_paths: Vec<String> = [0u8, 1u8]
.iter()
.map(|sid| format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid))
.collect();
for p in &shard_paths {
std::fs::write(p, b"shard data nonempty").unwrap();
}
// Stale .dat from interrupted encoding.
let dat_path = format!("{}/{}_{}.dat", dir, collection, vid.0);
std::fs::write(&dat_path, b"unfinished").unwrap();
let mut loc = DiskLocation::new(
dir,
dir,
10,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
// load_existing_volumes processes .dat first; ec auto-load
// runs after and finds the orphans. The .dat itself may or may
// not be loadable as a regular volume — that's not what this
// test is about. Just check that the orphan shard files were
// removed.
let _ = loc.load_existing_volumes(NeedleMapKind::InMemory);
for p in &shard_paths {
assert!(
!std::path::Path::new(p).exists(),
"orphan shard {} should have been cleaned up because .dat exists",
p,
);
}
}
}
@@ -16,28 +16,11 @@ use crate::storage::volume::volume_file_name;
/// Calculate .dat file size from the max offset entry in .ecx.
/// Reads the volume version from the first EC shard (.ec00) superblock,
/// then scans .ecx entries to find the largest (offset + needle_actual_size).
///
/// `dir` is used both for reading `.ec00` and `.ecx`. For split-disk
/// reconciled volumes call [`find_dat_file_size_with_dirs`] instead.
pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
find_dat_file_size_with_dirs(dir, dir, collection, volume_id)
}
/// Like [`find_dat_file_size`] but lets the caller pass separate dirs
/// for `.ec00` (the data shard) and `.ecx` (the sealed index). This
/// is the form needed when shards are split across data dirs and the
/// `.ecx` lives on a sibling disk's idx dir (#9252).
pub fn find_dat_file_size_with_dirs(
ec00_dir: &str,
ecx_dir: &str,
collection: &str,
volume_id: VolumeId,
) -> io::Result<i64> {
let ec00_base = volume_file_name(ec00_dir, collection, volume_id);
let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
let base = volume_file_name(dir, collection, volume_id);
// Read volume version from .ec00 superblock
let ec00_path = format!("{}.ec00", ec00_base);
let ec00_path = format!("{}.ec00", base);
let mut ec00 = File::open(&ec00_path)?;
let mut sb_buf = [0u8; SUPER_BLOCK_SIZE];
ec00.read_exact(&mut sb_buf)?;
@@ -47,7 +30,7 @@ pub fn find_dat_file_size_with_dirs(
let mut dat_size: i64 = SUPER_BLOCK_SIZE as i64;
// Scan .ecx entries
let ecx_path = format!("{}.ecx", ecx_base);
let ecx_path = format!("{}.ecx", base);
let ecx_data = std::fs::read(&ecx_path)?;
let entry_count = ecx_data.len() / NEEDLE_MAP_ENTRY_SIZE;
@@ -69,10 +52,7 @@ pub fn find_dat_file_size_with_dirs(
/// Reconstruct a .dat file from EC data shards.
///
/// Reads from .ec00-.ec09 and writes a new .dat file. All data shards
/// must live in `dir`. For the cross-disk reconciled layout where
/// shards are split across multiple data dirs of the same node, use
/// [`write_dat_file_from_shards_with_dirs`] instead.
/// Reads from .ec00-.ec09 and writes a new .dat file.
pub fn write_dat_file_from_shards(
dir: &str,
collection: &str,
@@ -80,53 +60,12 @@ pub fn write_dat_file_from_shards(
dat_file_size: i64,
data_shards: usize,
) -> io::Result<()> {
let dirs: Vec<String> = (0..data_shards).map(|_| dir.to_string()).collect();
write_dat_file_from_shards_with_dirs(
dir,
collection,
volume_id,
dat_file_size,
data_shards,
&dirs,
)
}
/// Reconstruct a .dat file from EC data shards, taking the source
/// directory for each shard separately.
///
/// `dat_dir` is where the produced `.dat` is written. `shard_dirs[i]`
/// is the directory holding shard `i`. For the simple "all shards in
/// one dir" case both can be the same value.
///
/// Mirrors Go's `WriteDatFile(baseFileName, datFileSize,
/// shardFileNames)` shape — Go passes per-shard paths so a
/// reconciled volume with shards split across disks of the same
/// volume server can still be decoded back to a regular .dat
/// (seaweedfs/seaweedfs#9252).
pub fn write_dat_file_from_shards_with_dirs(
dat_dir: &str,
collection: &str,
volume_id: VolumeId,
dat_file_size: i64,
data_shards: usize,
shard_dirs: &[String],
) -> io::Result<()> {
if shard_dirs.len() < data_shards {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"shard_dirs len {} < data_shards {}",
shard_dirs.len(),
data_shards
),
));
}
let base = volume_file_name(dat_dir, collection, volume_id);
let base = volume_file_name(dir, collection, volume_id);
let dat_path = format!("{}.dat", base);
// Open data shards from their individual home dirs.
// Open data shards
let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
.map(|i| EcVolumeShard::new(&shard_dirs[i as usize], collection, volume_id, i))
.map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
.collect();
for shard in &mut shards {
@@ -3,10 +3,9 @@
//! Each EcVolume has a sorted index (.ecx) and a deletion journal (.ecj).
//! Shards (.ec00-.ec13) may be distributed across multiple servers.
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::fs::{self, File, OpenOptions};
use std::io::{self, Write};
use std::sync::RwLock;
use std::time::{SystemTime, UNIX_EPOCH};
use crate::pb::master_pb;
@@ -29,17 +28,6 @@ pub struct EcVolume {
ecx_file: Option<File>,
ecx_file_size: i64,
ecj_file: Option<File>,
/// On-disk size of the .ecj deletion journal. Used only by IO helpers
/// (seek / set_len on partial writes) — the authoritative runtime
/// delete count comes from `deleted_needles.len()`.
ecj_file_size: i64,
/// In-memory set of needle ids that have been deleted since the volume
/// was encoded. .ecx is immutable at runtime — it only stores the
/// sorted (id, offset, size) index written at encode time — and runtime
/// deletes are journaled to .ecj + tracked here. Reads consult this
/// set to mask out deleted needles on top of the sealed .ecx lookup.
/// Seeded from .ecj in `new()` and updated by `journal_delete`.
deleted_needles: RwLock<HashSet<NeedleId>>,
pub disk_type: DiskType,
/// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
ecx_actual_dir: String,
@@ -50,42 +38,11 @@ pub struct EcVolume {
pub expire_at_sec: u64,
}
/// Locate the `.vif` for a (collection, vid) by preferring the data dir
/// and falling back to the idx dir when it lives there instead. The
/// fallback covers the cross-disk reconcile path: when a volume's
/// shards live on one disk but its `.ecx` / `.ecj` / `.vif` live on a
/// sibling disk (seaweedfs/seaweedfs#9212 / #9244), we want to read the
/// real `.vif` from the sibling rather than write a stub on the shard
/// disk and lose the EC config + dat file size.
fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
let data_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir, collection, volume_id),
);
if dir_idx != dir && !std::path::Path::new(&data_vif).exists() {
let idx_vif = format!(
"{}.vif",
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
);
if std::path::Path::new(&idx_vif).exists() {
return idx_vif;
}
}
data_vif
}
/// Read EC data/parity shard counts from `.vif`, defaulting to the
/// build's standard ratio when no `.vif` is present or is malformed.
/// Looks at the data dir first, then the idx dir — see [`locate_vif_path`].
pub fn read_ec_shard_config(
dir: &str,
dir_idx: &str,
collection: &str,
volume_id: VolumeId,
) -> (u32, u32) {
pub fn read_ec_shard_config(dir: &str, collection: &str, volume_id: VolumeId) -> (u32, u32) {
let mut data_shards = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
let mut parity_shards = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
let vif_path = format!("{}.vif", base);
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
if let Ok(vif_info) =
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
@@ -112,7 +69,7 @@ impl EcVolume {
collection: &str,
volume_id: VolumeId,
) -> io::Result<Self> {
let (data_shards, parity_shards) = read_ec_shard_config(dir, dir_idx, collection, volume_id);
let (data_shards, parity_shards) = read_ec_shard_config(dir, collection, volume_id);
let total_shards = (data_shards + parity_shards) as usize;
let mut shards = Vec::with_capacity(total_shards);
@@ -120,16 +77,10 @@ impl EcVolume {
shards.push(None);
}
// Read expire_at_sec, version, and dat_file_size from .vif if
// present (matches Go's MaybeLoadVolumeInfo). Prefer the data
// dir; fall back to the idx dir for the cross-disk reconcile
// case (#9212 / #9244). `dat_file_size` is the source .dat
// size at encode time, used both by `locate_ec_shard_needle`
// for shard-size math and by the Store-level prune in
// `store_ec_reconcile.rs` to verify a sibling-disk .dat is
// plausibly the encoding source (#9478).
let (expire_at_sec, vif_version, vif_dat_file_size) = {
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
// Read expire_at_sec and version from .vif if present (matches Go's MaybeLoadVolumeInfo)
let (expire_at_sec, vif_version) = {
let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
let vif_path = format!("{}.vif", base);
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
if let Ok(vif_info) =
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
@@ -139,12 +90,12 @@ impl EcVolume {
} else {
Version::current()
};
(vif_info.expire_at_sec, ver, vif_info.dat_file_size)
(vif_info.expire_at_sec, ver)
} else {
(0, Version::current(), 0)
(0, Version::current())
}
} else {
(0, Version::current(), 0)
(0, Version::current())
}
};
@@ -155,14 +106,12 @@ impl EcVolume {
dir_idx: dir_idx.to_string(),
version: vif_version,
shards,
dat_file_size: vif_dat_file_size,
dat_file_size: 0,
data_shards,
parity_shards,
ecx_file: None,
ecx_file_size: 0,
ecj_file: None,
ecj_file_size: 0,
deleted_needles: RwLock::new(HashSet::new()),
disk_type: DiskType::default(),
ecx_actual_dir: dir_idx.to_string(),
shard_locations: HashMap::new(),
@@ -192,12 +141,10 @@ impl EcVolume {
}
}
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency.
// Note: Go does NOT replay .ecj into .ecx at volume load (RebuildEcxFile
// is only invoked from specific decode/rebuild gRPC handlers), so we
// don't either. Tombstones from prior sessions were already written
// in-place in .ecx, and the journal grows monotonically until a
// decode/rebuild operation folds it in.
// Replay .ecj journal into .ecx on startup (matches Go's RebuildEcxFile).
vol.rebuild_ecx_from_journal()?;
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency
let ecj_base =
crate::storage::volume::volume_file_name(&vol.ecx_actual_dir, collection, volume_id);
let ecj_path = format!("{}.ecj", ecj_base);
@@ -207,75 +154,11 @@ impl EcVolume {
.create(true)
.append(true)
.open(&ecj_path)?;
vol.ecj_file_size = ecj_file.metadata()?.len() as i64;
vol.ecj_file = Some(ecj_file);
// Seed the in-memory deleted set from the journal.
vol.load_deleted_needles_from_ecj()?;
Ok(vol)
}
/// Walk the .ecj journal and populate `deleted_needles`. Called once
/// from `new()` under exclusive ownership of the just-constructed
/// EcVolume, so locking is not strictly required — but we take the
/// write lock anyway for symmetry with later mutations.
fn load_deleted_needles_from_ecj(&mut self) -> io::Result<()> {
let ecj_file = match self.ecj_file.as_ref() {
Some(f) => f,
None => return Ok(()),
};
if self.ecj_file_size < NEEDLE_ID_SIZE as i64 {
return Ok(());
}
let mut buf = [0u8; NEEDLE_ID_SIZE];
let mut set = self
.deleted_needles
.write()
.map_err(|_| io::Error::new(io::ErrorKind::Other, "deleted_needles lock poisoned"))?;
let mut off: i64 = 0;
while off + NEEDLE_ID_SIZE as i64 <= self.ecj_file_size {
#[cfg(unix)]
{
use std::os::unix::fs::FileExt;
ecj_file.read_exact_at(&mut buf, off as u64)?;
}
set.insert(NeedleId::from_bytes(&buf));
off += NEEDLE_ID_SIZE as i64;
}
Ok(())
}
/// Returns (file_count, delete_count) for this EC volume. Mirrors Go's
/// `EcVolume.FileAndDeleteCount`:
///
/// file_count = ecx_file_size / NEEDLE_MAP_ENTRY_SIZE — total
/// entries in the sealed sorted .ecx index.
/// delete_count = deleted_needles.len() — unique
/// runtime deletes tracked in memory (seeded from
/// .ecj on load and updated by `journal_delete`).
///
/// Because each needle delete is applied on exactly one shard holder,
/// the admin aggregation sums delete_count across nodes while taking
/// file_count from a single holder (they are identical per volume).
pub fn file_and_delete_count(&self) -> (u64, u64) {
let file_count = (self.ecx_file_size as u64) / (NEEDLE_MAP_ENTRY_SIZE as u64);
let delete_count = self
.deleted_needles
.read()
.map(|s| s.len() as u64)
.unwrap_or(0);
(file_count, delete_count)
}
/// Reports whether the given needle id is in the in-memory deleted set.
pub fn is_needle_deleted(&self, needle_id: NeedleId) -> bool {
self.deleted_needles
.read()
.map(|s| s.contains(&needle_id))
.unwrap_or(false)
}
// ---- File names ----
#[allow(dead_code)]
@@ -283,16 +166,8 @@ impl EcVolume {
crate::storage::volume::volume_file_name(&self.dir, &self.collection, self.volume_id)
}
/// Base path for the .ecx / .ecj index pair. Resolved from
/// `ecx_actual_dir` (initialized to `dir_idx` and only updated after a
/// successful idx-dir → data-dir fallback in `new()`), so every call site
/// agrees on the same file regardless of whether the fallback fired.
fn idx_base_name(&self) -> String {
crate::storage::volume::volume_file_name(
&self.ecx_actual_dir,
&self.collection,
self.volume_id,
)
crate::storage::volume::volume_file_name(&self.dir_idx, &self.collection, self.volume_id)
}
pub fn ecx_file_name(&self) -> String {
@@ -332,20 +207,6 @@ impl EcVolume {
Ok(())
}
/// Override the disk type the EC volume (and its already-mounted
/// shards) reports under. Used by the `VolumeEcShardsMount` handler
/// so the source volume's disk type is preserved across encoding
/// (#9423). Not persisted across restarts — disk-scan reload paths
/// default to the physical location's disk type.
pub fn set_disk_type(&mut self, d: DiskType) {
self.disk_type = d.clone();
for slot in self.shards.iter_mut() {
if let Some(shard) = slot {
shard.disk_type = d.clone();
}
}
}
/// Remove and close a shard.
pub fn remove_shard(&mut self, shard_id: ShardId) {
if let Some(ref mut shard) = self.shards[shard_id as usize] {
@@ -370,24 +231,6 @@ impl EcVolume {
self.shards.iter().filter(|s| s.is_some()).count()
}
/// Reports whether `shard_id` is currently registered to this
/// EcVolume (used by the cross-disk reconcile to skip already-
/// loaded shards).
pub fn has_shard(&self, shard_id: u8) -> bool {
self.shards
.get(shard_id as usize)
.map(|s| s.is_some())
.unwrap_or(false)
}
/// Directory where this EcVolume's `.ecx` was actually opened
/// (may differ from `dir_idx` when the legacy "written before
/// -dir.idx was set" fallback or the cross-disk reconcile path
/// pointed it elsewhere).
pub fn ecx_actual_dir(&self) -> &str {
&self.ecx_actual_dir
}
pub fn is_time_to_destroy(&self) -> bool {
self.expire_at_sec > 0
&& SystemTime::now()
@@ -412,8 +255,6 @@ impl EcVolume {
return Vec::new();
}
let (file_count, delete_count) = self.file_and_delete_count();
vec![master_pb::VolumeEcShardInformationMessage {
id: self.volume_id.0,
collection: self.collection.clone(),
@@ -422,8 +263,6 @@ impl EcVolume {
disk_type: self.disk_type.to_string(),
expire_at_sec: self.expire_at_sec,
disk_id,
file_count,
delete_count,
..Default::default()
}]
}
@@ -474,13 +313,6 @@ impl EcVolume {
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
if key == needle_id {
// Apply runtime deletion state on top of the sealed .ecx
// lookup: a needle in the in-memory deleted set is
// reported with TOMBSTONE_FILE_SIZE even though the .ecx
// record itself is untouched.
if self.is_needle_deleted(needle_id) {
return Ok(Some((offset, TOMBSTONE_FILE_SIZE)));
}
return Ok(Some((offset, size)));
} else if key < needle_id {
lo = mid + 1;
@@ -738,41 +570,38 @@ impl EcVolume {
// ---- Deletion ----
/// Write `TOMBSTONE_FILE_SIZE` over the Size field of an existing .ecx
/// entry, matching Go's `MarkNeedleDeleted`. Only used by the offline
/// `rebuild_ecx_from_journal` path — the runtime delete path does not
/// touch .ecx because the index is treated as an immutable sorted
/// (id, offset, size) table. Returns `false` if the needle is not in
/// the index (ignored by callers) and an error on IO failure.
fn tombstone_ecx_entry(&self, needle_id: NeedleId) -> io::Result<bool> {
let ecx_file = self.ecx_file.as_ref().ok_or_else(|| {
io::Error::new(
io::ErrorKind::Other,
format!(
"ec volume {} has no open .ecx file (closed or corrupt)",
self.volume_id.0
),
)
})?;
/// Mark a needle as deleted in the .ecx file in-place.
/// Matches Go's MarkNeedleDeleted: binary search the .ecx, then overwrite
/// the size field with TOMBSTONE_FILE_SIZE.
fn mark_needle_deleted_in_ecx(&self, needle_id: NeedleId) -> io::Result<bool> {
let ecx_file = match self.ecx_file.as_ref() {
Some(f) => f,
None => return Ok(false),
};
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
if entry_count == 0 {
return Ok(false);
}
// Binary search for the needle
let mut lo: usize = 0;
let mut hi: usize = entry_count;
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
while lo < hi {
let mid = lo + (hi - lo) / 2;
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
#[cfg(unix)]
{
use std::os::unix::fs::FileExt;
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
}
let (key, _offset, _old_size) = idx_entry_from_bytes(&entry_buf);
let (key, _offset, _size) = idx_entry_from_bytes(&entry_buf);
if key == needle_id {
// Found — overwrite the size field with TOMBSTONE_FILE_SIZE
let size_offset = file_offset + NEEDLE_ID_SIZE as u64 + OFFSET_SIZE as u64;
let mut size_buf = [0u8; SIZE_SIZE];
TOMBSTONE_FILE_SIZE.to_bytes(&mut size_buf);
@@ -788,18 +617,13 @@ impl EcVolume {
hi = mid;
}
}
Ok(false)
Ok(false) // not found
}
/// Replay .ecj journal entries into .ecx: for each needle id in .ecj,
/// overwrite its .ecx size field with a tombstone, then remove the
/// journal file. Mirrors Go's `RebuildEcxFile`, which is invoked from
/// specific decode / rebuild gRPC handlers — it is intentionally
/// **not** called on volume load (runtime reads consult
/// `deleted_needles` instead). The rebuild is atomic with respect to
/// the journal: if any individual write fails the .ecj file is left
/// in place and the error is propagated so tombstones are not lost.
#[allow(dead_code)]
/// Replay .ecj journal entries into .ecx on startup.
/// Matches Go's RebuildEcxFile: for each needle ID in .ecj, marks it
/// deleted in .ecx, then removes the .ecj file.
fn rebuild_ecx_from_journal(&mut self) -> io::Result<()> {
let ecj_path = self.ecj_file_name();
if !std::path::Path::new(&ecj_path).exists() {
@@ -818,23 +642,14 @@ impl EcVolume {
break;
}
let needle_id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
// A needle that never made it into .ecx is fine (e.g. the
// delete raced against encode). Any other IO error aborts the
// rebuild so the journal survives to be retried later.
self.tombstone_ecx_entry(needle_id)?;
// Errors for individual entries are non-fatal (needle may not exist in .ecx)
let _ = self.mark_needle_deleted_in_ecx(needle_id);
}
// Durably flush the newly-written .ecx tombstones before dropping
// the journal: the writes went through write_all_at and may still
// be in page cache.
if let Some(ref ecx_file) = self.ecx_file {
ecx_file.sync_all()?;
}
// Remove the .ecj file after replay (matches Go)
let _ = fs::remove_file(&ecj_path);
// Fold successful — drop and recreate the journal, clear the
// in-memory deleted set (all of its contents are now materialized
// in .ecx), and reset the cached size.
fs::remove_file(&ecj_path)?;
// Re-create .ecj for future deletions
let ecj_file = OpenOptions::new()
.read(true)
.write(true)
@@ -842,126 +657,27 @@ impl EcVolume {
.append(true)
.open(&ecj_path)?;
self.ecj_file = Some(ecj_file);
self.ecj_file_size = 0;
if let Ok(mut set) = self.deleted_needles.write() {
set.clear();
}
Ok(())
}
// ---- Deletion journal ----
/// Record a needle delete: append the id to the .ecj deletion journal
/// and insert it into the in-memory deleted set. `.ecx` is not touched
/// at runtime — it is a sealed sorted (id, offset, size) index and
/// runtime deletion state lives exclusively in .ecj + `deleted_needles`.
/// A lookup via `find_needle_from_ecx` masks the id out by returning
/// `TOMBSTONE_FILE_SIZE` on a subsequent read.
///
/// The .ecj append is the durable commit point. On any failure the
/// file is truncated back to the pre-append length so the on-disk
/// journal and in-memory state cannot drift. Only after the sync
/// succeeds is the id published into the set, so a failure leaves
/// the delete invisible to readers.
/// Append a deleted needle ID to the .ecj journal and mark in .ecx.
/// Matches Go's DeleteNeedleFromEcx: marks in .ecx first, then journals.
pub fn journal_delete(&mut self, needle_id: NeedleId) -> io::Result<()> {
// Look the needle up read-only. Missing is a silent no-op; a
// pre-existing .ecx tombstone (from a prior decode/rebuild) is
// mirrored into the in-memory set so delete_count stays accurate
// without needing to walk .ecx on every heartbeat.
match self.find_needle_from_ecx_raw(needle_id)? {
None => return Ok(()),
Some((_, size)) if size.is_deleted() => {
if let Ok(mut set) = self.deleted_needles.write() {
set.insert(needle_id);
}
return Ok(());
}
Some(_) => {}
}
// Mark deleted in .ecx in-place (matches Go's MarkNeedleDeleted)
let _ = self.mark_needle_deleted_in_ecx(needle_id);
let ecj_file = self
.ecj_file
.as_mut()
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
// Idempotent fast path for repeat deletes — avoids the journal
// append entirely so the derived delete_count stays stable.
if self.is_needle_deleted(needle_id) {
return Ok(());
}
let prev_ecj_size = self.ecj_file_size;
let append_result: io::Result<()> = {
let ecj_file = self
.ecj_file
.as_mut()
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
let mut buf = [0u8; NEEDLE_ID_SIZE];
needle_id.to_bytes(&mut buf);
ecj_file
.write_all(&buf)
.and_then(|_| ecj_file.sync_all())
};
match append_result {
Ok(()) => {
self.ecj_file_size += NEEDLE_ID_SIZE as i64;
if let Ok(mut set) = self.deleted_needles.write() {
set.insert(needle_id);
}
Ok(())
}
Err(e) => {
// write_all may have extended the file on disk before
// sync_all failed; truncate back to the known-good size so
// the on-disk journal never drifts past `deleted_needles`.
if let Some(ecj) = self.ecj_file.as_mut() {
if let Err(trunc_err) = ecj.set_len(prev_ecj_size as u64) {
tracing::error!(
volume_id = self.volume_id.0,
needle_id = needle_id.0,
truncate_error = %trunc_err,
"failed to truncate ecj after append failure"
);
}
}
Err(e)
}
}
}
/// Internal: binary search .ecx without masking by `deleted_needles`.
/// Used by `journal_delete` so a repeat delete can still see the raw
/// pre-existing .ecx tombstone from a prior rebuild.
fn find_needle_from_ecx_raw(
&self,
needle_id: NeedleId,
) -> io::Result<Option<(Offset, Size)>> {
let ecx_file = self
.ecx_file
.as_ref()
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecx file not open"))?;
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
if entry_count == 0 {
return Ok(None);
}
let mut lo: usize = 0;
let mut hi: usize = entry_count;
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
while lo < hi {
let mid = lo + (hi - lo) / 2;
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
#[cfg(unix)]
{
use std::os::unix::fs::FileExt;
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
}
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
if key == needle_id {
return Ok(Some((offset, size)));
} else if key < needle_id {
lo = mid + 1;
} else {
hi = mid;
}
}
Ok(None)
let mut buf = [0u8; NEEDLE_ID_SIZE];
needle_id.to_bytes(&mut buf);
ecj_file.write_all(&buf)?;
ecj_file.sync_all()?;
Ok(())
}
/// Append a deleted needle ID to the .ecj journal, validating the cookie first.
@@ -1062,17 +778,15 @@ impl EcVolume {
let _ = fs::remove_file(format!("{}.ecx", actual_base));
let _ = fs::remove_file(format!("{}.ecj", actual_base));
let _ = fs::remove_file(format!("{}.vif", actual_base));
// Also sweep the originally-configured idx dir in case stale files
// exist there (ecx_file_name() / ecj_file_name() now resolve from
// ecx_actual_dir, so we have to build the idx-dir paths explicitly).
// Also try the configured idx dir and data dir in case files exist in either
if self.ecx_actual_dir != self.dir_idx {
let _ = fs::remove_file(self.ecx_file_name());
let _ = fs::remove_file(self.ecj_file_name());
let idx_base = crate::storage::volume::volume_file_name(
&self.dir_idx,
&self.collection,
self.volume_id,
);
let _ = fs::remove_file(format!("{}.ecx", idx_base));
let _ = fs::remove_file(format!("{}.ecj", idx_base));
let _ = fs::remove_file(format!("{}.vif", idx_base));
}
if self.ecx_actual_dir != self.dir && self.dir_idx != self.dir {
@@ -1145,33 +859,16 @@ mod tests {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
// .ecj append is gated on a live->tombstone transition in .ecx, so
// the fixture must contain the needles we are about to delete.
let entries = vec![
(NeedleId(10), Offset::from_actual_offset(8), Size(100)),
(NeedleId(20), Offset::from_actual_offset(200), Size(200)),
];
write_ecx_file(dir, "", VolumeId(1), &entries);
// Need ecx file for EcVolume::new to succeed
write_ecx_file(dir, "", VolumeId(1), &[]);
let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
let (fc0, dc0) = vol.file_and_delete_count();
assert_eq!((fc0, dc0), (2, 0));
vol.journal_delete(NeedleId(10)).unwrap();
vol.journal_delete(NeedleId(20)).unwrap();
let deleted = vol.read_deleted_needles().unwrap();
assert_eq!(deleted, vec![NeedleId(10), NeedleId(20)]);
let (fc, dc) = vol.file_and_delete_count();
assert_eq!((fc, dc), (2, 2));
// Idempotent re-delete must not bump delete_count.
vol.journal_delete(NeedleId(10)).unwrap();
// Deleting a missing needle must not bump delete_count either.
vol.journal_delete(NeedleId(999)).unwrap();
let (fc, dc) = vol.file_and_delete_count();
assert_eq!((fc, dc), (2, 2));
}
#[test]
-1
View File
@@ -4,7 +4,6 @@ pub mod idx;
pub mod needle;
pub mod needle_map;
pub mod store;
pub mod store_ec_reconcile;
pub mod super_block;
pub mod types;
pub mod volume;
+12 -74
View File
@@ -19,7 +19,6 @@ use compact_map::CompactMap;
use redb::{Database, Durability, ReadableDatabase, ReadableTable, TableDefinition};
use crate::storage::idx;
use crate::storage::needle::needle::get_actual_size;
use crate::storage::types::*;
// ============================================================================
@@ -64,11 +63,6 @@ pub struct NeedleMapMetric {
pub deletion_count: AtomicI64,
pub deletion_byte_count: AtomicU64,
pub max_file_key: AtomicU64,
/// Largest (offset.to_actual_offset() + get_actual_size(size, version))
/// observed during the load walk. Used at volume load to verify that no
/// .idx entry references bytes past the end of .dat (issue #8928)
/// without paying for a second linear scan.
pub max_needle_end: AtomicI64,
}
impl NeedleMapMetric {
@@ -114,29 +108,6 @@ impl NeedleMapMetric {
}
}
}
/// Update `max_needle_end` if this entry's (offset + actual size) exceeds
/// the running maximum. Skips deleted/zero-offset entries because they
/// don't reserve space in .dat.
fn maybe_set_max_needle_end(&self, offset: Offset, size: Size, version: Version) {
if offset.is_zero() || !size.is_valid() {
return;
}
let end = offset.to_actual_offset() + get_actual_size(size, version);
loop {
let current = self.max_needle_end.load(Ordering::Relaxed);
if end <= current {
break;
}
if self
.max_needle_end
.compare_exchange(current, end, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
break;
}
}
}
}
// ============================================================================
@@ -146,9 +117,9 @@ impl NeedleMapMetric {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NeedleMapKind {
InMemory,
Redb,
RedbMedium,
RedbLarge,
LevelDb,
LevelDbMedium,
LevelDbLarge,
}
// ============================================================================
@@ -192,10 +163,9 @@ impl CompactNeedleMap {
}
/// Load from an .idx file, building the in-memory map.
pub fn load_from_idx<R: Read + Seek>(reader: &mut R, version: Version) -> io::Result<Self> {
pub fn load_from_idx<R: Read + Seek>(reader: &mut R) -> io::Result<Self> {
let mut nm = CompactNeedleMap::new();
idx::walk_index_file(reader, 0, |key, offset, size| {
nm.metric.maybe_set_max_needle_end(offset, size, version);
if offset.is_zero() || size.is_deleted() {
nm.delete_from_map(key);
} else {
@@ -314,12 +284,6 @@ impl CompactNeedleMap {
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
}
/// Largest (offset + actual size) seen during the load walk; 0 if the
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
pub fn max_needle_end(&self) -> i64 {
self.metric.max_needle_end.load(Ordering::Relaxed)
}
pub fn index_file_size(&self) -> u64 {
self.idx_file_offset
}
@@ -467,7 +431,7 @@ impl RedbNeedleMap {
/// Rebuild metrics by scanning all entries in the redb table.
/// Called when reusing an existing .rdb without a full rebuild.
fn rebuild_metrics_from_db(&self, version: Version) -> io::Result<()> {
fn rebuild_metrics_from_db(&self) -> io::Result<()> {
let txn = self
.db
.begin_read()
@@ -489,8 +453,6 @@ impl RedbNeedleMap {
arr.copy_from_slice(bytes);
let nv = unpack_needle_value(&arr);
self.metric.maybe_set_max_file_key(key);
self.metric
.maybe_set_max_needle_end(nv.offset, nv.size, version);
if nv.size.is_valid() {
self.metric.file_count.fetch_add(1, Ordering::Relaxed);
self.metric
@@ -515,24 +477,20 @@ impl RedbNeedleMap {
/// 2. If .idx size matches → reuse .rdb, rebuild metrics from scan
/// 3. If .idx is larger → replay new entries incrementally
/// 4. Otherwise (missing, corrupted, .idx smaller) → full rebuild
pub fn load_from_idx<R: Read + Seek>(
db_path: &str,
reader: &mut R,
version: Version,
) -> io::Result<Self> {
pub fn load_from_idx<R: Read + Seek>(db_path: &str, reader: &mut R) -> io::Result<Self> {
let idx_size = reader.seek(io::SeekFrom::End(0))?;
reader.seek(io::SeekFrom::Start(0))?;
// Try to reuse existing .rdb
if Path::new(db_path).exists() {
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size, version) {
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size) {
return Ok(nm);
}
// Reuse failed — fall through to full rebuild
reader.seek(io::SeekFrom::Start(0))?;
}
Self::full_rebuild(db_path, reader, idx_size, version)
Self::full_rebuild(db_path, reader, idx_size)
}
/// Try to reuse an existing .rdb file. Returns Ok if successful,
@@ -541,7 +499,6 @@ impl RedbNeedleMap {
db_path: &str,
reader: &mut R,
idx_size: u64,
version: Version,
) -> io::Result<Self> {
let db = Database::open(db_path)
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("redb open: {}", e)))?;
@@ -566,7 +523,7 @@ impl RedbNeedleMap {
}
// Rebuild metrics from existing data
nm.rebuild_metrics_from_db(version)?;
nm.rebuild_metrics_from_db()?;
if stored_idx_size < idx_size {
// .idx grew — replay new entries incrementally
@@ -577,7 +534,6 @@ impl RedbNeedleMap {
io::Error::new(io::ErrorKind::Other, format!("redb open_table: {}", e))
})?;
idx::walk_index_file(reader, start_entry, |key, offset, size| {
nm.metric.maybe_set_max_needle_end(offset, size, version);
let key_u64: u64 = key.into();
if offset.is_zero() || size.is_deleted() {
// Delete: look up old value for metric update, then
@@ -651,7 +607,6 @@ impl RedbNeedleMap {
db_path: &str,
reader: &mut R,
idx_size: u64,
version: Version,
) -> io::Result<Self> {
let _ = std::fs::remove_file(db_path);
let nm = RedbNeedleMap::new(db_path)?;
@@ -659,7 +614,6 @@ impl RedbNeedleMap {
// Collect entries from idx file, resolving duplicates/deletions
let mut entries: HashMap<NeedleId, Option<NeedleValue>> = HashMap::new();
idx::walk_index_file(reader, 0, |key, offset, size| {
nm.metric.maybe_set_max_needle_end(offset, size, version);
if offset.is_zero() || size.is_deleted() {
entries.insert(key, None);
} else {
@@ -834,12 +788,6 @@ impl RedbNeedleMap {
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
}
/// Largest (offset + actual size) seen during the load walk; 0 if the
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
pub fn max_needle_end(&self) -> i64 {
self.metric.max_needle_end.load(Ordering::Relaxed)
}
pub fn index_file_size(&self) -> u64 {
self.idx_file_offset
}
@@ -1040,16 +988,6 @@ impl NeedleMap {
}
}
/// Largest (offset + actual size) seen during the load walk; 0 if the
/// map is empty. Used at volume load to detect .idx entries that
/// reference past the end of .dat (issue #8928) without a second scan.
pub fn max_needle_end(&self) -> i64 {
match self {
NeedleMap::InMemory(nm) => nm.max_needle_end(),
NeedleMap::Redb(nm) => nm.max_needle_end(),
}
}
/// Index file size in bytes.
pub fn index_file_size(&self) -> u64 {
match self {
@@ -1219,7 +1157,7 @@ mod tests {
.unwrap();
let mut cursor = Cursor::new(idx_data);
let nm = CompactNeedleMap::load_from_idx(&mut cursor, Version::current()).unwrap();
let nm = CompactNeedleMap::load_from_idx(&mut cursor).unwrap();
assert!(nm.get(NeedleId(1)).is_some());
assert!(nm.get(NeedleId(2)).is_none()); // deleted
@@ -1362,7 +1300,7 @@ mod tests {
.unwrap();
let mut cursor = Cursor::new(idx_data);
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor, Version::current()).unwrap();
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor).unwrap();
assert!(nm.get(NeedleId(1)).is_some());
assert!(nm.get(NeedleId(2)).is_none()); // deleted and removed
@@ -1439,7 +1377,7 @@ mod tests {
// Load back with CompactNeedleMap to verify
let mut idx_file = std::fs::File::open(&idx_path).unwrap();
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file, Version::current()).unwrap();
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file).unwrap();
assert_eq!(loaded.file_count(), 2); // only live entries
assert!(loaded.get(NeedleId(1)).is_some());
assert!(loaded.get(NeedleId(2)).is_none()); // deleted, not saved
+10 -385
View File
@@ -84,28 +84,6 @@ impl Store {
}
self.locations.push(loc);
// First scrub partial EC artefacts left on one disk by an
// interrupted encode while the source .dat still lives on a
// sibling disk of the same store. The per-disk loader cannot
// see the sibling .dat and so loads the partial shards as if
// they were a distributed-EC layout, which makes the volume
// server heartbeat both a regular replica and an EC shard set
// for the same vid (seaweedfs/seaweedfs#9478). Running before
// the cross-disk reconcile keeps that pass from later
// re-loading shards we just cleaned up.
self.prune_incomplete_ec_with_sibling_dat();
// After every disk has finished its per-disk EC scan, sweep
// the store for shards that live on a disk without local index
// files and load them by reaching across to a sibling disk's
// .ecx / .ecj / .vif (seaweedfs/seaweedfs#9212 / #9244).
// ec.balance / ec.rebuild can move shards onto a destination
// node's second disk while leaving the index on the disk that
// already held the volume; without this pass those orphan
// shards stay invisible to the master.
self.reconcile_ec_shards_across_disks();
Ok(())
}
@@ -117,8 +95,6 @@ impl Store {
tracing::error!("load_new_volumes error in {}: {}", loc.directory, e);
}
}
self.prune_incomplete_ec_with_sibling_dat();
self.reconcile_ec_shards_across_disks();
}
// ---- Volume lookup ----
@@ -221,72 +197,6 @@ impl Store {
best.map(|(i, _)| i)
}
/// Returns the index of the disk that should receive a new EC
/// shard / index file for `(collection, vid)`. Selection order:
///
/// 1. a disk that already has the EC volume mounted (in-memory state),
/// 2. a disk that owns the `.ecx` file on disk (volume not yet mounted),
/// 3. any HDD with free space,
/// 4. any disk with free space.
///
/// Step 2 is the missing primitive that pinned subsequent shards to
/// the first-shard disk during `ec.rebuild`: rebuild only sets
/// `CopyEcxFile=true` on the first shard, then relies on auto-select
/// to land later shards on the same disk. Without an on-disk check,
/// `has_ec_volume` returns false (no mount yet) and the fallback
/// picks "any HDD with free space" — which can split shards from
/// their index files across disks of the same node and lose them at
/// startup. See seaweedfs/seaweedfs#9212.
///
/// `data_shard_count` is taken as a parameter rather than read from
/// `DATA_SHARDS_COUNT` so custom-ratio builds can swap the default
/// without touching this helper.
///
/// Single pass over `self.locations` with tier scoring; the
/// highest-tier disk wins, ties broken by free shard-slot count.
/// Mirrors `Store.FindEcShardTargetLocation` in
/// `weed/storage/store_ec.go`.
pub fn find_ec_shard_target_location(
&self,
collection: &str,
vid: VolumeId,
data_shard_count: u32,
) -> Option<usize> {
const TIER_ANY_DISK: u8 = 1;
const TIER_HDD: u8 = 2;
const TIER_ECX_ON_DISK: u8 = 3;
const TIER_MOUNTED: u8 = 4;
let mut best: Option<(usize, u8, i64)> = None;
for (i, loc) in self.locations.iter().enumerate() {
if loc.is_disk_space_low.load(Ordering::Relaxed) {
continue;
}
let free = ec_free_shard_count(loc, data_shard_count);
if free <= 0 {
continue;
}
let mut tier = TIER_ANY_DISK;
if loc.disk_type == DiskType::HardDrive {
tier = TIER_HDD;
}
if loc.has_ecx_file_on_disk(collection, vid) {
tier = TIER_ECX_ON_DISK;
}
if loc.has_ec_volume(vid) {
tier = TIER_MOUNTED;
}
let better = match best {
None => true,
Some((_, b_tier, b_free)) => tier > b_tier || (tier == b_tier && free > b_free),
};
if better {
best = Some((i, tier, free));
}
}
best.map(|(i, _, _)| i)
}
/// Create a new volume, placing it on the location with the most free space.
pub fn add_volume(
&mut self,
@@ -319,18 +229,11 @@ impl Store {
)
}
/// Delete a volume from any location. When keep_remote_data is true the
/// cloud-tier object backing the volume is left intact — used by moves
/// where another server is taking over the same .vif.
pub fn delete_volume(
&mut self,
vid: VolumeId,
only_empty: bool,
keep_remote_data: bool,
) -> Result<(), VolumeError> {
/// Delete a volume from any location.
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
for loc in &mut self.locations {
if loc.find_volume(vid).is_some() {
return loc.delete_volume(vid, only_empty, keep_remote_data);
return loc.delete_volume(vid, only_empty);
}
}
Err(VolumeError::NotFound)
@@ -589,8 +492,7 @@ impl Store {
let vol_count = loc.volumes_len() as i32;
let loc_ec_shards = loc.ec_shard_count();
let ec_equivalent = ((loc_ec_shards
+ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT
- 1)
+ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT)
/ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT)
as i32;
let mut max_count = vol_count + ec_equivalent;
@@ -648,25 +550,22 @@ impl Store {
))
})?;
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids, "")
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids)
}
/// Mount a single EC shard, searching all locations for the shard file.
/// Matches Go's Store.MountEcShards which mounts one shard at a time.
/// `source_disk_type` is the source volume's disk type from the
/// `VolumeEcShardsMount` RPC (#9423); pass `""` for non-RPC paths.
pub fn mount_ec_shard(
&mut self,
vid: VolumeId,
collection: &str,
shard_id: u32,
source_disk_type: &str,
) -> Result<(), VolumeError> {
for loc in &mut self.locations {
// Check if the shard file exists on this location
let shard = EcVolumeShard::new(&loc.directory, collection, vid, shard_id as u8);
if std::path::Path::new(&shard.file_name()).exists() {
loc.mount_ec_shards(vid, collection, &[shard_id], source_disk_type)?;
loc.mount_ec_shards(vid, collection, &[shard_id])?;
return Ok(());
}
}
@@ -677,18 +576,11 @@ impl Store {
}
/// Unmount EC shards for a volume (batch).
///
/// Iterates every location that has an EcVolume for `vid` and
/// asks it to unmount whatever subset of `shard_ids` it actually
/// has — required for cross-disk reconciled volumes where the
/// requested shards may legitimately live on different disks of
/// the same store (#9252). DiskLocation::unmount_ec_shards
/// already skips shards that aren't mounted, so this is safe to
/// fan out blindly.
pub fn unmount_ec_shards(&mut self, vid: VolumeId, shard_ids: &[u32]) {
for loc in &mut self.locations {
if loc.has_ec_volume(vid) {
loc.unmount_ec_shards(vid, shard_ids);
return;
}
}
}
@@ -696,12 +588,10 @@ impl Store {
/// Unmount a single EC shard, searching all locations.
/// Matches Go's Store.UnmountEcShards which unmounts one shard at a time.
pub fn unmount_ec_shard(&mut self, vid: VolumeId, shard_id: u32) -> Result<(), VolumeError> {
// Walk all locations rather than stopping at the first with the
// vid — split-disk reconciled volumes can have the same vid on
// multiple disks, with the target shard on any of them.
for loc in &mut self.locations {
if loc.has_ec_volume(vid) {
loc.unmount_ec_shards(vid, &[shard_id]);
return Ok(());
}
}
// Go returns nil if shard not found (no error)
@@ -733,75 +623,6 @@ impl Store {
self.locations.iter().any(|loc| loc.has_ec_volume(vid))
}
/// Returns the index of the disk location that has `(vid, shard_id)`
/// mounted, if any. Mirrors Go's `Store.findEcShard` and is the
/// right primitive for read/unmount/delete operations on a single
/// shard, since reconciliation can mount the same `vid` on multiple
/// disks (each holding a disjoint subset of the shards). Without
/// this, callers using `find_ec_volume(vid)` would only see the
/// first disk and miss shards that live on a sibling.
pub fn find_ec_shard_location(&self, vid: VolumeId, shard_id: u32) -> Option<usize> {
for (i, loc) in self.locations.iter().enumerate() {
if let Some(ecv) = loc.find_ec_volume(vid) {
if ecv.has_shard(shard_id as u8) {
return Some(i);
}
}
}
None
}
/// Like [`Self::find_ec_shard_location`] but returns the EcVolume
/// reference directly. Borrows the store immutably for the
/// EcVolume's lifetime.
pub fn find_ec_volume_with_shard(
&self,
vid: VolumeId,
shard_id: u32,
) -> Option<&EcVolume> {
for loc in &self.locations {
if let Some(ecv) = loc.find_ec_volume(vid) {
if ecv.has_shard(shard_id as u8) {
return Some(ecv);
}
}
}
None
}
/// Aggregate the data dir of each mounted shard for `vid` across
/// all locations. Returns the EcVolume to use for metadata
/// (`.ecx`, `dir_idx`, etc — any per-disk EcVolume for this vid
/// will do, since they all open the same `.ecx`) and a vector of
/// per-shard data dirs (`None` when the shard isn't mounted on
/// any disk).
///
/// Mirrors Go's `Store.CollectEcShards` in
/// `weed/storage/store_ec.go`. The decoder needs per-shard paths
/// to support cross-disk reconciled volumes whose shards are
/// split across data dirs.
pub fn collect_ec_shard_dirs(
&self,
vid: VolumeId,
max_shard_count: usize,
) -> Option<(&EcVolume, Vec<Option<String>>)> {
let mut found_vol: Option<&EcVolume> = None;
let mut dirs: Vec<Option<String>> = vec![None; max_shard_count];
for loc in &self.locations {
if let Some(ecv) = loc.find_ec_volume(vid) {
if found_vol.is_none() {
found_vol = Some(ecv);
}
for shard_id in 0..max_shard_count {
if dirs[shard_id].is_none() && ecv.has_shard(shard_id as u8) {
dirs[shard_id] = Some(loc.directory.clone());
}
}
}
}
found_vol.map(|v| (v, dirs))
}
pub fn delete_expired_ec_volumes(
&mut self,
) -> (
@@ -1098,48 +919,6 @@ fn save_vif_volume_info(path: &str, info: &VifVolumeInfo) -> Result<(), VolumeEr
Ok(())
}
/// Free EC shard capacity of `loc`, expressed in shard slots (not
/// volume-equivalent slots). `find_free_location_predicate` does similar
/// math but divides by `data_shard_count` at the end. That truncation can
/// exclude a disk that still has room for several individual shards
/// (e.g. `MaxVolumeCount=1`, `EcShardCount=1`, `data_shard_count=10` →
/// reports 0 despite 9 free shard slots), which would re-route subsequent
/// shards off the `.ecx`-owning disk and re-introduce the orphan-shard
/// layout this helper is meant to prevent (seaweedfs/seaweedfs#9212).
///
/// `MaxVolumeCount == 0` is the "unlimited" sentinel honoured elsewhere
/// in the store; report a synthetic large free count decremented by
/// current usage so unlimited disks are eligible and tie-breaks still
/// prefer the less-loaded one.
///
/// Mirrors `ecFreeShardCount` in `weed/storage/store_ec.go`.
fn ec_free_shard_count(loc: &DiskLocation, data_shard_count: u32) -> i64 {
if data_shard_count == 0 {
return 0;
}
let dsc = data_shard_count as i64;
let max = loc.max_volume_count.load(Ordering::Relaxed) as i64;
if max <= 0 {
// Synthetic "unlimited" capacity. Use a large but
// well-below-overflow base (1 << 60 ≈ 1.15e18) and saturating
// arithmetic so even pathological usage never wraps and
// tie-breaks among unlimited disks still meaningfully prefer
// the less-loaded one. Clamp to ≥ 1 so the disk stays eligible
// for placement no matter how loaded it is.
const UNLIMITED_FREE: i64 = 1 << 60;
let used = (loc.volumes_len() as i64)
.saturating_mul(dsc)
.saturating_add(loc.ec_shard_count() as i64);
return UNLIMITED_FREE.saturating_sub(used).max(1);
}
let mut free = (max - loc.volumes_len() as i64) * dsc;
free -= loc.ec_shard_count() as i64;
if free < 0 {
return 0;
}
free
}
// ============================================================================
// Tests
// ============================================================================
@@ -1148,7 +927,6 @@ fn ec_free_shard_count(loc: &DiskLocation, data_shard_count: u32) -> i64 {
mod tests {
use super::*;
use crate::storage::needle::needle::Needle;
use crate::storage::volume::volume_file_name;
use tempfile::TempDir;
fn make_test_store(dirs: &[&str]) -> Store {
@@ -1380,13 +1158,7 @@ mod tests {
Vec::new(),
)
.unwrap();
// Use a large volume_size_limit so the unused-space difference between
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
// that integer-division rounding and disk-free fluctuations between the
// two get_disk_stats calls cannot make the quotients equal.
store
.volume_size_limit
.store(100 * 1024 * 1024, Ordering::Relaxed);
store.volume_size_limit.store(1024, Ordering::Relaxed);
store
.add_volume(
VolumeId(61),
@@ -1496,7 +1268,7 @@ mod tests {
std::fs::write(format!("{}/expired_ec_case_9.ec00", dir), b"expired").unwrap();
store.locations[0]
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0], "")
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0])
.unwrap();
store.find_ec_volume_mut(VolumeId(9)).unwrap().expire_at_sec = 1;
@@ -1522,151 +1294,4 @@ mod tests {
let err = store.read_volume_needle(VolumeId(99), &mut n);
assert!(matches!(err, Err(VolumeError::NotFound)));
}
/// Build a Store with N HDD disk locations under a single TempDir.
/// Returns the store and the TempDir guard so callers keep the dirs
/// alive for the test's lifetime.
fn make_ec_target_test_store(numdirs: usize) -> (Store, TempDir) {
let tmp = TempDir::new().unwrap();
let mut store = Store::new(NeedleMapKind::InMemory);
for i in 0..numdirs {
let path = tmp.path().join(format!("data{}", i));
std::fs::create_dir_all(&path).unwrap();
store
.add_location(
path.to_str().unwrap(),
path.to_str().unwrap(),
100,
DiskType::HardDrive,
MinFreeSpace::Percent(0.0),
Vec::new(),
)
.unwrap();
}
(store, tmp)
}
/// Reproduces the placement half of seaweedfs/seaweedfs#9212. After
/// `ec.rebuild`'s first VolumeEcShardsCopy lands `.ecx` on disk N,
/// subsequent shards arrive with `CopyEcxFile=false` and rely on
/// auto-select. Without an on-disk check, `has_ec_volume` returns
/// false (no mount yet) and the fallback picks "any HDD with free
/// space" — splitting shards from their index files across disks.
/// `find_ec_shard_target_location` must pin to the `.ecx`-owning
/// disk via the on-disk check.
#[test]
fn test_find_ec_shard_target_location_pins_to_ecx_on_disk() {
let (store, _tmp) = make_ec_target_test_store(3);
let collection = "grafana-loki";
let vid = VolumeId(1093);
// Drop a sealed .ecx onto disk 2. Nothing is mounted yet — this
// is the state right after the first VolumeEcShardsCopy with
// CopyEcxFile=true and before any VolumeEcShardsMount has run.
let base = volume_file_name(&store.locations[2].idx_directory, collection, vid);
std::fs::write(format!("{}.ecx", base), vec![0u8; 20]).unwrap();
let got = store.find_ec_shard_target_location(collection, vid, 10);
assert_eq!(
got,
Some(2),
"placement leaked off the .ecx-owning disk; got {:?}",
got,
);
}
/// An already-mounted EC volume on disk 1 must win over a stray
/// `.ecx` on disk 2. Protects the post-startup steady state from
/// being perturbed by leftover index files from a prior failed move.
#[test]
fn test_find_ec_shard_target_location_prefers_mounted_over_ecx() {
let (mut store, _tmp) = make_ec_target_test_store(3);
let collection = "grafana-loki";
let vid = VolumeId(2222);
// Mount an EC shard on disk 1 so has_ec_volume returns true.
std::fs::write(
format!("{}/{}_{}.ec00", store.locations[1].directory, collection, vid.0),
b"shard data",
)
.unwrap();
store.locations[1]
.mount_ec_shards(vid, collection, &[0], "")
.unwrap();
// Stray .ecx on disk 2 must not win.
let base = volume_file_name(&store.locations[2].idx_directory, collection, vid);
std::fs::write(format!("{}.ecx", base), vec![0u8; 20]).unwrap();
let got = store.find_ec_shard_target_location(collection, vid, 10);
assert_eq!(got, Some(1), "expected the mounted disk to win; got {:?}", got);
}
/// Cold-volume case: no mount, no `.ecx` anywhere on this server.
/// Selection should still fall through to an HDD fallback.
#[test]
fn test_find_ec_shard_target_location_falls_through_to_hdd_when_nothing_matches() {
let (store, _tmp) = make_ec_target_test_store(2);
let got = store.find_ec_shard_target_location("grafana-loki", VolumeId(3333), 10);
assert!(got.is_some(), "expected an HDD fallback");
assert_eq!(store.locations[got.unwrap()].disk_type, DiskType::HardDrive);
}
/// `MaxVolumeCount=0` is the "unlimited disk" sentinel. The previous
/// formula returned a negative free count for unlimited disks,
/// making placement skip them entirely. The unlimited branch in
/// `ec_free_shard_count` must report a synthetic large free count.
#[test]
fn test_find_ec_shard_target_location_honours_unlimited_disk() {
let (store, _tmp) = make_ec_target_test_store(1);
store.locations[0]
.max_volume_count
.store(0, Ordering::Relaxed);
let got = store.find_ec_shard_target_location("grafana-loki", VolumeId(4444), 10);
assert_eq!(
got,
Some(0),
"expected the only (unlimited) disk to be picked",
);
}
/// Truncation hazard: with `MaxVolumeCount=1, EcShardCount=1,
/// data_shard_count=10`, the old formula `(1*10 - 1) / 10 = 0`
/// would have rounded the disk to "full" and routed subsequent
/// shards elsewhere — the orphan-shard layout this PR exists to
/// prevent. Accounting in shard slots fixes it.
#[test]
fn test_find_ec_shard_target_location_tight_provisioning_keeps_ecx_disk() {
let (mut store, _tmp) = make_ec_target_test_store(2);
store.locations[0]
.max_volume_count
.store(1, Ordering::Relaxed);
store.locations[1]
.max_volume_count
.store(1, Ordering::Relaxed);
let collection = "grafana-loki";
let vid = VolumeId(5555);
// Seed disk 1 with one EC shard so it owns .ecx and has 9
// free shard slots remaining; the old formula would have
// rounded that to 0.
std::fs::write(
format!("{}/{}_{}.ec00", store.locations[1].directory, collection, vid.0),
b"shard data",
)
.unwrap();
store.locations[1]
.mount_ec_shards(vid, collection, &[0], "")
.unwrap();
let got = store.find_ec_shard_target_location(collection, vid, 10);
assert_eq!(
got,
Some(1),
"expected the .ecx-owning disk (1 shard placed, 9 free shard slots) to be picked; got {:?}",
got,
);
}
}
File diff suppressed because it is too large Load Diff
+29 -302
View File
@@ -19,8 +19,6 @@ use std::time::{SystemTime, UNIX_EPOCH};
use tracing::warn;
#[cfg(test)]
use crate::storage::idx;
use crate::storage::needle::needle::{self, get_actual_size, Needle, NeedleError};
use crate::storage::needle_map::{CompactNeedleMap, NeedleMap, NeedleMapKind, RedbNeedleMap};
use crate::storage::super_block::{ReplicaPlacement, SuperBlock, SUPER_BLOCK_SIZE};
@@ -75,24 +73,6 @@ pub enum VolumeError {
StreamingUnsupported,
}
/// Returns true when a needle read failed because the on-disk bytes are
/// unreadable in a permanent way (offset past EOF, header corruption, CRC
/// mismatch, malformed v2/v3/v4 fields). Vacuum can safely drop such entries
/// during compaction. Anything else (real disk EIO, ERROR_CRC on Windows,
/// network timeouts, EROFS, etc.) is transient or environmental and must
/// abort the compaction so an operator notices.
fn is_skippable_needle_read_error(e: &VolumeError) -> bool {
match e {
VolumeError::Io(io_err) => io_err.kind() == io::ErrorKind::UnexpectedEof,
VolumeError::Needle(NeedleError::SizeMismatch { .. }) => true,
VolumeError::Needle(NeedleError::CrcMismatch { .. }) => true,
VolumeError::Needle(NeedleError::IndexOutOfRange(_)) => true,
VolumeError::Needle(NeedleError::TailTooShort) => true,
VolumeError::SizeMismatch => true,
_ => false,
}
}
// ============================================================================
// VolumeInfo (.vif persistence)
// ============================================================================
@@ -732,28 +712,6 @@ impl Volume {
"volumeDataIntegrityChecking failed"
);
}
// Structural check: no .idx entry may reference bytes past the
// end of .dat. The needle map's load walk above already
// populated max_needle_end, so this is a numeric comparison
// — no extra disk I/O. A violation marks the volume read-only
// so vacuum doesn't silently drop reachable data based on a
// corrupt .idx left over from a crashed batched write.
// See issue #8928.
if let Some(ref nm) = self.nm {
if let Ok(dat_size) = self.current_dat_file_size() {
let max_end = nm.max_needle_end();
if dat_size > 0 && max_end > dat_size as i64 {
self.no_write_or_delete = true;
warn!(
volume_id = self.id.0,
max_needle_end = max_end,
dat_size,
"idx references bytes past end of .dat; marking volume read-only"
);
}
}
}
}
}
@@ -773,17 +731,10 @@ impl Volume {
Ok(())
}
fn nm_or_not_found(&self) -> Result<&NeedleMap, VolumeError> {
self.nm.as_ref().ok_or_else(|| {
tracing::warn!(volume_id = self.id.0, "needle map not loaded");
VolumeError::NotFound
})
}
fn load_index(&mut self) -> Result<(), VolumeError> {
let use_redb = matches!(
self.needle_map_kind,
NeedleMapKind::Redb | NeedleMapKind::RedbMedium | NeedleMapKind::RedbLarge
NeedleMapKind::LevelDb | NeedleMapKind::LevelDbMedium | NeedleMapKind::LevelDbLarge
);
let idx_path = self.file_name(".idx");
@@ -808,7 +759,7 @@ impl Volume {
// Open read-only
if Path::new(&idx_path).exists() {
let mut idx_file = File::open(&idx_path)?;
let nm = CompactNeedleMap::load_from_idx(&mut idx_file, self.version())?;
let nm = CompactNeedleMap::load_from_idx(&mut idx_file)?;
self.nm = Some(NeedleMap::InMemory(nm));
} else {
// Missing .idx with existing .dat could orphan needles
@@ -834,7 +785,7 @@ impl Volume {
let idx_size = idx_file.metadata()?.len();
let mut idx_reader = io::BufReader::new(&idx_file);
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader, self.version())?;
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader)?;
// Re-open for append-only writes
let write_file = OpenOptions::new()
@@ -857,7 +808,7 @@ impl Volume {
// Open read-only
if Path::new(&idx_path).exists() {
let mut idx_file = File::open(&idx_path)?;
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file, self.version())?;
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file)?;
self.nm = Some(NeedleMap::Redb(nm));
} else {
// Missing .idx with existing .dat could orphan needles
@@ -883,7 +834,7 @@ impl Volume {
let idx_size = idx_file.metadata()?.len();
let mut idx_reader = io::BufReader::new(&idx_file);
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader, self.version())?;
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader)?;
// Re-open for append-only writes
let write_file = OpenOptions::new()
@@ -1065,7 +1016,7 @@ impl Volume {
read_option: &mut ReadOption,
) -> Result<i32, VolumeError> {
let _guard = self.data_file_access_control.read_lock();
let nm = self.nm_or_not_found()?;
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
if nv.offset.is_zero() {
@@ -1307,7 +1258,7 @@ impl Volume {
read_deleted: bool,
) -> Result<NeedleStreamInfo, VolumeError> {
let _guard = self.data_file_access_control.read_lock();
let nm = self.nm_or_not_found()?;
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
if nv.offset.is_zero() {
@@ -1409,7 +1360,7 @@ impl Volume {
&self,
needle_id: NeedleId,
) -> Result<(u64, u16), VolumeError> {
let nm = self.nm_or_not_found()?;
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
let nv = nm.get(needle_id).ok_or(VolumeError::NotFound)?;
if nv.offset.is_zero() {
return Err(VolumeError::NotFound);
@@ -1726,7 +1677,7 @@ impl Volume {
/// Read all live needles from the volume (for ReadAllNeedles streaming RPC).
pub fn read_all_needles(&self) -> Result<Vec<Needle>, VolumeError> {
let _guard = self.data_file_access_control.read_lock();
let nm = self.nm_or_not_found()?;
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
let version = self.version();
let dat_size = self.current_dat_file_size()? as i64;
let mut needles = Vec::new();
@@ -1801,11 +1752,6 @@ impl Volume {
let version = self.version();
// The deeper-than-tail structural check (every (offset + actual size)
// fits inside .dat — issue #8928) is now handled in load() via the
// needle map's max_needle_end accumulator, so we don't pay for a
// second linear scan of the .idx here.
// Check last 10 index entries (matching Go's CheckVolumeDataIntegrity).
// Go starts healthyIndexSize = indexSize and reduces on EOF.
// On success: break (err != ErrorSizeMismatch when err == nil).
@@ -1915,7 +1861,7 @@ impl Volume {
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
return Err(VolumeError::NotFound);
}
let nm = self.nm_or_not_found()?;
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
let dat_size = self.dat_file_size().map_err(VolumeError::Io)?;
let mut files_checked: u64 = 0;
@@ -1975,7 +1921,7 @@ impl Volume {
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
return Err(VolumeError::NotFound);
}
let nm = self.nm_or_not_found()?;
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
let dat_size = self.dat_file_size().map_err(|e| VolumeError::Io(e))?;
let version = self.version();
@@ -2737,8 +2683,6 @@ impl Volume {
}
entries.sort_by_key(|(_, offset, _)| *offset);
let mut skipped_needles: u64 = 0;
let mut skipped_data_bytes: u64 = 0;
for (id, offset, size) in entries {
// Progress callback
if !progress_fn(offset.to_actual_offset()) {
@@ -2755,41 +2699,7 @@ impl Volume {
id,
..Needle::default()
};
match self.read_needle_data_at(&mut n, offset.to_actual_offset(), size) {
Ok(()) => {}
Err(e) => {
// Record EIO for health monitoring (parity with Go's checkReadWriteError).
if let VolumeError::Io(ref io_err) = e {
self.check_read_write_error(Some(io_err));
}
// Only drop the entry when the failure is one of the well-
// known permanent-corruption shapes. A transient disk fault,
// a tiered-read timeout, or a Windows hardware error (which
// surfaces as a generic Io rather than UnexpectedEof) must
// abort so an operator notices, rather than silently
// compacting away data that might come back on retry.
// See issue #8928.
if !is_skippable_needle_read_error(&e) {
return Err(VolumeError::Io(io::Error::new(
io::ErrorKind::Other,
format!("cannot hydrate needle from file: {}", e),
)));
}
skipped_needles += 1;
if size.is_valid() {
skipped_data_bytes += size.0 as u64;
}
warn!(
volume_id = self.id.0,
key = id.0,
offset = offset.to_actual_offset(),
size = size.0,
error = %e,
"vacuum: dropping unreadable needle"
);
continue;
}
}
self.read_needle_data_at(&mut n, offset.to_actual_offset(), size)?;
// Skip TTL-expired needles using the volume's TTL (matches Go's volume_vacuum.go)
if n.has_ttl() {
@@ -2811,15 +2721,6 @@ impl Volume {
new_offset += bytes.len() as i64;
}
if skipped_needles > 0 {
warn!(
volume_id = self.id.0,
skipped_needles,
skipped_data_bytes,
"vacuum: dropped unreadable index entries during compaction"
);
}
dst.sync_all()?;
// Save new index
@@ -3088,10 +2989,7 @@ impl Volume {
}
/// Remove all volume files from disk.
/// Destroy removes everything related to this volume. When keep_remote_data
/// is true the cloud-tier object backing the volume is left intact — used
/// by moves where another server is taking over the same .vif.
pub fn destroy(&mut self, only_empty: bool, keep_remote_data: bool) -> Result<(), VolumeError> {
pub fn destroy(&mut self, only_empty: bool) -> Result<(), VolumeError> {
if only_empty && self.file_count() > 0 {
return Err(VolumeError::NotEmpty);
}
@@ -3103,11 +3001,7 @@ impl Volume {
}
let (storage_name, storage_key) = self.remote_storage_name_key();
if !keep_remote_data
&& self.has_remote_file
&& !storage_name.is_empty()
&& !storage_key.is_empty()
{
if self.has_remote_file && !storage_name.is_empty() && !storage_key.is_empty() {
let backend = crate::remote_storage::s3_tier::global_s3_tier_registry()
.read()
.unwrap()
@@ -3203,17 +3097,11 @@ fn get_append_at_ns(last: u64) -> u64 {
}
/// Remove all files associated with a volume.
/// .dat/.idx removals log at info level so destructive calls are traceable.
pub(crate) fn remove_volume_files(base: &str) {
for ext in &[
".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
] {
let path = format!("{}{}", base, ext);
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
let existed = fs::remove_file(&path).is_ok();
if existed && (*ext == ".dat" || *ext == ".idx") {
tracing::info!("removed volume file {} (size={})", path, size);
}
let _ = fs::remove_file(format!("{}{}", base, ext));
}
// leveldb uses a directory
let _ = fs::remove_dir_all(format!("{}.ldb", base));
@@ -3337,7 +3225,6 @@ mod tests {
let addr = listener.local_addr().unwrap();
listener.set_nonblocking(true).unwrap();
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
std::thread::spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread()
@@ -3392,7 +3279,6 @@ mod tests {
}));
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
let _ = ready_tx.send(());
axum::serve(listener, app)
.with_graceful_shutdown(async move {
let _ = shutdown_rx.await;
@@ -3402,8 +3288,6 @@ mod tests {
});
});
// Wait for the server thread to be ready before returning.
ready_rx.recv().unwrap();
(format!("http://{}", addr), shutdown_tx)
}
@@ -3679,7 +3563,7 @@ mod tests {
dat_path = v.file_name(".dat");
idx_path = v.file_name(".idx");
assert!(Path::new(&dat_path).exists());
v.destroy(false, false).unwrap();
v.destroy(false).unwrap();
}
assert!(!Path::new(&dat_path).exists());
@@ -3822,138 +3706,6 @@ mod tests {
v.cleanup_compact().unwrap();
}
/// Vacuum compaction must tolerate an .idx entry whose offset points past
/// the end of the .dat file (the failure mode in issue #8928). The bad
/// entry is silently dropped from the resulting .cpx; healthy needles
/// survive untouched.
#[test]
fn test_compact_by_index_drops_dangling_needle() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut v = make_test_volume(dir);
// Write a handful of healthy needles to establish a baseline .dat/.idx.
for i in 1..=5u64 {
let mut n = Needle {
id: NeedleId(i),
cookie: Cookie(i as u32),
data: format!("payload-{}", i).into_bytes(),
data_size: format!("payload-{}", i).len() as u32,
..Needle::default()
};
v.write_needle(&mut n, true).unwrap();
}
v.sync_to_disk().unwrap();
let dat_size = v.dat_file_size().unwrap();
let bad_key = NeedleId(9999);
let bad_offset = Offset::from_actual_offset((dat_size + 1024 * 1024) as i64);
let bad_size = Size(2048);
v.nm
.as_mut()
.expect("needle map present")
.put(bad_key, bad_offset, bad_size)
.unwrap();
v.nm.as_ref().unwrap().sync().unwrap();
// Vacuum must succeed in spite of the dangling entry.
v.compact_by_index(0, 0, |_| true)
.expect("compact_by_index should tolerate dangling entries");
// Walk the resulting .cpx and confirm the bad key was dropped while
// every healthy key made it through.
let cpx_path = v.file_name(".cpx");
let mut cpx = File::open(&cpx_path).unwrap();
let mut kept: Vec<u64> = Vec::new();
idx::walk_index_file(&mut cpx, 0, |key, _, size| {
if !size.is_deleted() {
kept.push(key.0);
}
Ok(())
})
.unwrap();
assert!(
!kept.contains(&bad_key.0),
"dangling key {} should have been dropped from .cpx, got {:?}",
bad_key.0,
kept
);
for i in 1..=5u64 {
assert!(
kept.contains(&i),
"healthy key {} missing from compacted .cpx, got {:?}",
i,
kept
);
}
}
/// The needle map's max_needle_end accumulator must let volume.load
/// detect an .idx whose entries point past the end of the .dat — the
/// deeper-than-tail corruption shape from issue #8928 that the existing
/// last-10-entries scan cannot see. The check is populated by the load
/// walk and read in volume.load() to flip the volume read-only.
#[test]
fn test_max_needle_end_detects_dangling_entry() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let mut v = make_test_volume(dir);
for i in 1..=4u64 {
let mut n = Needle {
id: NeedleId(i),
cookie: Cookie(i as u32),
data: format!("data-{}", i).into_bytes(),
data_size: format!("data-{}", i).len() as u32,
..Needle::default()
};
v.write_needle(&mut n, true).unwrap();
}
v.sync_to_disk().unwrap();
let dat_size = v.dat_file_size().unwrap() as i64;
let idx_path = v.file_name(".idx");
let version = v.version();
// Sanity: a fresh load walk over the healthy .idx puts max_needle_end
// somewhere inside the .dat.
let mut idx_reader = File::open(&idx_path).unwrap();
let healthy_nm =
CompactNeedleMap::load_from_idx(&mut idx_reader, version).unwrap();
let healthy_end = healthy_nm.max_needle_end();
assert!(
healthy_end > 0 && healthy_end <= dat_size,
"healthy volume should have max_needle_end ({}) in [1, dat_size={}]",
healthy_end,
dat_size
);
// Inject a dangling entry by appending a bogus 16/17-byte record
// directly to the .idx, then reload. The load walk should observe
// max_needle_end past dat_size — which is exactly the signal
// volume.load uses to mark the volume read-only.
let bad_offset = Offset::from_actual_offset(dat_size + 4 * 1024 * 1024);
let mut idx_append = OpenOptions::new()
.write(true)
.append(true)
.open(&idx_path)
.unwrap();
idx::write_index_entry(&mut idx_append, NeedleId(9999), bad_offset, Size(1024))
.unwrap();
idx_append.sync_all().unwrap();
let mut idx_reread = File::open(&idx_path).unwrap();
let bad_nm = CompactNeedleMap::load_from_idx(&mut idx_reread, version).unwrap();
let bad_end = bad_nm.max_needle_end();
assert!(
bad_end > dat_size,
"after dangling-entry inject max_needle_end ({}) should exceed dat_size ({})",
bad_end,
dat_size
);
}
#[test]
fn test_compaction_revision_relookup() {
// Verifies that re_lookup_needle_data_offset returns the correct data offset
@@ -4081,7 +3833,7 @@ mod tests {
let vif = VifVolumeInfo {
files: vec![VifRemoteFile {
backend_type: "s3".to_string(),
backend_id: "vif_rw_test".to_string(),
backend_id: "default".to_string(),
key: "remote-key".to_string(),
offset: 0,
file_size: v.dat_file_size().unwrap(),
@@ -4097,27 +3849,6 @@ mod tests {
)
.unwrap();
// Register a dummy S3 backend so load_remote_dat_file can look it
// up. This test never reads remote data, so a dead endpoint is fine.
// Use a test-specific backend_id to avoid racing with other tests
// that share the global registry.
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
region: "us-east-1".to_string(),
bucket: "bucket-a".to_string(),
endpoint: "http://127.0.0.1:1".to_string(),
storage_class: "STANDARD".to_string(),
force_path_style: true,
};
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.register(
"s3.vif_rw_test".to_string(),
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
);
v.dat_file_size().unwrap()
};
@@ -4161,11 +3892,6 @@ mod tests {
.unwrap();
assert!(deleted_size.0 > 0);
assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.remove("s3.vif_rw_test");
}
#[test]
@@ -4328,9 +4054,10 @@ mod tests {
std::fs::remove_file(&dat_path).unwrap();
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
// Use a test-specific backend_id to avoid racing with other tests
// that share the global registry. Never call clear() — only
// register/remove our own entries.
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.clear();
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
access_key: "access".to_string(),
secret_key: "secret".to_string(),
@@ -4345,7 +4072,11 @@ mod tests {
.write()
.unwrap();
registry.register(
"s3.remote_only_rw".to_string(),
"s3.default".to_string(),
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
);
registry.register(
"s3".to_string(),
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
);
}
@@ -4353,7 +4084,7 @@ mod tests {
let vif = VifVolumeInfo {
files: vec![VifRemoteFile {
backend_type: "s3".to_string(),
backend_id: "remote_only_rw".to_string(),
backend_id: "default".to_string(),
key: "remote-key".to_string(),
offset: 0,
file_size: dat_bytes.len() as u64,
@@ -4410,10 +4141,6 @@ mod tests {
assert_eq!(meta.data_size, 11);
let _ = shutdown_tx.send(());
crate::remote_storage::s3_tier::global_s3_tier_registry()
.write()
.unwrap()
.remove("s3.remote_only_rw");
}
/// Volume destroy removes .vif alongside the primary data files.
@@ -4444,7 +4171,7 @@ mod tests {
assert!(std::path::Path::new(&idx_path).exists());
// Destroy the volume
v.destroy(false, false).unwrap();
v.destroy(false).unwrap();
// .dat and .idx should be gone
assert!(
@@ -4501,7 +4228,7 @@ mod tests {
assert!(std::path::Path::new(&dat_path).exists());
assert!(std::path::Path::new(&idx_path).exists());
v.destroy(false, false).unwrap();
v.destroy(false).unwrap();
assert!(
!std::path::Path::new(&dat_path).exists(),
-2
View File
@@ -101,8 +101,6 @@ fn test_state_with_guard(
read_mode: seaweed_volume::config::ReadMode::Local,
master_url: String::new(),
master_urls: Vec::new(),
seed_master_set: std::collections::HashSet::new(),
current_master_url: tokio::sync::RwLock::new(String::new()),
self_url: String::new(),
http_client: reqwest::Client::new(),
outgoing_http_scheme: "http".to_string(),
+4 -4
View File
@@ -33,11 +33,11 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/image v0.38.0 // indirect
golang.org/x/net v0.52.0 // indirect
golang.org/x/image v0.36.0 // indirect
golang.org/x/net v0.50.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/text v0.35.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
golang.org/x/text v0.34.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
google.golang.org/grpc v1.79.3 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)
+20 -20
View File
@@ -27,8 +27,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -84,35 +84,35 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+2 -2
View File
@@ -999,9 +999,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "rand"
version = "0.9.4"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [
"rand_chacha",
"rand_core",
@@ -1,187 +0,0 @@
# V3 Phase 14 S8 Assignment
Date: 2026-04-19
Status: draft
Owner: sw, with architect and tester review
Purpose: execute the final bounded P14 close by producing evidence, scenario classification, and a clean P15 handoff
## 1. One-Line Goal
Close P14 as a bounded internal single-active-master topology/control-plane loop, backed by L0/L1/L2 evidence and honest L3/P15 carry-forward.
## 2. Scope
Included:
1. S4-S7 route evidence consolidation
2. acceptance matrix for all P14 internal claims
3. final supported topology statement
4. unsupported/deferred list
5. V2 testrunner/scenario port classification
6. process smoke gaps review
7. 14A final targeted review packet
8. P15 handoff mapping
Excluded:
1. CSI implementation
2. external API implementation
3. user data-path production readiness
4. migration implementation
5. security implementation
6. multi-master / HA
7. performance or soak claims
## 3. Required Work
### A. Evidence Inventory
Create an evidence table that maps S4-S7 claims to concrete tests and commands.
Must include:
1. observation institution
2. durable authority institution
3. convergence institution
4. real-route restart closure
5. placement/rebalance/failover policy for accepted topology
6. unsupported topology evidence
7. old-slot/stale observation rejection
8. per-volume isolation
### B. Gap Classification
Classify every missing proof as one of:
1. P14 internal blocker
2. P14 S8 test/harness gap
3. P15 product-surface gap
4. 14A verification follow-up
5. explicit non-goal
Any P14 internal blocker must be fixed before S8 close. A P15 product-surface gap must be carried forward, not disguised as P14 closure.
### C. V2 Scenario Port Classification
Inspect V2 scenario/test assets and produce a table:
| V2 asset | P14 use | P15 use | Port shape | Blocker |
|---|---|---|---|---|
Start with:
1. `weed/storage/blockvol/testrunner/`
2. `weed/storage/blockvol/testrunner/scenarios/public/ha-restart-recovery.yaml`
3. `weed/storage/blockvol/testrunner/scenarios/public/ha-failover.yaml`
4. `weed/storage/blockvol/testrunner/scenarios/public/fault-partition.yaml`
5. `weed/storage/blockvol/test/component/`
6. `weed/server/qa_block_*`
7. `learn/test` evidence layout
### D. L2 Process Smoke Decision
Answer whether current `sparrow` hidden smoke surfaces are sufficient for S8 L2 internal close.
If not sufficient, add the minimum internal S8 smoke surface needed to prove the S4-S7 route without inventing P15 external product APIs.
The smoke must check:
1. real process starts
2. durable authority reloads
3. observation or fixture input drives controller
4. assignment reaches adapter through `VolumeBridge`
5. restart does not mint backward
6. structured output can be consumed by tester
### E. 14A Final Targeted Pass
Prepare a 14A checklist for:
1. stale authority after restart
2. stale observation after reassignment
3. old-slot delivery after authority move
4. unsupported topology no-op/evidence
5. convergence stuck/supersede clearing
6. publication honesty during transition
7. per-volume isolation
14A should review findings first. It should not expand P14 scope.
### F. P15 Handoff
Produce a P15 handoff table:
| Gap | Why not P14 | P15 owner track | Required first proof |
|---|---|---|---|
Must include:
1. CSI lifecycle
2. frontend data path
3. external volume API
4. security/auth
5. diagnostics/operator workflow
6. migration/coexistence
7. deployment/hardening
8. final cluster validation agent
## 4. Required Commands
At minimum run or document blockers for:
```text
go test ./core/engine ./core/adapter ./core/authority -count=1
go test ./cmd/sparrow ./core/authority -count=1
go test ./... -count=1
```
If sandbox or Windows filesystem behavior blocks durable-store tests, rerun outside the sandbox and record the reason.
If L3 hardware scenarios are not runnable, do not fake them. Record the scenario names and blockers.
## 5. Expected Files
Expected docs:
1. `sw-block/design/v3-phase-14-s8-final-bounded-close.md` — this scope document
2. `sw-block/design/v3-phase-14-s8-assignment.md` — this assignment
3. `sw-block/design/v3-phase-14-s8-closure.md` — final closure statement, created after implementation/evidence
4. updates to `v3-phase-14-checklist.md`
5. updates to `v3-phase-14-log.md`
6. updates to `v3-phase-14a-checklist.md` if 14A final pass finds coverage gaps
Expected code/tests only if needed:
1. internal process smoke helper in `cmd/sparrow`
2. component evidence consolidation tests in `core/authority` or a shared component package
3. scenario classification artifacts under the chosen test/runner directory
## 6. Acceptance
S8 can be accepted only when:
1. the evidence matrix exists and is honest
2. all P14 internal blockers are either fixed or explicitly rejected from the P14 claim
3. L0 and L1 tests pass
4. L2 process smoke is present or its absence is classified as a P15 product-surface blocker
5. L3 scenarios are classified with runnable/deferred/blocker status
6. 14A final targeted pass has no open P14 safety blocker
7. P15 handoff maps every product gap to a P15 track
8. final closure statement does not claim production readiness
## 7. Review Focus
Architect should review:
1. whether S8 closure is product-honest
2. whether any P15 work was pulled backward into P14
3. whether any P14 internal truth gap was pushed forward into P15
4. whether test evidence matches the claimed layer
5. whether V2 porting stays mechanism-only
Tester should review:
1. whether evidence is reproducible
2. whether commands and scenario blockers are specific
3. whether L2/L3 gaps are visible
4. whether final artifacts are enough to start P15 without re-litigating P14
-114
View File
@@ -1,114 +0,0 @@
# V3 Phase 14 S8 Closure
Date: 2026-04-20
Status: draft — pending 14A final pass + architect + tester sign-off
Purpose: the final P14 closure statement per `v3-phase-14-s8-final-bounded-close.md` §10
## 1. What P14 Now Owns
After S4-S7, P14 closes **one bounded single-active-master internal topology/control-plane truth loop** for the accepted topology:
1. **Observation institution (S4)**: heartbeats and inventory flow through `ObservationHost` into a synthesized `ClusterSnapshot` + `SupportabilityReport`. Observation never mints authority. Partial / conflicting / duplicate / unknown / expired inventory becomes explicit `VolumeUnsupportedEvidence`; pending state is distinct from unsupported; per-volume isolation holds.
2. **Durable authority institution (S5)**: current authority line is durable, single-owner, atomically written, and reloaded synchronously at publisher boot. One record per volume. Corrupt records per-volume-fail-closed and surface as structured `ReloadSkips`. Process lock is exclusive and idempotent. Store never mints authority (boundary-guard test).
3. **Convergence institution (S6)**: desired assignments have bounded fate — confirmed (cleared), publish-not-observed (stuck-evidence, no re-mint, no churn), or superseded (by newer publisher line or different decision). Passive retry only; no active re-drive. Normal observation lag does NOT supersede. Per-volume retry clock; no global state.
4. **Real-route restart closure (S7)**: the full `ObservationHost → TopologyController → Publisher(reloaded) → VolumeBridge → VolumeReplicaAdapter` composes correctly at restart. Restart recomputes desired from durable authority + fresh observation (no durable desired store). Old-slot per-replica state is NOT revived from the store (S5 one-record-per-volume rule); live-route old-slot delivery via bridge is rejected by the adapter's monotonic guard. Stale observation cannot move authority backward (4 simulation shapes). Unsupported topology after restart records evidence, not silent idle. L2 process smoke proves real-binary restart × 2 preserves durable truth with no backward mint, via pinned structured JSON.
5. **Accepted topology** (S8 §4): single active master; multiple volumes; three replica slots per volume on distinct servers; one primary + two candidates; publisher-owned `Epoch` / `EndpointVersion`; durable current line; passive convergence; real `VolumeBridge` into adapter.
## 2. What P14 Explicitly Does NOT Own
1. CSI lifecycle (Create/Delete/Publish/Node*).
2. iSCSI / NVMe-oF frontend data path.
3. External volume management API (REST / gRPC).
4. Security / auth (authn, CHAP, encryption).
5. Operator diagnostics surface (dashboards, alerts, `/metrics`).
6. Operator workflow (drain, planned failover, manual reassign, supervised rebuild).
7. V2 ↔ V3 migration and coexistence.
8. Deployment / packaging / hardening (systemd, container, backup of durable store).
9. Performance / soak / release-hardening claims.
10. Multi-master / leader election / distributed authority (explicit non-goal, not just deferred).
11. V2 `HandleAssignment` / `promote` / `demote` semantics (explicit non-goal).
12. Heartbeat-as-authority (explicit non-goal).
Every item 1–9 is mapped to a P15 track in `v3-phase-14-s8-p15-handoff.md` §2. Items 10–12 are permanent non-goals.
## 3. Evidence At L0 / L1 / L2 / L3
Full matrix: `v3-phase-14-s8-evidence-matrix.md` §3. Summary of coverage:
| Claim | L0 | L1 | L2 | L3 |
|---|---|---|---|---|
| 1. Observation system-fed | ✓ | ✓ | — (via L1) | L3 shape only |
| 2. Supportability explicit | ✓ | ✓ | — | L3 shape only |
| 3. Durable authority / reload | ✓ (13 tests) | ✓ | ✓ (subprocess) | L3 shape only |
| 4. No old-slot durable revival | ✓ | ✓ | ✓ | L3 shape only |
| 5. Live-route old-slot rejected | ✓ | ✓ | N/A (live-route) | N/A |
| 6. Controller-driven bind/reassign/refresh | ✓ (7 tests) | ✓ | ✓ (Bind only — Reassign / RefreshEndpoint are L1-only; subprocess does not ingest heartbeats) | L3 shape only |
| 7. Confirmation clears desired | ✓ | ✓ | N/A | L3 shape only |
| 8. Stuck evidence bounded / no churn | ✓ (5 tests) | ✓ | N/A | L3 shape only |
| 9. Supersede (2 modes) | ✓ (3 tests) | ✓ | N/A | L3 shape only |
| 10. Restart re-anchors via VolumeBridge | ✓ | ✓ | **subclaim only** — L2 proves durable `Publisher` reload (Claim 3); the bridge/adapter re-anchor is NOT at L2 because `sparrow` subprocess does not construct `ObservationHost` / controller / bridge / adapter | L3 shape only |
| 11. Stale observation cannot go backward | ✓ | ✓ (4 sub-cases) | — | L3 shape only |
| 12. Unsupported → evidence | ✓ | ✓ | **not at L2** — `ReloadSkips` is S5 durable-corruption evidence, not observation-layer unsupported-topology. Controller `LastUnsupported` is L1 | L3 shape only |
| 13. Per-volume isolation | ✓ (4 tests) | ✓ | N/A | L3 shape only |
| 14. Placement / rebalance | ✓ (5 tests) | ✓ | N/A | L3 shape only |
| 15. Fence route bounded | ✓ (adapter) | PARTIAL | N/A | L3 shape only |
| 16. Boundary guards | ✓ | N/A (structural) | N/A | N/A |
Sixteen claims; fifteen PROVEN at their target level; one (Claim 15) PARTIAL — adapter-package follow-up per §5 below. L3 is classification-only at P14 (`v3-phase-14-s8-v2-scenario-classification.md`) — L3 entries are scenario SHAPES, not executed evidence. Runnable L3 is P15 Final Gate (Cluster Validation Agent). Claims 6, 10, and 12 have NARROWED L2 cells where the subprocess binary cannot carry the full claim surface; each is handed off to the correct canonical P15 track.
## 4. Test Baseline (2026-04-20)
```
$ go test ./core/engine ./core/adapter ./core/authority ./cmd/sparrow -count=1
ok github.com/seaweedfs/seaweed-block/core/engine 0.016s
ok github.com/seaweedfs/seaweed-block/core/adapter 2.344s
ok github.com/seaweedfs/seaweed-block/core/authority 4.569s
ok github.com/seaweedfs/seaweed-block/cmd/sparrow 2.264s
$ go test ./... -count=1
(14 packages, all PASS)
```
193 Go tests across the P14 scope. L2 subprocess smoke (`TestS7Process_RealSubprocessRestartSmoke`) spawns a real `sparrow` binary twice on the same store directory and asserts the pinned JSON pass-line schema.
## 5. Residual Risks
| # | Risk | Scope / carry-forward |
|---|---|---|
| 1 | Windows `os.RemoveAll` on `t.TempDir()` can race with lock-file release during teardown. No correctness impact. | S7 sketch §8.5 — logged as `t.Logf`, not a failure. |
| 2 | Claim 15 PARTIAL: fence quantitative timeout-bound is adapter-package-owned, depends on the uncommitted fence-watchdog branch committing. | `v3-phase-14-s8-p15-handoff.md` item 11 — P14 internal follow-up, NOT P15. Single adapter test replaces PARTIAL status. |
| 3 | L2 subprocess drives mint via `StaticDirective`; it does NOT construct `ObservationHost` / controller / bridge / adapter. Full heartbeat-ingress and real-binary adapter route are L1-only in S8. | **P15 T1** Frontend + Data Path (handoff items #1 and #2). |
**No S8-blocking correctness residuals.** One adapter-owned quantitative fence proof (Claim 15) remains as P14-internal follow-up per row 11 of the handoff table — not a P15 track, not an S8 gate. Zero backward-authority-mint residuals. Zero silent-idle residuals.
## 6. P15 Track Owners
Handoff complete per `v3-phase-14-s8-p15-handoff.md`, aligned to canonical `v3-phase-15-product-plan.md` §4. Every P14 residual product gap has a named P15 track and a concrete first-proof gate:
- T1 Frontend + Data Path Contract (includes subprocess heartbeat ingress)
- T2 CSI / External Lifecycle Surface
- T3 External Control API
- T4 Security And Auth Posture
- T5 Diagnostics And Explainability
- T6 Operator Workflow
- T7 V2/V3 Coexistence And Migration
- T8 Deployment / Upgrade / Release Hardening
- Final Gate — Cluster Validation Agent
## 7. Final P14 Claim
> **P14 has closed one bounded single-active-master internal topology/control-plane truth loop for the accepted topology set.** Observation is system-fed; authority is durable, single-owner, and restart-recovered; convergence has bounded fate; the real `VolumeBridge → adapter` route re-anchors across restart without backward mint or old-slot revival.
>
> **P14 has NOT closed CSI, external API, frontend data path, migration, security, deployment, performance, or production readiness.** Those nine tracks are handed off to P15 with concrete first-proof gates. Multi-master, V2 `HandleAssignment/promote/demote` semantics, and heartbeat-as-authority are explicit non-goals — not deferrals.
## 8. Sign-off State
- **Architect** — pending review of this matrix + handoff.
- **Tester** — pending reproducibility check (commands in §4).
- **14A final targeted pass** — pending (scope in `v3-phase-14a-checklist.md`).
S8 cannot be accepted until all three sign off. After that, P14 is closed and the next active work is P15 T1 Frontend.
@@ -1,241 +0,0 @@
# V3 Phase 14 S8 Evidence Matrix
Date: 2026-04-20
Status: draft (S8 evidence consolidation)
Purpose: map every internal P14 claim from `v3-phase-14-s8-final-bounded-close.md` §5 to concrete tests and commands, classify L0/L1/L2/L3 coverage, and surface residual gaps for 14A / P15 handoff
## 1. How To Read This Matrix
Each row is one internal P14 claim. Columns:
- **L0 Unit** — package-local invariant / policy test that proves the claim inside one package.
- **L1 Component** — in-process multi-package route test (real `ObservationHost` + `TopologyController` + `Publisher` + `VolumeBridge` + `VolumeReplicaAdapter`, no shell, no kernel).
- **L2 Process** — real `sparrow` binary or in-process `Bootstrap()` against a real filesystem store.
- **L3 Scenario** — hardware / YAML-driven scenario. For P14 S8, L3 is **classification-only**: the rows below say "L3 shape only, not executed" where a V2 YAML scenario describes the same claim shape. L3 entries are NOT executed evidence; they are the scenario shapes that P15 Cluster Validation (Final Gate) will run. A claim with an L3 shape listed is NOT proven at L3 by S8.
- **Status** — `PROVEN` (explicit test); `PARTIAL` (route covered but some sub-claim deferred); `DEFERRED` (intentionally pushed to later slice / P15); `N/A` (not applicable at that level).
- **Residual** — what is NOT proven and where it is carried.
All cited L0/L1/L2 tests live in the committed tree; re-run with:
```
go test ./core/engine ./core/adapter ./core/authority ./cmd/sparrow -count=1
```
Baseline on 2026-04-20: engine 0.02s / adapter 2.3s / authority 4.6s / sparrow 2.3s — all green, 193 Go tests total across the S4-S7 scope.
## 2. Accepted Topology Claim Recap
Pinned by the S8 close doc §4. Reproduced here so every row in the matrix below is read against the correct bounded shape:
1. single active master
2. multiple volumes
3. three bounded replica slots per volume, distinct servers
4. one current authoritative primary, two failover/rebalance candidates
5. publisher-owned `Epoch` / `EndpointVersion`
6. durable current authority line, one record per volume
7. passive convergence (publish → observe → confirm / stuck / supersede)
8. real `VolumeBridge` delivery into adapter/engine
Anything outside this set is P15 or explicit non-goal.
## 3. Evidence Matrix
### Claim 1 — Observation is system-fed, not test-authored
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestObservation_BuilderStampsAuthorityFromReader` / `TestObservation_SupportabilityDoesNotReadClusterSnapshotOutput` / `TestObservation_NeverMintsAuthority_BoundaryGuard` | `TestObservation_EndToEnd_SystemFedSnapshotReachesController` / `TestConvergenceRoute_ObservationFedFailover_ConfirmsAndClearsDesired` / `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` | (covered at L1; `sparrow` run-path uses the same `ObservationHost` construction) | L3 shape only, not executed — `ha-restart-recovery.yaml` / `ha-failover.yaml` carry the L3 shape | PROVEN | none at L0/L1 |
### Claim 2 — Topology supportability is explicit and fail-closed (partial inventory / missing-server / conflict / duplicate / unknown-volume / expired)
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestObservation_PartialInventory_PendingThenUnsupported` / `TestObservation_MissingServerObservation_NeverOrdinaryIneligible` / `TestObservation_ConflictingPrimaryClaim_Unsupported` / `..._NoWinnerEverChosen` / `TestObservation_DuplicateServerTopology_UnsupportedAndEvidenceRecorded` / `TestObservation_UnknownObservedVolume_ProducesUnsupportedEvidence` / `TestObservation_ExpiredFact_SemanticallyIneligible` / `TestObservation_ExpiredDoesNotReachController` / `TestObservation_FreshnessWindowHonored_NotTickMultiples` | `TestObservation_SupportabilityCollapsePropagatesToController` / `TestObservation_SupportabilityCollapseClearsControllerDesired` / `TestS7_Restart_UnsupportedTopologyRecordsEvidence` | N/A (covered by L0/L1) | L3 shape only, not executed — `fault-partition.yaml` is the L3 shape | PROVEN | none |
### Claim 3 — Durable authority is single-owner, atomic, and restart-recovered
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestFileAuthorityStore_RoundTrip` / `TestFileAuthorityStore_LatestWinsPerVolumeID` / `TestFileAuthorityStore_EnvelopeShape` / `TestDurableAuthority_AtomicWrite_NoTornRecord` / `TestDurableAuthority_CorruptRecord_PerVolumeFailClosed` / `TestDurableAuthority_ProcessLock_ExclusiveOwner` / `TestDurableAuthority_ProcessLock_IdempotentRelease` / `TestDurableAuthority_ProcessLock_SurvivesStaleFileOnDisk` / `TestDurableAuthority_UnreadableIndex_BootFailsClosed` / `TestDurableAuthority_StoreNeverMintsAuthority_BoundaryGuard` / `TestDurableAuthority_StoreWriteCallerAllowlist_BoundaryGuard` / `TestEncodeVolumeIDForFilename_Injective` / `TestEncodeVolumeIDForFilename_NoFilesystemUnsafeOutput` | `TestDurableAuthority_ReloadReproducesCurrentLine` / `TestDurableAuthority_ControllerSeesReloadedLineAtBoot` / `TestDurableAuthority_StoreBackedPublisherRunsNormalRoute` / `TestDurableAuthority_PutFailureRollsBackInMemoryState` / `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` | `TestBootstrap_FreshDir_AcquiresLockAndReloads` / `TestBootstrap_WithExistingRecord_ReloadsPublisherState` / `TestBootstrap_CorruptRecord_LoggedAtStartup` / `TestBootstrap_LockContention_RefusesSecondBootstrap` / `TestS7Process_BootstrapReloadRouteSmoke` / **subprocess**: `TestS7Process_RealSubprocessRestartSmoke` | L3 shape only, not executed — `ha-restart-recovery.yaml` / `diag-restart-recovery.yaml` | PROVEN | Windows `os.RemoveAll` may race with lock release at teardown — logged as residual per S7 sketch §8.5; does NOT affect correctness |
### Claim 4 — No old-slot durable revival after restart
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestDurableAuthority_StalePreFailoverRecordRejectedOnReload` | `TestS7_Restart_OldSlotNotRevivedFromDurable` (explicitly asserts `LastAuthorityBasis("vr","r1")==false` and `("vr","r2")==Epoch=2` post-restart) | covered by S7 subprocess smoke plus in-process two-run `TestS7Process_RestartDoesNotRegressAuthorityLine` | L3 shape only, not executed | PROVEN | none — row 4a/4b split in S7 closes the original ambiguity |
### Claim 5 — Live-route monotonic guard rejects stale old-slot delivery (pre-restart)
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| adapter-internal: engine monotonic guards covered in `core/adapter/adapter_test.go` and `core/engine/*_test.go` | `TestS7_LiveRoute_OldSlotDeliveryRejectedByAdapter` (real `VolumeBridge`, in-memory publisher holds both per-replica keys, lower-epoch catch-up rejected) | N/A (live-route, pre-restart) | N/A | PROVEN | none |
### Claim 6 — Controller-driven bind / reassign / refresh over accepted topology (system-driven, not test-driven)
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestTopologyController_InitialPlacementBalancesAcrossVolumes` / `..._InitialPlacementUsesEvidenceTieBreakOnEqualLoad` / `..._FailoverUsesHighestEvidenceCandidate` / `..._RebalanceMovesToLighterServer` / `..._RebalanceSkipsWhenLoadAlreadyWithinBound` / `..._ObservedAuthorityClearsPending` / `..._StalePreConfirmSnapshotDoesNotDuplicateQueuedMove` | Placement / failover: `TestTopologyControllerToPublisher_E2E_MultiVolumePlacementAndFailover` / `TestObservationHost_IdenticalSupportedSnapshots_NoDuplicateAsks` / `TestObservationHost_VolumeRecovers_ClearsStaleDesired`. Rebalance: `TestTopologyControllerToPublisher_E2E_Rebalance`. **Refresh: `TestConvergenceRoute_RefreshEndpoint_ConfirmsAndClearsDesired`** (host heartbeat changes slot DataAddr/CtrlAddr while publisher holds old addrs → controller emits `IntentRefreshEndpoint` → publisher bumps `EndpointVersion` on same `Epoch` → bridge delivers refreshed `AssignmentInfo` → confirmation clears desired). | `TestS7Process_RealSubprocessRestartSmoke` covers the Bind mint via the real binary (Reassign/Refresh are L1-only in `sparrow` today; real binary does not ingest heartbeats — see §5 L2 scope note). | L3 shape only, not executed | PROVEN | none |
### Claim 7 — Convergence: confirmation clears desired
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestConvergence_Confirmation_ClearsDesired` / `TestConvergence_Confirmation_RequiresBothSources` / `TestConvergence_StaleObservation_NoConfirmation` / `TestConvergence_StuckThenConfirm_ClearsStuckEvidence` | `TestConvergenceRoute_ObservationFedFailover_ConfirmsAndClearsDesired` | N/A | L3 shape only, not executed | PROVEN | none |
### Claim 8 — Publish-not-observed stuck evidence is bounded and non-churning
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestConvergence_PublishNotObserved_StuckAfterWindow` / `TestConvergence_Stuck_DoesNotReMintReassign` / `TestConvergence_Reassign_CalledAtMostOncePerDesired` / `TestConvergence_RetryStateIsPerVolume` / `TestConvergence_SupportedRecovery_ClearsPriorUnsupportedEvidence` | `TestConvergenceRoute_PublishNotObserved_StuckWithoutChurn` | N/A | L3 shape only, not executed | PROVEN | none |
### Claim 9 — Supersede by newer authority or different decision
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestConvergence_Supersede_PublisherAdvancedToOtherReplica` / `TestConvergence_NormalLag_OldObservationDoesNotSupersede` / `TestConvergence_Supersede_DecisionTableProducesDifferentAsk` | `TestConvergenceRoute_Supersede_PublisherMovedElsewhere_DropsStaleDesired` | N/A | L3 shape only, not executed | PROVEN | Case 2 of the supersede rule was intentionally dropped (see S6 sketch §9) — that decision is tested indirectly by `TestConvergence_NormalLag_OldObservationDoesNotSupersede` which would otherwise fail |
### Claim 10 — Restart re-anchors authority via real `VolumeBridge` into the adapter/engine route
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestDurableAuthority_PublisherAdvancesFromReloaded` | `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` / `TestS7_Restart_RecomputesDesiredNotTransientState` (hard precondition prevents vacuous pass) | **Not at L2.** The `sparrow` subprocess smoke (`TestS7Process_RealSubprocessRestartSmoke`) reloads durable `Publisher` state only; it does NOT construct `ObservationHost`, `TopologyController`, `VolumeBridge`, or `VolumeReplicaAdapter`, so the bridge/adapter re-anchor cannot be counted at L2. L2 proves a strictly narrower subclaim (durable `Publisher` reload — Claim 3). | L3 shape only, not executed — `ha-restart-recovery.yaml` is the L3 shape | PROVEN at L1 (full route); L2 COVERS a subclaim only (durable reload, see Claim 3) | Subprocess heartbeat ingress + real-binary adapter route → **P15 T1** (handoff item #2) |
### Claim 11 — Stale observation cannot move authority backward
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestConvergence_StaleObservation_NoConfirmation` | `TestS7_Restart_StaleObservationCannotMoveBackward` (4 sub-cases: stale-epoch / stale-replica / stale-endpointVersion / unassigned via `snapshottingReader` host-reader freeze) / `TestConvergenceRoute_PublishNotObserved_StuckWithoutChurn` (proves host-reader lag scenario) | covered at L1 | L3 shape only, not executed — `fault-partition.yaml` | PROVEN | stale-basis simulation is via host-reader freeze; heartbeat wire does not carry authority (documented in S7 sketch §9.1) |
### Claim 12 — Unsupported topology after restart records evidence, not silent idle
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestConvergence_UnsupportedClearsDesiredWithEvidence` / `TestConvergence_PendingClearsDesired` | `TestS7_Restart_UnsupportedTopologyRecordsEvidence` | **Not at L2.** This claim is about the controller's `LastUnsupported(vid)` surface being populated from conflicting / incomplete observation; that requires `ObservationHost` + controller composition, which the subprocess does not construct. `Bootstrap.ReloadSkips` is a DIFFERENT surface (S5 durable-record corruption, see Claim 3 residual path) and is NOT observation-layer unsupported-topology evidence. | L3 shape only, not executed — `fault-partition.yaml` + `diag-restart-recovery.yaml` are the L3 shapes | PROVEN at L1 | Operator-visible surface over `LastUnsupported` / `LastConvergenceStuck` → **P15 T5** Diagnostics (handoff item #6) |
### Claim 13 — Per-volume isolation (one volume's failure does not block others)
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestObservation_BadVolumeDoesNotBlockHealthyVolume_Isolation` / `TestObservation_VolumeUnsupportedEvidence_IsolatedPerVolume` / `TestConvergence_OneStuckVolumeDoesNotBlockOthers` / `TestTopologyController_UnsupportedVolumeDoesNotBlockOtherVolumes` | `TestS7_Restart_PerVolumeIsolation` | N/A | L3 shape only, not executed | PROVEN | none |
### Claim 14 — Placement / rebalance / failover within the accepted topology
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestTopologyController_InitialPlacementBalancesAcrossVolumes` / `..._FailoverUsesHighestEvidenceCandidate` / `..._RebalanceMovesToLighterServer` / `..._RebalanceSkipsWhenLoadAlreadyWithinBound` / `..._OutOfTopologyAuthority_NoAskPlusEvidence` | `TestTopologyControllerToPublisher_E2E_MultiVolumePlacementAndFailover` / `TestTopologyControllerToPublisher_E2E_Rebalance` | N/A | L3 shape only, not executed — `ha-failover.yaml` carries the L3 shape | PROVEN | none |
### Claim 15 — Fence path routes through bridge + adapter without spurious Healthy
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| adapter fence-slot / monotonic-guard tests in `core/adapter/adapter_test.go` | `TestS7_Restart_FenceRouteBoundedWithoutCallback` (Fence command reaches executor via real bridge; withheld callback does not produce Healthy) | N/A | L3 shape only, not executed | PARTIAL | Full timeout-expiry quantitative bound (watchdog fires at exact deadline + structured event) is adapter-owned proof (sketch §10.1 Path B — S7 may not reconfigure the fence path). Carried to the adapter package's own tests; not an L1 S7 proof |
### Claim 16 — Boundary guards (no unauthorized `AssignmentInfo` minting, no observation-side asks, no store minting, no hidden adapter ingress)
| L0 | L1 | L2 | L3 | Status | Residual |
|---|---|---|---|---|---|
| `TestObservation_NeverMintsAuthority_BoundaryGuard` / `TestDurableAuthority_StoreNeverMintsAuthority_BoundaryGuard` / `TestDurableAuthority_StoreWriteCallerAllowlist_BoundaryGuard` / `TestNonForgeability_*` (AST-level guards in `authority_test.go`) | N/A (structural, not route) | N/A | N/A | PROVEN | none |
## 4. Gap Classification
Per S8 assignment §3.B, every missing proof must be classified. Scanning the matrix above:
### 4.1 P14 internal blockers
**None.** Every P14 internal claim has at least one PROVEN row.
### 4.2 P14 S8 test/harness gap
**Claim 15 (fence quantitative bound).** S7 deliberately capped at L1 route coverage. The full `fence timeout fires at exact deadline` proof lives in the adapter package and depends on the uncommitted fence-watchdog branch landing committed. After that commits, a single-file adapter test replaces the current PARTIAL status. Not an S8 gate.
### 4.3 P15 product-surface gaps (carried forward)
See `v3-phase-14-s8-p15-handoff.md` for the full mapping (aligned to canonical `v3-phase-15-product-plan.md` §4 track numbering). Cross-reference summary:
- Frontend data path + subprocess heartbeat ingress → **P15 T1** Frontend + Data Path
- CSI lifecycle → **P15 T2** CSI / External Lifecycle Surface
- External control API → **P15 T3** External Control API
- Security / auth → **P15 T4** Security And Auth Posture
- Operator diagnostics for `LastUnsupported` / `LastConvergenceStuck` / `ReloadSkips` → **P15 T5** Diagnostics
- Operator workflow (drain / supervised reassign / rebuild) → **P15 T6** Operator Workflow
- V2/V3 migration → **P15 T7** Migration
- Deployment / hardening → **P15 T8** Deployment / Release Hardening
- End-to-end cluster validation → **P15 Final Gate**
### 4.4 14A verification follow-ups
See `v3-phase-14a-checklist.md` for the sidecar list. The S8 matrix above is the input to 14A's final targeted review.
### 4.5 Explicit non-goals (do NOT carry)
- multi-master / leader election
- performance / soak claims at P14 close
- long-running stability under production load
## 5. L2 Process Smoke Decision
Per S8 assignment §3.D, S8 must answer: are current `sparrow` smoke surfaces sufficient?
**Answer: scoped YES — L2 proves durable-restart only; the full S4-S7 route remains L1-proven.** (Round-2 architect correction.)
### 5.1 What L2 proves today
| Required check | Covered by |
|---|---|
| real process starts | `TestS7Process_RealSubprocessRestartSmoke` spawns the real `sparrow` binary. |
| durable authority reloads | same test + component `TestS7Process_BootstrapReloadRouteSmoke`. |
| restart does not mint backward | subprocess asserts `backwardMint: false` in the pinned JSON pass-line; component `TestS7Process_RestartDoesNotRegressAuthorityLine` asserts reloaded `Epoch` equals pre-restart `Epoch`. |
| structured output can be consumed by tester | subprocess pass-line schema pinned in S7 sketch §8.4. |
### 5.2 What L2 deliberately does NOT prove
| Not covered at L2 | Where it IS proven | Carry-forward |
|---|---|---|
| "heartbeat wire → `ObservationHost` → `TopologyController`" in a real process. Subprocess drives mints via `StaticDirective`; there is no heartbeat ingress in `sparrow` today. | L1: `TestObservation_EndToEnd_SystemFedSnapshotReachesController`, `TestConvergenceRoute_ObservationFedFailover_ConfirmsAndClearsDesired`, `TestConvergenceRoute_RefreshEndpoint_ConfirmsAndClearsDesired`, `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` (all run real `ObservationHost` → controller → publisher → bridge → adapter, in-process). | **P15 T1 Frontend + Data Path** (subprocess heartbeat ingress is part of the frontend/transport contract). Handoff item #2 in `v3-phase-14-s8-p15-handoff.md`. |
| "assignment reaches adapter via bridge IN THE REAL BINARY". The subprocess binary does not construct a VolumeReplicaAdapter — only the in-process component test does. | L1: `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` composes the full stack in-process and waits for adapter `ModeHealthy`. | **P15 T1 Frontend + Data Path** — the first real-binary adapter ↔ frontend route is a T1 deliverable. |
| Multi-process mixed-route scenarios (separate authority + observing processes). | N/A at P14 | **P15 Final Gate** (Cluster Validation). |
### 5.3 Decision
**No new S8 smoke is added.** The scope split above is honest:
- L2 subprocess smoke correctly proves durable restart truth.
- The full S4-S7 composed route is proven at L1 — including the `RefreshEndpoint` path added in this pass.
- Subprocess heartbeat ingress and full real-binary adapter route are genuinely P15 T1 surfaces; inventing a hidden `--s8-route-smoke` heartbeat fixture would be a P15 surface in P14 clothing.
This is a downgrade from the round-1 matrix claim ("L2 covers the S4-S7 internal route"), which was an overclaim.
## 6. Baseline Command Result (2026-04-20)
```
$ go test ./core/engine ./core/adapter ./core/authority ./cmd/sparrow -count=1
ok github.com/seaweedfs/seaweed-block/core/engine 0.016s
ok github.com/seaweedfs/seaweed-block/core/adapter 2.344s
ok github.com/seaweedfs/seaweed-block/core/authority 4.569s
ok github.com/seaweedfs/seaweed-block/cmd/sparrow 2.264s
$ go test ./... -count=1
(14 packages, all PASS; runtime/schema have no tests)
```
Evidence is reproducible by any reviewer on Windows / Linux from the committed tree on `phase-14` at commit `a6421f9` or later.
## 7. Residual Risks
| # | Risk | Scope |
|---|------|-------|
| 1 | Windows `os.RemoveAll` on the lock file during `t.TempDir()` teardown races with lock release. Does NOT affect correctness — only teardown. | S7 sketch §8.5 residual; logged as `t.Logf` in `restart_route_test.go`. |
| 2 | Claim 15 partial: L1 covers "fence command dispatched via real bridge + withheld callback does not produce Healthy"; full timeout-expiry bound is adapter-package-owned. | Closes when adapter fence-watchdog branch commits. Not an S8 blocker. |
| 3 | L2 subprocess smoke drives the directive via `StaticDirective`, not real heartbeat ingress to `ObservationHost`. The full heartbeat-path is L1-proven; subprocess widening requires a P15 external surface. | P15 Frontend / Cluster Validation. |
**No S8-blocking correctness residuals.** One adapter-owned quantitative fence proof (Claim 15) remains as a P14-internal follow-up, landing when the fence-watchdog branch commits. No backward-authority-mint risk on restart. No unsupported-topology silent-idle path.
## 8. Closure Readiness
Against S8 acceptance gate (`v3-phase-14-s8-assignment.md` §6):
1. evidence matrix exists and is honest — **yes** (this document)
2. P14 internal blockers fixed or explicitly non-P14 — **yes** (§4.1 empty)
3. L0 + L1 tests pass — **yes** (baseline §6)
4. L2 process smoke present or absence classified — **yes** (§5; present)
5. L3 classified — **yes** (L3 shape only, not executed this slice; see `v3-phase-14-s8-v2-scenario-classification.md`)
6. 14A final pass has no open P14 safety blocker — **pending** (14A review on this matrix)
7. P15 handoff maps every product gap — **yes** (see `v3-phase-14-s8-p15-handoff.md`)
8. final closure statement does not claim production readiness — **yes** (see `v3-phase-14-s8-closure.md`, draft)
**S8 is evidence-complete pending 14A sign-off.** Awaiting architect + tester review.
@@ -1,212 +0,0 @@
# V3 Phase 14 S8 Final Bounded Close
Date: 2026-04-19
Status: draft
Purpose: close the bounded P14 internal topology/control-plane claim and hand off product surfaces to P15 without pretending P14 is production-ready block storage
## 1. Target
`P14 S8` closes one bounded internal control-plane product shape:
`heartbeat/observation -> ClusterSnapshot -> TopologyController -> Publisher durable authority -> VolumeBridge -> adapter/engine -> observed convergence/restart evidence`.
S8 is not another protocol slice and not a broad product surface slice. It is the final acceptance and evidence package for the internal single-active-master topology/control-plane loop built by S4-S7.
## 2. What S8 Must Prove
S8 must prove, at the appropriate test layer, that:
1. observation is system-fed, not test-authored
2. topology supportability is explicit and fail-closed
3. authority is durable, single-owner, and restart-recovered
4. controller decisions are system-driven for the accepted topology set
5. convergence has bounded fate: confirmed, stuck, or superseded
6. restart re-anchors current authority through the real bridge/adapter route
7. stale observation, stale old-slot delivery, and unsupported topology do not silently move the system backward
8. per-volume isolation holds under mixed supported/unsupported states
9. the accepted topology set and unsupported list are explicit
10. every product-facing gap is handed to P15, not hidden inside P14 closure
## 3. What S8 Does Not Prove
S8 does not prove:
1. CSI completion
2. iSCSI/NVMe product frontend completion
3. external volume lifecycle API completion
4. real user data-path production readiness
5. V2/V3 migration readiness
6. multi-master HA / leader election / distributed authority
7. final performance or soak readiness
Those are P15 tracks and the P15 final cluster validation gate.
S8 may prepare scenario shapes and port test runner muscle for those later gates, but it must not claim them as P14 closure.
## 4. Accepted Topology Claim
The S8 supported topology claim is bounded to:
1. single active master
2. multiple volumes
3. three bounded replica slots per volume
4. distinct server per slot
5. one current authoritative primary
6. two bounded failover/rebalance candidates
7. publisher-owned `Epoch` / `EndpointVersion`
8. durable current authority line, one record per volume
9. passive convergence: publish, observe, confirm / stuck / supersede
10. real `VolumeBridge` delivery into the adapter/engine path
Anything outside this set is unsupported or deferred.
## 5. Required Evidence Matrix
S8 must produce a closure table with these columns:
| Claim | L0 Unit | L1 Component | L2 Process | L3 Scenario | Status | Residual |
|---|---|---|---|---|---|---|
Minimum claims:
1. observation supportability
2. duplicate/missing/conflicting inventory handling
3. durable authority reload
4. no old-slot durable revival
5. controller-driven bind/reassign/refresh
6. convergence confirm/clear
7. publish-not-observed stuck evidence
8. supersede by newer authority
9. restart re-anchor via `VolumeBridge`
10. stale observation cannot move backward
11. unsupported topology records evidence
12. per-volume isolation
13. placement/rebalance within accepted topology
14. P15 handoff gaps identified
## 6. Required Test Levels
### L0 Unit / Invariant
Required command baseline:
```text
go test ./core/engine ./core/adapter ./core/authority -count=1
```
Must include boundary guards for:
1. no non-authority `AssignmentInfo` minting
2. no observation code constructing assignment asks
3. no store minting authority
4. no hidden adapter ingress
### L1 Component
Required routes:
1. `ObservationHost -> TopologyController -> Publisher -> VolumeBridge -> VolumeReplicaAdapter`
2. observation-fed failover confirms and clears desired
3. stuck evidence appears without ask churn
4. supersede drops stale desired
5. restart route reloads and re-anchors authority
6. old-slot live delivery is rejected by adapter monotonicity
7. unsupported topology records evidence
### L2 Local Process
Required shapes:
1. real `sparrow` durable authority restart smoke
2. process-level S4-S7 route smoke if the current binary surface can expose it cleanly
3. if process route smoke cannot be added without inventing P15 surfaces, S8 must record the blocker and carry it to P15 T1/T2/T3
L2 tests must check exit code, artifact/log output, lock release, reload count, and no backward mint.
### L3 Hardware / Scenario
S8 should port scenario shapes from V2 and classify each as:
1. runnable in P14 internal mode
2. blocked by missing P15 frontend/API/data-path surface
3. deferred to P15 final cluster validation
Initial scenario candidates:
1. restart recovery
2. failover
3. partition / delayed heartbeat
4. unsupported topology
5. IO continuity only if frontend/data path exists
S8 can close P14 with L3 blockers only if the blockers are genuinely P15 product-surface gaps, not internal truth gaps.
## 7. V2 Port Plan
### Classify now (S8 scope), port deferred to P15
S8 **does not port testrunner machinery into the V3 tree** (round-2 doc-consistency fix, aligned with `v3-phase-14-s8-v2-scenario-classification.md` §2). The items below are the V2 muscles S8 inventories and classifies; actual porting is a P15 Final Gate (Cluster Validation Agent) deliverable, not an S8 one.
1. `weed/storage/blockvol/testrunner/`
- **classify now**: runner architecture, scenario YAML shape, action vocabulary, artifact collection, report generation — recorded as PORT-MECHANISM in the classification table. **Port in P15 Final Gate.**
2. `weed/storage/blockvol/testrunner/scenarios/public` and selected internal scenarios
- **classify now**: map each scenario shape to RUNNABLE-P14 / BLOCKED-FRONTEND / BLOCKED-OPS / BLOCKED-HA / BLOCKED-PERF. **Port in P15 Final Gate.**
3. `weed/storage/blockvol/test/component/`
- **classify now**: component harness structure and failure-injection style. **Port in P15 T1 Frontend + Data Path** as that track's real-route harness.
4. `weed/server/qa_block_*`
- **classify now**: coverage taxonomy and route scenario ideas as reference for the evidence matrix (§5). **Port only as applicable per P15 track needs.**
5. `learn/test` evidence convention
- **classify now**: manifest / result / report / run-notes convention referenced as the target shape for future P15 evidence bundles. **Adopt when P15 ships real L3/L4 runs.**
### Do not port
1. `promotion.go`
2. `HandleAssignment` / `promote` / `demote`
3. local runtime self-promotion
4. heartbeat timing directly becoming authority
5. V2 authority ownership or HA assumptions
6. CSI/iSCSI/NVMe product integration as P14 close criteria
## 8. Deliverables
S8 must deliver:
1. final P14 supported-topology statement
2. final P14 unsupported/deferred list
3. acceptance evidence matrix
4. scenario-port classification table
5. tester sign-off packet template
6. 14A final targeted review checklist
7. P15 handoff statement mapping every remaining gap to a P15 track
8. code/tests only where needed to make the evidence route real
## 9. Reject Conditions
Reject S8 if:
1. closure is based only on local unit tests
2. a claimed route still uses manual assignment choreography
3. S8 invents CSI/API/operator surfaces to make tests look complete
4. unsupported topology becomes silent idle
5. stale observation can emit a newer decision from an old basis
6. durable restart can revive stale old-slot truth
7. convergence can churn epochs without observation confirmation
8. S8 claims production readiness without P15 frontend/data-path/security/cluster validation
## 10. Closure Statement Shape
The final S8 closure statement must say exactly:
1. what P14 now owns
2. what P14 explicitly does not own
3. what evidence exists at L0/L1/L2/L3
4. what risks remain
5. which P15 track owns each product gap
The expected final claim is:
`P14 has closed one bounded single-active-master internal topology/control-plane truth loop for the accepted topology set. P14 has not closed CSI, external API, frontend data path, migration, security, deployment, performance, or production readiness.`
@@ -1,86 +0,0 @@
# V3 Phase 14 S8 — P15 Handoff
Date: 2026-04-20
Status: draft (S8 P15 handoff, round-2 aligned to canonical P15 plan)
Purpose: map every P14-residual product gap to the canonical P15 track in `v3-phase-15-product-plan.md`, so P15 execution starts from the correct ownership map
## 1. Handoff Contract
Per `v3-phase-14-s8-assignment.md` §3.F: every gap that P14 does NOT close must appear here with four columns — the gap, why it is not P14, which canonical P15 track owns it, and the minimum first proof gate.
**Canonical P15 track numbering** (from `v3-phase-15-product-plan.md` §4):
1. `T1` Frontend + Data Path Contract
2. `T2` CSI / External Lifecycle Surface
3. `T3` External Control API
4. `T4` Security And Auth Posture
5. `T5` Diagnostics And Explainability
6. `T6` Operator Workflow
7. `T7` V2/V3 Coexistence And Migration
8. `T8` Deployment / Upgrade / Release Hardening
9. `Final Gate` Cluster Validation Agent
S8 does NOT invent new tracks, renumber existing ones, or prescribe P15 implementation — only the acceptance gate each P15 track must pass for the handed-off gap.
## 2. Handoff Table
| # | Gap | Why not P14 | P15 owner track | Required first proof |
|---|---|---|---|---|
| 1 | **Frontend data path** (attach → real read → real write → stale-primary fence at frontend boundary) | P14 stops at adapter/engine projection; there is no user-data I/O path in P14, and `adapter.PublishHealthy` is a control-plane signal, not a data-path handshake. | **P15 T1** Frontend + Data Path | L2: V3 frontend backend attaches to an internal volume, writes data, reads it back, triggers P14 reassignment/failover, and verifies the stale primary can no longer serve writes. Covered by T1 §5 Acceptance. |
| 2 | **Subprocess heartbeat ingress** — the L2 subprocess smoke (`TestS7Process_RealSubprocessRestartSmoke`) drives mint via `StaticDirective`, not a real heartbeat wire. The full `heartbeat → observation → controller` path is L1-only in the subprocess. | Real heartbeat ingress is a frontend/transport surface. P14 has no intention of adding one. | **P15 T1** Frontend + Data Path (observation ingress is part of the frontend/transport contract) | L2: subprocess accepts heartbeats over a real transport; observation store mutates; controller sees the change; adapter advances — end-to-end in a real process. |
| 3 | **CSI driver lifecycle** (CreateVolume / DeleteVolume / ControllerPublish / NodeStage / NodePublish / NodeUnpublish / ControllerUnpublish) | P14 has no orchestrator-facing lifecycle surface; CSI sits above the P14 authority/adapter route. | **P15 T2** CSI / External Lifecycle Surface | L2: CSI sidecar ↔ sparrow process, full Create → Publish → use → Unpublish → Delete cycle via real CSI gRPC against V3 authority (T2 §5 Acceptance). |
| 4 | **External control API** (REST / gRPC verbs for list / inspect / create / delete / resize / pause / resume) | P14 exposes hidden test flags on `sparrow` only. An external API is a net-new surface. | **P15 T3** External Control API | L2: OpenAPI / gRPC verbs backed by V3 authority state, exercised by an external client that does NOT construct `AssignmentInfo` directly. |
| 5 | **Security / auth posture** (API authn/authz, CHAP for iSCSI, TLS/mTLS where applicable, audit trail) | P14 has no external authn surface. | **P15 T4** Security And Auth | L1+L2: CHAP positive+negative for any iSCSI target shipped; API authn positive+negative; tenant-scoped listing; audit log for mutating verbs. |
| 6 | **Operator diagnostics surface** (health dashboard, structured readout of `LastUnsupported` / `LastConvergenceStuck` / `ReloadSkips`, alerts) | P14 records internal evidence (per-volume maps + structured JSON pass-line in S7 subprocess smoke) but does NOT expose it to operators. | **P15 T5** Diagnostics And Explainability | L2: structured `/healthz` + `/metrics` surfacing P14 internal states; L3 `cp85-metrics-verify.yaml` scenario class reshaped for V3. |
| 7 | **Operator workflow** (planned failover, graceful drain, supervised reassign, supervised rebuild) | P14 rejects V2 `promote/demote/HandleAssignment`. Operator-initiated actions MUST route through the publisher mint path as ordinary `AssignmentAsk`s. A new workflow surface is required. | **P15 T6** Operator Workflow | L2: operator-triggered reassign lands in `TopologyController` as an `AssignmentAsk`; publisher mints new epoch; adapter converges. Manual promote is explicitly reshaped from V2 semantics. |
| 8 | **V2 ↔ V3 migration / coexistence** (online migrate a V2 volume to V3 authority; rollback; mixed V2+V3 cluster) | Migration is a product-level transition requiring both T1 and T6. P14 is V3-only. | **P15 T7** V2/V3 Coexistence And Migration | L3 + L4: `op-upgrade-rollback.yaml` scenario class on a mixed cluster; explicit no-split-brain check under migration. |
| 9 | **Deployment / hardening** (systemd / container packaging, lockfile+storedir pathing, crash policy, log rotation, store backup) | `sparrow` is currently a test binary with hidden smoke flags; no deployment story. | **P15 T8** Deployment / Upgrade / Release Hardening | L2: deployable package starts `sparrow` with the durable store correctly mounted; crash-restart cycle preserves the P14 restart truth. L4: release-hardening soak (`cp84-soak-4h.yaml`, `cp85-soak-24h.yaml`). |
| 10 | **Final cluster validation** (end-to-end: operator creates volume → data written → primary killed → failover → operator reads data back → no loss, on real hardware) | P14 covers the control plane only; full cluster validation needs T1 + T2 + T3 + T4 + T5 + T6 all landed. | **P15 Final Gate** Cluster Validation Agent | Composite L3/L4 pack: `smoke-block-api` + `ha-restart-recovery` + `ha-failover` + `ha-io-continuity` + `fault-partition` in one run on real hardware, with evidence artifacts in `learn/test/`. |
| 11 | **Adapter fence-watchdog quantitative timeout bound** (watchdog fires at exact fence deadline + structured watchdog event for fence lineage) | S7 L1 covers "Fence command reaches executor via bridge + withheld callback does not produce Healthy"; the quantitative bound is adapter-package-owned and depends on the currently-uncommitted fence-watchdog branch. S7 sketch §10.1 Path B forbids S7 from reconfiguring the fence path. | **P14 internal follow-up** (NOT a P15 track) | Single adapter-package test replacing the PARTIAL row for evidence-matrix Claim 15. Lands when the fence-watchdog branch commits. Not an S8 gate. |
## 3. Out-Of-Scope (NOT Handed Off)
These are explicit non-goals for both P14 and current P15 bounds — not failures, not deferrals, permanent exclusions.
| Non-goal | Reason |
|---|---|
| Multi-master / leader election / distributed authority store | `v3-phase-14-s8-final-bounded-close.md` §3.6 and `v3-phase-15-product-plan.md` §2 both reject. Single-active-master is the accepted topology. Any multi-master claim would require a separate phase with a distributed-authority institution. |
| V2 `HandleAssignment` / `promote` / `demote` authority-owning semantic | Rejected at S2, reaffirmed S3-S8. Not ported under any P15 track. Operator promote surface in T6 is reshaped, not ported. |
| V2 heartbeat-as-authority | Rejected at S4. Heartbeats are observation inputs only under any P14 or P15 surface. |
## 4. Handoff Integrity Check
The table splits into two distinct row classes, each with its own integrity rule:
**Rows 1-10 — P15 product-surface gaps.** Each of these must satisfy all four checks as of 2026-04-20:
1. **Gap is not proven at any P14 level.** Cross-reference `v3-phase-14-s8-evidence-matrix.md` §3 — none of rows 1-10 appear as PROVEN or PARTIAL rows of the internal matrix. These are genuinely product-surface gaps P14 never attempted.
2. **Gap is assigned to a canonical P15 track.** Numbering matches `v3-phase-15-product-plan.md` §4.
3. **First-proof gate is concrete**, not a hand-wave. Every row names a specific test layer and a specific scenario or surface shape.
4. **Gap does not silently depend on a P14 claim being wider than is actually proven.** Example: P15 T1 is allowed to depend on P14 Claim 10 (restart re-anchors via VolumeBridge, PROVEN at L1) but NOT on any data-path claim — P14 makes none.
**Row 11 — P14 internal follow-up (exception to rule #1).** Row 11 is NOT a P15 gap; it is a targeted P14 internal completion that lands when the fence-watchdog branch commits. Its integrity rules are narrower:
1. **Row 11 MUST correspond to an existing PARTIAL or subclaim-only row of the internal matrix** — the whole point of the row is to name the follow-up that closes that partial. In this case row 11 corresponds to evidence-matrix **Claim 15 PARTIAL** (fence quantitative timeout bound). This is the inverse of rule #1 above: row 11 is listed here precisely BECAUSE it appears as PARTIAL in the matrix.
2. **Row 11 MUST NOT be assigned to any P15 track** — it is explicitly labeled "P14 internal follow-up (NOT a P15 track)" in the gap table.
3. **Row 11's first-proof gate MUST be a single-package test that closes the matrix PARTIAL row** — not a multi-track composite.
Splitting the integrity check this way keeps rule #1 for P15 gaps strict (any cross-reference to PROVEN/PARTIAL in the matrix is a bug there) while making row 11's purpose explicit and auditable (every P14 internal follow-up row should have a matrix PARTIAL anchor).
## 5. Per-Track Dependency Summary
| P15 Track | Depends on P14 claims | Notes |
|---|---|---|
| T1 Frontend + Data Path | Claims 3, 6, 10, 14 (durable reload, controller-driven, restart re-anchor, placement/rebalance) | Core product dependency. |
| T2 CSI / Lifecycle | T1 + Claims 6, 10 | CSI needs a working data path. |
| T3 External API | Claims 6, 14 | API translates external verbs to `AssignmentAsk`s. |
| T4 Security | T1 + T3 | Authn/authz sits above the surfaces T1 and T3 expose. |
| T5 Diagnostics | Claims 2, 8, 11, 12 (supportability / stuck / unsupported evidence) | Surfaces what P14 already records internally. |
| T6 Operator Workflow | Claims 6, 9, 14 (controller-driven, supersede, placement) | Operator actions are `AssignmentAsk`s through the controller. |
| T7 Migration | T1 + T6 | Needs both frontend and operator surfaces. |
| T8 Deployment | Claim 3 (durable reload) + T1 | Depends on durable store shape and frontend. |
| Final Gate | All P14 claims + all P15 tracks | Composite. |
## 6. Closure
Every product gap that S8 identifies is listed here with a canonical P15 track or explicit P14 follow-up. Track numbering matches `v3-phase-15-product-plan.md`. P15 execution starts from this table without needing to re-derive the ownership map.
@@ -1,107 +0,0 @@
# V3 Phase 14 S8 — V2 Scenario Port Classification
Date: 2026-04-20
Status: draft (S8 scenario classification)
Purpose: classify every V2 testrunner scenario (`weed/storage/blockvol/testrunner/scenarios/`) as P14-runnable / P15-blocked / deferred, producing the table required by `v3-phase-14-s8-assignment.md` §3.C
## 1. Classification Vocabulary
| Class | Meaning | P14 S8 action |
|---|---|---|
| **RUNNABLE-P14** | Shape maps onto S4-S7 internal surfaces. Runnable with V3 binary + no external frontend. | Keep scenario YAML shape; adapt actions to V3 route; run as L3 classification only (S8 does not need to execute L3 itself). |
| **BLOCKED-FRONTEND** | Requires iSCSI / NVMe / CSI / real data path. P15 Frontend track gate. | Preserve scenario reference; hand off to P15 with required precondition. |
| **BLOCKED-OPS** | Requires operator CLI / HTTP / admin workflow. P15 Ops track gate. | Hand off to P15 Ops. |
| **BLOCKED-HA** | Requires multi-master / leader election / distributed authority. Outside P14/P15 bounded claim. | Mark as out-of-scope for both P14 and current P15 bounds. Document only. |
| **BLOCKED-PERF** | Benchmark / soak / perf-baseline; not a correctness gate. | Defer to release-hardening track; document only. |
| **PORT-MECHANISM** | Testrunner machinery itself (not a scenario): action vocabulary, artifact collection, scenario YAML shape. | Port machinery without V2 authority semantics (see `v3-phase-14-s6-s8-v2-port-plan.md` §5). |
## 2. Testrunner Machinery (port decision)
`weed/storage/blockvol/testrunner/` — YAML-driven runner with action vocabulary, artifact collection, report generation. V3 acceptance pack (P14 S8 carries scenario SHAPES forward; actual V3 testrunner integration is a P15 Cluster Validation track deliverable).
| Component | Classification | S8 action |
|---|---|---|
| `testrunner/*.go` (engine / parser / registry / reporter / metrics) | PORT-MECHANISM | Port to V3 when V3 has a stable CLI surface. S8 does not run the runner directly. |
| `testrunner/actions/*.go` (37 registered actions) | PORT-MECHANISM | Port per-action as V3 surfaces appear. S8 ports NONE directly (no V3 CLI surface exists yet outside `sparrow` hidden smoke flags). |
| `testrunner/scenarios/*.yaml` | scenario-by-scenario below | — |
**S8 decision:** do not port testrunner machinery into the V3 tree as part of S8. The scenario SHAPES below are what S8 classifies; the runner itself is a P15 Cluster Validation deliverable.
## 3. Public Scenarios Table
Path: `weed/storage/blockvol/testrunner/scenarios/public/`.
| # | Scenario | P14 route it touches | Class | Notes |
|---|---|---|---|---|
| 1 | `smoke-block-api.yaml` | block create / write / read / verify | **BLOCKED-FRONTEND** | Requires V3 block API surface + real data path. No V3 equivalent exists. P15 Frontend track. |
| 2 | `smoke-iscsi.yaml` | kernel iSCSI sanity | **BLOCKED-FRONTEND** | Requires iSCSI target + kernel initiator. P15 Frontend. |
| 3 | `smoke-kv.yaml` | KV layer smoke | **BLOCKED-FRONTEND** | KV path is out of P14 scope. P15 Frontend. |
| 4 | `e2e-block.yaml` / `e2e-block-auto.yaml` | end-to-end block I/O with V3 backend | **BLOCKED-FRONTEND** | Same preconditions as #1. |
| 5 | `e2e-kv.yaml` / `e2e-kv-auto.yaml` / `e2e-combined-auto.yaml` | KV + block combined | **BLOCKED-FRONTEND** | Same as #3. |
| 6 | `ha-restart-recovery.yaml` | restart-and-reload closed loop | **RUNNABLE-P14** | Shape maps directly onto S7 restart route. L1 / L2 subprocess smoke already proves the route in-process; scenario YAML would just drive the same route under scenario harness. Keep shape; port when V3 testrunner lands. Covered by Claim 3 / Claim 10 of the evidence matrix at L0/L1/L2. |
| 7 | `ha-failover.yaml` | failover under live I/O | **BLOCKED-FRONTEND** | Shape touches S4-S7 (observation → controller → reassign → adapter), but "under live I/O" requires the data path. Classification: control-plane portions are RUNNABLE-P14 (covered by Claim 14 at L1); I/O-continuity portions are BLOCKED-FRONTEND. Split at P15 port time. |
| 8 | `ha-full-lifecycle.yaml` | bind → heal → failover → rebuild → restart full cycle | **BLOCKED-FRONTEND** | Contains rebuild + I/O. Rebuild is P14-internal but has no V3 equivalent outside V2 bridge; I/O is frontend. Defer whole scenario to P15. |
| 9 | `ha-io-continuity.yaml` | zero data loss across failover | **BLOCKED-FRONTEND** | Entirely data-path. P15 Frontend. |
| 10 | `ha-rebuild.yaml` | full-extent rebuild via transport | **BLOCKED-FRONTEND** | Rebuild transport is V2-side; V3 adapter has no rebuild surface in current slice. Deferred. |
| 11 | `crash-recovery.yaml` | process kill + restart + verify | **RUNNABLE-P14** | Control-plane portion identical to #6. Data-verify portion is BLOCKED-FRONTEND. Split at port time. |
| 12 | `diag-restart-recovery.yaml` | diagnostics on restart | **RUNNABLE-P14** (control-plane subset) | S7 subprocess smoke emits structured JSON (`Bootstrap.ReloadedRecords`, `ReloadSkips`, no-backward-mint); maps onto this shape. Operator dashboards / full diag bundle is P15 Diagnostics. |
| 13 | `fault-partition.yaml` | network partition + recovery | **RUNNABLE-P14** (control-plane) | Control-plane partition → stale observation / convergence stuck is covered by Claims 8, 11 at L1. Real netem + data-path partition is BLOCKED-FRONTEND. |
| 14 | `fault-netem.yaml` | generic network fault injection | **BLOCKED-FRONTEND** | Needs data path under load. P15. |
| 15 | `fault-disk-full.yaml` | ENOSPC on primary | **BLOCKED-FRONTEND** | Needs write path. P15. |
| 16 | `consistency-epoch.yaml` | epoch monotonicity across failover | **RUNNABLE-P14** | Pure control-plane claim — already covered at L0 by `TestDurableAuthority_PublisherAdvancesFromReloaded` and at L1 by `TestS7_*` restart tests. L3 version would add cross-process multi-primary validation. |
| 17 | `consistency-lease.yaml` | lease-based guard | **BLOCKED-OPS** | Lease semantics are V2-specific; V3 uses publisher-owned Epoch. Would need a reshaped V3 scenario. Defer. |
| 18 | `lease-expiry-write-gate.yaml` | write blocked on lease expiry | **BLOCKED-FRONTEND** | Write path + lease. P15. |
| 19 | `lease-renewal-under-io.yaml` | lease renewal during I/O | **BLOCKED-FRONTEND** | Same. |
| 20 | `cp11b3-auto-failover.yaml` | automatic failover trigger | **RUNNABLE-P14** (control-plane) | Control-plane covered at L1 by `TestTopologyControllerToPublisher_E2E_MultiVolumePlacementAndFailover`. |
| 21 | `cp11b3-manual-promote.yaml` | operator-initiated promote | **BLOCKED-OPS** | Requires operator CLI + admin API. P15 Ops. Also — manual promote is V2 semantics; V3 does not have a direct equivalent by design (S6-S8 V2 port plan §2 rejects V2 `promote` ownership). |
| 22 | `cp11b3-fast-reconnect.yaml` | reconnect skips unnecessary failover | **RUNNABLE-P14** (control-plane) | S6 normal-lag handling covers this — `TestConvergence_NormalLag_OldObservationDoesNotSupersede`. Full scenario requires real reconnect transport = BLOCKED-FRONTEND for the transport layer, RUNNABLE-P14 for the control-plane. |
**Public scenarios summary:**
- RUNNABLE-P14: 6 scenarios (ha-restart-recovery, crash-recovery control plane, diag-restart-recovery, fault-partition control plane, consistency-epoch, cp11b3-auto-failover control plane) — all covered by Claims 3/10/11/12/14 at L0/L1/L2.
- BLOCKED-FRONTEND: 13 scenarios
- BLOCKED-OPS: 2 scenarios (lease-consistency, manual-promote)
- Mixed (split required at port time): ha-failover, cp11b3-fast-reconnect
## 4. Internal Scenarios Table (selected, by category)
Path: `weed/storage/blockvol/testrunner/scenarios/internal/`. 50+ files; classifying by category rather than per-file.
| Category | Example files | Class | Notes |
|---|---|---|---|
| Recovery baselines | `recovery-baseline-restart.yaml` / `recovery-baseline-failover.yaml` / `recovery-baseline-partition.yaml` / `recovery-baseline-rebuild.yaml` | RUNNABLE-P14 (control-plane) | Restart/failover/partition portions map onto S4-S7; rebuild portions need V3 rebuild surface (not in S8). |
| Coordination dev-cycle | `coord-dev-cycle.yaml` / `coord-ha-failover.yaml` / `coord-smoke-iscsi.yaml` | BLOCKED-FRONTEND | iSCSI / end-to-end workflows. |
| CP103 performance matrix | `cp103-*.yaml` | BLOCKED-PERF | Performance, not correctness. Defer to release hardening. |
| CP85 chaos | `cp85-chaos-partition.yaml` / `cp85-chaos-primary-kill-loop.yaml` / `cp85-chaos-replica-kill-loop.yaml` / `cp85-role-flap.yaml` / `cp85-session-storm.yaml` | Mixed | Control-plane portions RUNNABLE-P14 as property tests (kill-loop restart, role-flap convergence). I/O portions BLOCKED-FRONTEND. |
| CP85 metrics / observability | `cp85-metrics-verify.yaml` | BLOCKED-OPS | Operator metrics pipeline. P15 Diagnostics. |
| CP85 soak | `cp85-soak-24h.yaml` / `cp84-soak-4h.yaml` | BLOCKED-PERF | Long-running stability. Release hardening. |
| CP85 database / filesystem | `cp85-db-ext4-fsck.yaml` / `cp85-db-sqlite-crash.yaml` / `cp85-expand-failover.yaml` | BLOCKED-FRONTEND | Real filesystem + DB workload. |
| Snapshot | `cp11a4-snapshot-export-import.yaml` / `cp83-snapshot-expand.yaml` / `cp85-snapshot-stress.yaml` | BLOCKED-FRONTEND | Snapshot API is P15 Frontend / Ops. |
| EC / Erasure | `ec3-*.yaml` / `ec5-*.yaml` | BLOCKED-FRONTEND | Data path erasure. |
| HA extensions | `ha-failover-during-rebuild.yaml` / `ha-multi-client-failover.yaml` | BLOCKED-FRONTEND | Multi-client data path. |
| Benchmark | `benchmark-*.yaml` / `bench-validated.yaml` / `baseline-full-roce.yaml` / `fsync-only-test.yaml` | BLOCKED-PERF | Performance. |
| DM / stripe | `dm-stripe-two-server.yaml` | BLOCKED-FRONTEND | Device mapper. |
| Operator lifecycle | `op-upgrade-rollback.yaml` / `op-csi-lifecycle.yaml` / `op-failure-injection.yaml` | BLOCKED-OPS | P15 Ops + Migration. |
| Real-workload validation | `cp13-8-real-workload-validation.yaml` | BLOCKED-FRONTEND + BLOCKED-PERF | Full stack. |
**Internal scenarios summary:**
- Directly useful now as P14-internal control-plane L3 shape: ~8 scenarios (recovery-baseline × 3, cp85 chaos × 3, consistency-epoch at public, diag-restart-recovery at public). All already covered at L0/L1/L2 by the evidence matrix; L3 runnable is P15 Cluster Validation.
- BLOCKED-FRONTEND majority: ~30 scenarios tied to I/O / iSCSI / NVMe / snapshot / DB workloads.
- BLOCKED-OPS: ~8 scenarios tied to operator workflows.
- BLOCKED-PERF: ~10 scenarios tied to perf / soak.
- BLOCKED-HA: none of the current set explicitly requires multi-master, but `cp85-role-flap` and any future "multi-master" test would be BLOCKED-HA.
## 5. Port Shape (V3)
When V3 eventually ships a testrunner integration (P15 Cluster Validation), the port order should be:
1. **RUNNABLE-P14 scenarios first** — port scenario YAML shape (not V2 actions) against V3 `sparrow` binary + testrunner-wrapped smoke flags. Keep the structured-JSON stdout shape from S7 smoke; wrap it in testrunner action vocabulary.
2. **Mixed scenarios second** — split each into (control-plane sub-scenario, data-path sub-scenario). Port the control-plane half first.
3. **BLOCKED-FRONTEND last** — arrives when P15 Frontend ships iSCSI / NVMe / CSI.
4. **BLOCKED-OPS as needed** — arrives with P15 Ops admin surface.
5. **BLOCKED-PERF and BLOCKED-HA** — release-hardening and explicit non-goals respectively.
## 6. Closure
Per S8 assignment §3.C, this table is sufficient for S8. Actual scenario PORTING is P15 Cluster Validation work and is not an S8 deliverable. S8 produces the classification + the residual map; P15 executes.
Evidence that the RUNNABLE-P14 scenarios' underlying claims are proven today is in `v3-phase-14-s8-evidence-matrix.md` §3 — each RUNNABLE-P14 scenario above has at least one PROVEN row backing it at L0/L1/L2.
@@ -12,7 +12,6 @@ import (
"path/filepath"
"regexp"
"strings"
"sync"
"testing"
"time"
)
@@ -23,16 +22,10 @@ const (
FilerUrl = "http://localhost:8888"
)
// Helper to run commands in background and track PIDs for cleanup. Guarded
// by runningCmdsLock so parallel subprocess startup can append safely.
var (
runningCmds []*exec.Cmd
runningCmdsLock sync.Mutex
)
// Helper to run commands in background and track PIDs for cleanup
var runningCmds []*exec.Cmd
func cleanup() {
runningCmdsLock.Lock()
defer runningCmdsLock.Unlock()
for _, cmd := range runningCmds {
if cmd.Process != nil {
cmd.Process.Kill()
@@ -66,9 +59,7 @@ func startWeed(t *testing.T, name string, args ...string) *exec.Cmd {
if err != nil {
t.Fatalf("Failed to start weed %v: %v", args, err)
}
runningCmdsLock.Lock()
runningCmds = append(runningCmds, cmd)
runningCmdsLock.Unlock()
return cmd
}
@@ -116,23 +107,14 @@ func ensureEnvironment(t *testing.T) {
waitForUrl(t, MasterUrl+"/cluster/status", 10)
// 3. Start Volume Server (Worker)
// Start 14 volume servers to verify RS(10,4) default EC. Fork/exec in
// parallel because startWeed is non-blocking and the per-process fork +
// mkdir + log-file-open overhead stacks up sequentially on cold CI
// disks, eating most of the admin /health wait budget further down.
var volWg sync.WaitGroup
// Start 14 volume servers to verify RS(10,4) default EC
for i := 1; i <= 14; i++ {
volWg.Add(1)
go func(i int) {
defer volWg.Done()
volName := fmt.Sprintf("volume%d", i)
port := 8080 + i - 1
dir := filepath.Join("tmp", volName)
os.MkdirAll(dir, 0755)
startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost")
}(i)
volName := fmt.Sprintf("volume%d", i)
port := 8080 + i - 1
dir := filepath.Join("tmp", volName)
os.MkdirAll(dir, 0755)
startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost")
}
volWg.Wait()
// 4. Start Filer
os.MkdirAll(filepath.Join("tmp", "filer"), 0755)
@@ -154,12 +136,7 @@ func ensureEnvironment(t *testing.T) {
os.RemoveAll(filepath.Join("tmp", "admin"))
os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
// Admin is started after master, 14 volume servers, filer and 2 workers,
// so under cold CI conditions the wait here has to absorb the tail of
// every earlier subprocess coming up. 60s is too tight and has flaked;
// 180s gives comfortable headroom without meaningfully extending the
// fast path (the first successful /health usually hits well under 30s).
waitForUrl(t, AdminUrl+"/health", 180)
waitForUrl(t, AdminUrl+"/health", 60)
t.Log("Environment started successfully")
}
-277
View File
@@ -1,277 +0,0 @@
package fuse_dlm
import (
"fmt"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestDLMConcurrentWritersSameFile verifies that two mounts writing to the same
// file concurrently produce valid (non-corrupted) data. With DLM enabled, the
// writes are serialized — one blocks until the other completes.
//
// Note: cross-mount read consistency depends on FUSE kernel cache invalidation
// and filer metadata subscription, which are asynchronous. This test verifies
// write integrity, not instant read convergence.
func TestDLMConcurrentWritersSameFile(t *testing.T) {
if testing.Short() {
t.Skip("skipping DLM integration test in short mode")
}
cluster := startDLMTestCluster(t)
t.Cleanup(cluster.Stop)
const iterations = 5
for iter := 0; iter < iterations; iter++ {
fileName := fmt.Sprintf("concurrent_write_%d.txt", iter)
payloadA := []byte(fmt.Sprintf("mount0-iteration-%d-payload-AAAA", iter))
payloadB := []byte(fmt.Sprintf("mount1-iteration-%d-payload-BBBB", iter))
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), payloadA, 0644)
assert.NoError(t, err, "mount0 write iteration %d", iter)
}()
go func() {
defer wg.Done()
err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), payloadB, 0644)
assert.NoError(t, err, "mount1 write iteration %d", iter)
}()
wg.Wait()
// Verify file is readable and contains one of the expected payloads
// (read from mount0 — its own view is authoritative for write success).
content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
require.NoError(t, err, "read from mount0 iteration %d", iter)
validPayload := string(content) == string(payloadA) || string(content) == string(payloadB)
assert.True(t, validPayload,
"iteration %d: content must be one of the expected payloads, got: %q", iter, content)
}
}
// TestDLMRepeatedOpenWriteClose verifies that repeated open/write/close cycles
// from both mounts all succeed without errors.
func TestDLMRepeatedOpenWriteClose(t *testing.T) {
if testing.Short() {
t.Skip("skipping DLM integration test in short mode")
}
cluster := startDLMTestCluster(t)
t.Cleanup(cluster.Stop)
const cycles = 20
fileName := "repeated_write.txt"
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for i := 0; i < cycles; i++ {
data := []byte(fmt.Sprintf("mount0-cycle-%d", i))
err := os.WriteFile(filepath.Join(cluster.mountPoints[0], fileName), data, 0644)
assert.NoError(t, err, "mount0 cycle %d", i)
}
}()
go func() {
defer wg.Done()
for i := 0; i < cycles; i++ {
data := []byte(fmt.Sprintf("mount1-cycle-%d", i))
err := os.WriteFile(filepath.Join(cluster.mountPoints[1], fileName), data, 0644)
assert.NoError(t, err, "mount1 cycle %d", i)
}
}()
wg.Wait()
// File must be readable from at least one mount
content, err := os.ReadFile(filepath.Join(cluster.mountPoints[0], fileName))
require.NoError(t, err)
assert.NotEmpty(t, content, "file must not be empty")
}
// TestDLMWriteBlocksSecondWriter verifies the core DLM guarantee: while one
// mount has a file open for writing, another mount's write-open blocks until
// the first mount closes the file.
func TestDLMWriteBlocksSecondWriter(t *testing.T) {
if testing.Short() {
t.Skip("skipping DLM integration test in short mode")
}
cluster := startDLMTestCluster(t)
t.Cleanup(cluster.Stop)
fileName := "blocking_test.txt"
path0 := filepath.Join(cluster.mountPoints[0], fileName)
path1 := filepath.Join(cluster.mountPoints[1], fileName)
// Mount 0 opens the file for writing and holds it open
f, err := os.OpenFile(path0, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
require.NoError(t, err, "mount0 open")
_, err = f.Write([]byte("mount0-holds-lock"))
require.NoError(t, err, "mount0 write")
// Mount 1 tries to write — should block (we use a goroutine with atomic flag)
var mount1Completed atomic.Bool
mount1Done := make(chan error, 1)
go func() {
err := os.WriteFile(path1, []byte("mount1-waited"), 0644)
mount1Completed.Store(true)
mount1Done <- err
}()
// Give mount 1 a moment — it should NOT complete while mount 0 holds the file open
time.Sleep(3 * time.Second)
require.False(t, mount1Completed.Load(),
"mount1 write must not complete while mount0 holds the file open")
t.Log("mount1 write is blocked as expected while mount0 holds the file")
// Mount 0 closes the file — this releases the DLM lock
require.NoError(t, f.Close(), "mount0 close")
// Mount 1 should now complete
select {
case err := <-mount1Done:
assert.NoError(t, err, "mount1 write after mount0 close")
case <-time.After(30 * time.Second):
t.Fatal("mount1 write did not complete within 30s after mount0 closed")
}
}
// TestDLMRenameWhileWriteOpen verifies that a rename is coordinated with DLM:
// while mount0 has a file open for writing (re-opened after creation),
// mount1 cannot rename it until mount0 closes the file.
func TestDLMRenameWhileWriteOpen(t *testing.T) {
if testing.Short() {
t.Skip("skipping DLM integration test in short mode")
}
cluster := startDLMTestCluster(t)
t.Cleanup(cluster.Stop)
origName := "rename_source.txt"
newName := "rename_dest.txt"
// Create and close the file first so it's flushed to the filer and
// visible on both mounts.
require.NoError(t, os.WriteFile(
filepath.Join(cluster.mountPoints[0], origName),
[]byte("initial-content"), 0644))
time.Sleep(2 * time.Second) // metadata propagation
// Verify mount1 can see the file
_, err := os.Stat(filepath.Join(cluster.mountPoints[1], origName))
require.NoError(t, err, "mount1 should see the file")
// Mount 0 re-opens the file for writing and holds it open
f, err := os.OpenFile(
filepath.Join(cluster.mountPoints[0], origName),
os.O_WRONLY|os.O_TRUNC, 0644)
require.NoError(t, err, "mount0 reopen")
_, err = f.Write([]byte("data-while-holding-lock"))
require.NoError(t, err, "mount0 write")
// Mount 1 tries to rename — should block because mount0 holds the
// DLM lock on the old path
var renameCompleted atomic.Bool
renameDone := make(chan error, 1)
go func() {
err := os.Rename(
filepath.Join(cluster.mountPoints[1], origName),
filepath.Join(cluster.mountPoints[1], newName))
renameCompleted.Store(true)
renameDone <- err
}()
// Rename must NOT complete while mount0 holds the file open
time.Sleep(3 * time.Second)
require.False(t, renameCompleted.Load(),
"rename must not complete while mount0 holds the file open")
t.Log("rename is blocked as expected while mount0 holds the file")
// Mount 0 closes → releases DLM lock → rename should proceed
require.NoError(t, f.Close(), "mount0 close")
select {
case err := <-renameDone:
assert.NoError(t, err, "rename after mount0 close")
case <-time.After(30 * time.Second):
t.Fatal("rename did not complete within 30s after mount0 closed")
}
}
// Note: Same-mount rename while a file is open for writing is not tested here
// because macOS FUSE serializes operations on the same inode, causing a
// kernel-level deadlock between the Rename handler's internal flush and the
// pending Close. Same-mount coordination is already handled by the per-mount
// fhLockTable and FUSE kernel serialization, so DLM is not needed for it.
// TestDLMConcurrentRenames verifies that two concurrent renames of the same
// file from different mounts don't corrupt metadata. DLM locks on both old
// and new paths ensure renames are serialized.
func TestDLMConcurrentRenames(t *testing.T) {
if testing.Short() {
t.Skip("skipping DLM integration test in short mode")
}
cluster := startDLMTestCluster(t)
t.Cleanup(cluster.Stop)
// Create a file first
origPath := filepath.Join(cluster.mountPoints[0], "rename_race.txt")
require.NoError(t, os.WriteFile(origPath, []byte("original-content"), 0644))
time.Sleep(1 * time.Second) // propagation
// Both mounts try to rename the same file concurrently
var wg sync.WaitGroup
var errA, errB error
wg.Add(2)
go func() {
defer wg.Done()
errA = os.Rename(
filepath.Join(cluster.mountPoints[0], "rename_race.txt"),
filepath.Join(cluster.mountPoints[0], "renamed_by_mount0.txt"),
)
}()
go func() {
defer wg.Done()
errB = os.Rename(
filepath.Join(cluster.mountPoints[1], "rename_race.txt"),
filepath.Join(cluster.mountPoints[1], "renamed_by_mount1.txt"),
)
}()
wg.Wait()
// At least one rename should succeed; the other may fail with ENOENT
// since the source was already moved.
succeeded := 0
if errA == nil {
succeeded++
t.Logf("mount0 rename succeeded")
} else {
t.Logf("mount0 rename failed: %v", errA)
}
if errB == nil {
succeeded++
t.Logf("mount1 rename succeeded")
} else {
t.Logf("mount1 rename failed: %v", errB)
}
assert.GreaterOrEqual(t, succeeded, 1, "at least one rename must succeed")
}
-516
View File
@@ -1,516 +0,0 @@
package fuse_dlm
import (
"context"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"sync"
"syscall"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
const filerGroup = "fuse-dlm-test"
// dlmTestCluster manages a full SeaweedFS cluster with 2 filers and 2 FUSE
// mounts for testing DLM-based cross-mount write coordination.
type dlmTestCluster struct {
t testing.TB
baseDir string
weedBinary string
masterPort int
masterGrpcPort int
volumePort int
volumeGrpcPort int
filerPorts [2]int
filerGrpcPorts [2]int
mountPoints [2]string
masterCmd *exec.Cmd
volumeCmd *exec.Cmd
filerCmds [2]*exec.Cmd
mountCmds [2]*exec.Cmd
logFiles []*os.File
cleanupOnce sync.Once
}
func startDLMTestCluster(t testing.TB) *dlmTestCluster {
binary := findWeedBinary()
if binary == "" {
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
}
baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_dlm_test_")
require.NoError(t, err)
c := &dlmTestCluster{
t: t,
baseDir: baseDir,
weedBinary: binary,
}
// Register cleanup early so processes are stopped even if a require fails below.
t.Cleanup(c.Stop)
// Allocate ports: master(2) + volume(2) + filer0(2) + filer1(2) = 8
ports := allocatePorts(t, 8)
c.masterPort = ports[0]
c.masterGrpcPort = ports[1]
c.volumePort = ports[2]
c.volumeGrpcPort = ports[3]
c.filerPorts[0] = ports[4]
c.filerGrpcPorts[0] = ports[5]
c.filerPorts[1] = ports[6]
c.filerGrpcPorts[1] = ports[7]
// Write empty security.toml
configDir := filepath.Join(baseDir, "config")
require.NoError(t, os.MkdirAll(configDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
// Start master
require.NoError(t, c.startMaster(configDir))
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second),
"master not ready\n%s", c.tailLog("master"))
// Start volume
require.NoError(t, c.startVolume(configDir))
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second),
"volume not ready\n%s", c.tailLog("volume"))
// Start 2 filers
for i := 0; i < 2; i++ {
require.NoError(t, c.startFiler(i, configDir))
require.NoError(t, c.waitForTCP(fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[i]), 30*time.Second),
"filer %d not ready\n%s", i, c.tailLog(fmt.Sprintf("filer%d", i)))
}
require.NoError(t, c.waitForFilerCount(2, 30*time.Second), "filer group registration")
require.NoError(t, c.waitForLockRingConverged(30*time.Second), "lock ring convergence")
// Start 2 mounts, both pointing at filer0 for metadata consistency.
// (filer1 exists for the DLM lock ring but both mounts share filer0's
// metadata store since leveldb is per-filer.)
for i := 0; i < 2; i++ {
mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
require.NoError(t, os.MkdirAll(mp, 0755))
c.mountPoints[i] = mp
require.NoError(t, c.startMount(i, configDir))
require.NoError(t, c.waitForMount(mp, 30*time.Second),
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
return c
}
func (c *dlmTestCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
// Stop mounts first (triggers flush + DLM unlock).
// Use stopCmd for bounded wait to avoid hanging on wedged FUSE processes.
for i := 1; i >= 0; i-- {
stopCmd(c.mountCmds[i])
// Backup unmount in case FUSE teardown didn't clean up
exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
}
// Stop filers, volume, master
for i := 1; i >= 0; i-- {
stopCmd(c.filerCmds[i])
}
stopCmd(c.volumeCmd)
stopCmd(c.masterCmd)
for _, f := range c.logFiles {
f.Close()
}
// Copy logs for CI
c.copyLogsForCI()
if !c.t.Failed() {
os.RemoveAll(c.baseDir)
}
// Wait for ports to be fully released before the next test
// allocates new ports (avoids TIME_WAIT collisions).
time.Sleep(2 * time.Second)
})
}
// masterAddress returns the master address in the format that encodes both
// HTTP and gRPC ports: "host:httpPort.grpcPort". This is the format that
// SeaweedFS uses to communicate non-default gRPC ports between components.
func (c *dlmTestCluster) masterAddress() string {
return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
}
func (c *dlmTestCluster) startMaster(configDir string) error {
c.masterCmd = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"master",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.masterPort),
"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
"-mdir="+filepath.Join(c.baseDir, "master"),
)
return c.startCmd(c.masterCmd, "master")
}
func (c *dlmTestCluster) startVolume(configDir string) error {
volDir := filepath.Join(c.baseDir, "volume")
if err := os.MkdirAll(volDir, 0755); err != nil {
return fmt.Errorf("create volume dir: %w", err)
}
c.volumeCmd = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"volume",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.volumePort),
"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
"-master="+c.masterAddress(),
"-dir="+volDir,
"-max=10",
)
return c.startCmd(c.volumeCmd, "volume")
}
func (c *dlmTestCluster) startFiler(idx int, configDir string) error {
filerDir := filepath.Join(c.baseDir, fmt.Sprintf("filer%d", idx))
if err := os.MkdirAll(filerDir, 0755); err != nil {
return fmt.Errorf("create filer dir: %w", err)
}
c.filerCmds[idx] = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"filer",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.filerPorts[idx]),
"-port.grpc="+strconv.Itoa(c.filerGrpcPorts[idx]),
"-master="+c.masterAddress(),
"-filerGroup="+filerGroup,
"-defaultStoreDir="+filerDir,
)
return c.startCmd(c.filerCmds[idx], fmt.Sprintf("filer%d", idx))
}
func (c *dlmTestCluster) filerAddress(idx int) string {
return string(pb.NewServerAddress("127.0.0.1", c.filerPorts[idx], c.filerGrpcPorts[idx]))
}
func (c *dlmTestCluster) startMount(idx int, configDir string) error {
cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return fmt.Errorf("create cache dir: %w", err)
}
c.mountCmds[idx] = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"mount",
"-filer="+c.filerAddress(0), // both mounts use filer0 for shared metadata
"-dir="+c.mountPoints[idx],
"-filer.path=/",
"-dirAutoCreate",
"-allowOthers=false",
"-cacheDir="+cacheDir,
"-dlm",
)
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
}
func (c *dlmTestCluster) startCmd(cmd *exec.Cmd, name string) error {
logPath := filepath.Join(c.baseDir, "logs")
if err := os.MkdirAll(logPath, 0755); err != nil {
return fmt.Errorf("create log dir: %w", err)
}
logFile, err := os.Create(filepath.Join(logPath, name+".log"))
if err != nil {
return err
}
c.logFiles = append(c.logFiles, logFile)
cmd.Stdout = logFile
cmd.Stderr = logFile
return cmd.Start()
}
func (c *dlmTestCluster) tailLog(name string) string {
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
if err != nil {
return fmt.Sprintf("(log %s not available: %v)", name, err)
}
const maxTail = 8192
if len(data) > maxTail {
data = data[len(data)-maxTail:]
}
return string(data)
}
func (c *dlmTestCluster) copyLogsForCI() {
ciLogDir := "/tmp/seaweedfs-fuse-dlm-logs"
os.MkdirAll(ciLogDir, 0755)
logsDir := filepath.Join(c.baseDir, "logs")
entries, err := os.ReadDir(logsDir)
if err != nil {
return
}
for _, e := range entries {
data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
if err != nil {
continue
}
os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
}
}
func (c *dlmTestCluster) waitForTCP(addr string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", addr, time.Second)
if err == nil {
conn.Close()
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("service at %s not ready within timeout", addr)
}
// waitForMount waits for a FUSE filesystem to actually be mounted at
// mountPoint by comparing the device ID of the mount point against its parent.
// A plain directory (pre-created before mount) has the same device as its
// parent; a mounted FUSE filesystem has a different device.
func (c *dlmTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
parentDir := filepath.Dir(mountPoint)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
parentStat, err := os.Stat(parentDir)
if err != nil {
time.Sleep(200 * time.Millisecond)
continue
}
mountStat, err := os.Stat(mountPoint)
if err != nil {
time.Sleep(200 * time.Millisecond)
continue
}
parentSys := parentStat.Sys().(*syscall.Stat_t)
mountSys := mountStat.Sys().(*syscall.Stat_t)
if parentSys.Dev != mountSys.Dev {
// Different device = FUSE mounted
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
}
func (c *dlmTestCluster) filerGRPCAddress(idx int) string {
return fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPorts[idx])
}
func (c *dlmTestCluster) waitForFilerCount(expected int, timeout time.Duration) error {
addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return err
}
defer conn.Close()
client := master_pb.NewSeaweedClient(conn)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
ClientType: "filer",
FilerGroup: filerGroup,
})
cancel()
if err == nil && len(resp.ClusterNodes) >= expected {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("timed out waiting for %d filers in group %q", expected, filerGroup)
}
// waitForLockRingConverged verifies that both filers have a consistent view of
// the lock ring by acquiring the same lock through each filer and checking
// mutual exclusion. Adapted from test/s3/distributed_lock/.
func (c *dlmTestCluster) waitForLockRingConverged(timeout time.Duration) error {
deadline := time.Now().Add(timeout)
owners := []pb.ServerAddress{
pb.ServerAddress(c.filerGRPCAddress(0)),
pb.ServerAddress(c.filerGRPCAddress(1)),
}
ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
ring.SetServers(owners)
attempt := 0
for time.Now().Before(deadline) {
testKeys := convergenceKeysPerPrimary(ring, owners, attempt)
attempt++
allConverged := true
for _, key := range testKeys {
converged, _ := c.checkLockMutualExclusion(key)
if !converged {
allConverged = false
break
}
}
if allConverged {
return nil
}
time.Sleep(500 * time.Millisecond)
}
return fmt.Errorf("lock ring did not converge")
}
// convergenceKeysPerPrimary generates one test key per primary filer.
func convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
found := make(map[pb.ServerAddress]bool)
var keys []string
for i := 0; len(found) < len(owners) && i < 10000; i++ {
key := fmt.Sprintf("convergence-test-%d-%d", attempt, i)
primary, _ := ring.GetPrimaryAndBackup(key)
if !found[primary] {
found[primary] = true
keys = append(keys, key)
}
}
return keys
}
func (c *dlmTestCluster) checkLockMutualExclusion(key string) (bool, error) {
// Try to lock via filer0
conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return false, err
}
defer conn0.Close()
client0 := filer_pb.NewSeaweedFilerClient(conn0)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
resp0, err := client0.DistributedLock(ctx, &filer_pb.LockRequest{
Name: key,
SecondsToLock: 5,
Owner: "convergence-test-0",
})
if err != nil {
return false, err
}
if resp0.Error != "" {
return false, fmt.Errorf("lock0: %s", resp0.Error)
}
// Try to lock via filer1 — should fail (already locked)
conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return false, err
}
defer conn1.Close()
client1 := filer_pb.NewSeaweedFilerClient(conn1)
resp1, err := client1.DistributedLock(ctx, &filer_pb.LockRequest{
Name: key,
SecondsToLock: 5,
Owner: "convergence-test-1",
})
// Unlock via filer0
client0.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
Name: key,
RenewToken: resp0.RenewToken,
})
if err != nil {
return false, err
}
// If filer1 also got the lock, the ring hasn't converged
if resp1.Error == "" {
// Unlock the second one too
client1.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
Name: key,
RenewToken: resp1.RenewToken,
})
return false, nil
}
return true, nil
}
func stopCmd(cmd *exec.Cmd) {
if cmd == nil || cmd.Process == nil {
return
}
cmd.Process.Signal(syscall.SIGTERM)
done := make(chan struct{})
go func() {
cmd.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(10 * time.Second):
cmd.Process.Kill()
<-done
}
}
func allocatePorts(t testing.TB, n int) []int {
t.Helper()
// Hold all listeners open until all ports are collected, then close
// them together. This prevents the OS from reassigning a just-freed
// port to the next Listen call within the same allocation batch.
listeners := make([]net.Listener, 0, n)
ports := make([]int, 0, n)
defer func() {
for _, l := range listeners {
l.Close()
}
}()
for i := 0; i < n; i++ {
l, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
listeners = append(listeners, l)
ports = append(ports, l.Addr().(*net.TCPAddr).Port)
}
return ports
}
func findWeedBinary() string {
if p := os.Getenv("WEED_BINARY"); p != "" {
return p
}
if p, err := exec.LookPath("weed"); err == nil {
return p
}
candidates := []string{
"../../weed/weed",
"./weed",
}
for _, c := range candidates {
if info, err := os.Stat(c); err == nil && !info.IsDir() {
abs, _ := filepath.Abs(c)
return abs
}
}
return ""
}
+4 -75
View File
@@ -82,16 +82,12 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
}
}
// freePort asks the OS for a free TCP port in a range where the gRPC
// offset (port + 10000) won't collide with well-known ports.
// Stay below the Linux ephemeral floor (32768) so the kernel does not
// reuse the chosen port for an outbound connection between close() here
// and re-bind in the child "weed mini" process.
// freePort asks the OS for a free TCP port.
func freePort(t *testing.T) int {
t.Helper()
const (
minServicePort = 20000
maxServicePort = 32000
maxServicePort = 55535 // SeaweedFS gRPC service uses httpPort + 10000.
)
portCount := maxServicePort - minServicePort + 1
@@ -250,7 +246,6 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error {
"mini",
"-dir=" + f.dataDir,
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-filer.port=" + strconv.Itoa(f.filerPort),
"-s3=false",
"-webdav=false",
@@ -370,77 +365,11 @@ func findWeedBinary() string {
// Helper functions for test assertions
// AssertFileExists checks if a file exists in the mount point. On failure,
// it gathers diagnostic state (retry stat, parent listing, direct open) so
// transient FUSE flakes leave enough information to identify the layer that
// dropped the entry. The diagnostic block runs only on the failure path.
// AssertFileExists checks if a file exists in the mount point
func (f *FuseTestFramework) AssertFileExists(relativePath string) {
fullPath := filepath.Join(f.mountPoint, relativePath)
_, err := os.Stat(fullPath)
if err == nil {
return
}
f.dumpExistenceDiagnostics(relativePath, fullPath, err)
require.NoError(f.t, err, "file should exist: %s (see diagnostic logs above)", relativePath)
}
// dumpExistenceDiagnostics is invoked when AssertFileExists fails. It probes
// whether the missing entry is transient (retries appear), missing from the
// parent directory listing, or missing via a direct open syscall — three
// signals that triangulate where the entry was lost (kernel dentry cache vs
// mount lookup vs filer).
func (f *FuseTestFramework) dumpExistenceDiagnostics(relativePath, fullPath string, initialErr error) {
f.t.Logf("AssertFileExists diagnostic dump for %s", relativePath)
f.t.Logf(" initial stat: err=%v isNotExist=%v", initialErr, os.IsNotExist(initialErr))
// Retry stat to detect transient invisibility — if it becomes visible
// the issue is a short-lived dentry/cache window; if it stays missing
// the entry is genuinely gone from the filer or the local cache.
for attempt := 1; attempt <= 5; attempt++ {
time.Sleep(100 * time.Millisecond)
_, err := os.Stat(fullPath)
f.t.Logf(" retry #%d after %dms: err=%v", attempt, attempt*100, err)
if err == nil {
break
}
}
// List the parent directory: if the file appears here but stat fails,
// the failure is in the kernel's per-name dentry cache; if not, the
// mount-side LOOKUP itself is dropping the entry.
parentDir := filepath.Dir(fullPath)
target := filepath.Base(fullPath)
if entries, listErr := os.ReadDir(parentDir); listErr != nil {
f.t.Logf(" ReadDir(%s) failed: %v", parentDir, listErr)
} else {
found := false
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
if e.Name() == target {
found = true
}
}
f.t.Logf(" ReadDir(%s): %d entries, target %q present=%v", parentDir, len(entries), target, found)
if len(names) <= 32 {
f.t.Logf(" ReadDir entries: %v", names)
}
}
// Direct O_RDONLY open — exercises the same FUSE Lookup+Open path
// userspace would, but separately from stat, in case stat-only caching
// is interfering.
if fd, openErr := os.OpenFile(fullPath, os.O_RDONLY, 0); openErr != nil {
f.t.Logf(" Open(%s, O_RDONLY): %v", fullPath, openErr)
} else {
st, statErr := fd.Stat()
fd.Close()
size := int64(-1)
if statErr == nil && st != nil {
size = st.Size()
}
f.t.Logf(" Open(%s, O_RDONLY): success size=%d statErr=%v", fullPath, size, statErr)
}
require.NoError(f.t, err, "file should exist: %s", relativePath)
}
// AssertFileNotExists checks if a file does not exist in the mount point
+32 -64
View File
@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
@@ -81,25 +82,9 @@ func TestFcntlLockHelper(t *testing.T) {
start, _ := strconv.ParseInt(os.Getenv("LOCK_START"), 10, 64)
length, _ := strconv.ParseInt(os.Getenv("LOCK_LEN"), 10, 64)
// The parent test process created the file on the FUSE mount just before
// spawning this subprocess. A fresh lookup from a different process can
// briefly return ENOENT while the mount's dentry state propagates, so
// retry on ENOENT before giving up.
var f *os.File
var openErr error
for attempt := 0; attempt < 40; attempt++ {
f, openErr = os.OpenFile(filePath, os.O_RDWR, 0)
if openErr == nil {
break
}
if !errors.Is(openErr, os.ErrNotExist) && !errors.Is(openErr, syscall.ENOENT) {
break
}
_, _ = os.ReadDir(filepath.Dir(filePath))
time.Sleep(50 * time.Millisecond)
}
if openErr != nil {
fmt.Fprintf(os.Stderr, "open: %v\n", openErr)
f, err := os.OpenFile(filePath, os.O_RDWR, 0)
if err != nil {
fmt.Fprintf(os.Stderr, "open: %v\n", err)
os.Exit(1)
}
defer f.Close()
@@ -632,14 +617,9 @@ func testFcntlReleaseOnClose(t *testing.T, fw *FuseTestFramework) {
// reader goroutines (blocking FUSE SETLKW can starve unlock processing).
func testConcurrentLockContention(t *testing.T, fw *FuseTestFramework) {
path := filepath.Join(fw.GetMountPoint(), "lock_contention.txt")
// Hold a master fd open for the whole subtest. Workers take flock on their
// own fds; master stays passive. Keeping this reference pinned avoids the
// re-open-after-concurrent-close code path, which has shown an unresolved
// mount/metaCache coherence bug where a freshly-opened fd sees ENOENT for
// tens of seconds after a burst of concurrent Release calls.
master, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
require.NoError(t, err)
defer master.Close()
require.NoError(t, f.Close())
numWorkers := 8
writesPerWorker := 5
@@ -740,57 +720,45 @@ func testConcurrentLockContention(t *testing.T, fw *FuseTestFramework) {
}
wg.Wait()
if runtime.GOOS == "darwin" {
for _, err := range errs {
if err != nil && strings.Contains(err.Error(), "no such file or directory") {
t.Skip("lock contention file disappeared on darwin FUSE; skipping flaky check")
}
}
}
require.Empty(t, errs, "concurrent lock contention errors: %v", errs)
// Flush via the master fd. fsync on any fd to the file forces the
// kernel to send FUSE_FSYNC, which drains workers' writes to the filer.
require.NoError(t, master.Sync())
flush := func() {
verify, err := openWithRetry(os.O_RDWR)
if err != nil && runtime.GOOS == "darwin" && strings.Contains(err.Error(), "no such file or directory") {
t.Skip("lock contention file disappeared on darwin FUSE; skipping flaky check")
}
require.NoError(t, err)
defer verify.Close()
require.NoError(t, verify.Sync())
}
flush()
expectedLines := numWorkers * writesPerWorker
expectedBytes := expectedLines * recordSize
var data []byte
var lastReadErr error
ok := assert.Eventually(t, func() bool {
if _, err := master.Seek(0, io.SeekStart); err != nil {
lastReadErr = fmt.Errorf("seek: %w", err)
require.Eventually(t, func() bool {
verify, err := openWithRetry(os.O_RDONLY)
if err != nil {
if runtime.GOOS == "darwin" && strings.Contains(err.Error(), "no such file or directory") {
t.Skip("lock contention file disappeared on darwin FUSE; skipping flaky check")
}
return false
}
d, err := io.ReadAll(master)
defer verify.Close()
data, err = io.ReadAll(verify)
if err != nil {
lastReadErr = fmt.Errorf("read: %w", err)
return false
}
data = d
lastReadErr = nil
return len(data) == expectedBytes
}, 15*time.Second, 100*time.Millisecond, "file should eventually contain exactly %d records from all workers", expectedLines)
if !ok {
// Diagnostics: dump what the mount currently shows for the parent dir
// and the target path, so post-mortem analysis has something to go on.
parent := filepath.Dir(path)
if entries, readErr := os.ReadDir(parent); readErr != nil {
t.Logf("diag: ReadDir(%q) failed: %v", parent, readErr)
} else {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
t.Logf("diag: ReadDir(%q) returned %d entries: %v", parent, len(names), names)
}
if fi, statErr := os.Stat(path); statErr != nil {
t.Logf("diag: Stat(%q) failed: %v", path, statErr)
} else {
t.Logf("diag: Stat(%q) ok: size=%d mode=%s", path, fi.Size(), fi.Mode())
}
if freshFd, openErr := os.OpenFile(path, os.O_RDONLY, 0); openErr != nil {
t.Logf("diag: fresh open(%q, O_RDONLY) failed: %v", path, openErr)
} else {
fresh, readErr := io.ReadAll(freshFd)
freshFd.Close()
t.Logf("diag: fresh open(%q) read %d bytes (err=%v)", path, len(fresh), readErr)
}
t.Fatalf("read back incomplete: got %d bytes, want %d (lastReadErr=%v)", len(data), expectedBytes, lastReadErr)
}
actualLines := bytes.Count(data, []byte("\n"))
assert.Equal(t, expectedLines, actualLines,
"file should contain exactly %d lines from all workers", expectedLines)
@@ -1,314 +0,0 @@
package fuse_test
import (
"bytes"
"crypto/rand"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"runtime"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// mountDebugPort holds the debug/pprof port the test passed to the
// mount process via -debug.port. It is set once at TestWriteBufferCap
// entry and consulted from the write-timeout paths to fetch the mount
// process's goroutine dump (the test's own dumpAllGoroutines only
// covers the test process).
var mountDebugPort int
// fetchMountGoroutines pulls a full goroutine dump from the mount
// process's pprof endpoint. If the mount debug port isn't configured
// or the HTTP call fails, a short explanation is returned instead of
// an error — this is diagnostic best-effort, not a test assertion.
func fetchMountGoroutines() string {
if mountDebugPort == 0 {
return "(mount debug port not configured)"
}
url := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof/goroutine?debug=2", mountDebugPort)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Get(url)
if err != nil {
return fmt.Sprintf("(failed to reach mount pprof at %s: %v)", url, err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Sprintf("(failed to read mount pprof body: %v)", err)
}
return string(body)
}
// dumpAllGoroutines returns a full stack trace of every live goroutine.
// Used on write-timeout to give CI actionable diagnosis if the write
// buffer cap ever re-regresses into a hang.
func dumpAllGoroutines() string {
buf := make([]byte, 1<<20)
for {
n := runtime.Stack(buf, true)
if n < len(buf) {
return string(buf[:n])
}
buf = make([]byte, 2*len(buf))
}
}
// writeBufferCapConfig returns a TestConfig that exercises the new
// -writeBufferSizeMB flag. The cap is set below the aggregate in-flight
// write demand of the subtests below, so every new chunk has to pass
// through the Reserve/Release backpressure path at least some of the
// time. The cap is intentionally NOT minimal — over-tight settings
// interact with the per-file writable-chunk limit and the FUSE MaxWrite
// batching in ways that starve single-handle writers on slow CI.
//
// Also enables the mount's pprof debug endpoint so the test can fetch
// mount-process goroutine dumps on write-timeout, which is the only
// way to actually diagnose a backpressure deadlock (the test process
// itself is just blocked in syscall.Write waiting on FUSE).
func writeBufferCapConfig(debugPort int) *TestConfig {
return &TestConfig{
Collection: "",
Replication: "000",
ChunkSizeMB: 2, // 2 MiB chunks
CacheSizeMB: 100, // read cache (unrelated)
NumVolumes: 3,
EnableDebug: false,
MountOptions: []string{
// 16 MiB total write buffer ⇒ up to 8 chunks in flight
// across every open file handle on this mount. Large
// enough to avoid starving a single handle on a slow
// CI runner, small enough that the concurrent test
// below still has to drain through it.
"-writeBufferSizeMB=16",
"-debug=true",
fmt.Sprintf("-debug.port=%d", debugPort),
// Route glog to stderr so the framework's process log
// capture actually contains something — by default weed
// sends glog to /tmp/weed.* files which the CI artifact
// upload step never sees. Critical for diagnosing
// upload/saveToStorage errors on Linux runs.
"-logtostderr=true",
"-v=2",
},
SkipCleanup: false,
}
}
// writeWithTimeout wraps os.WriteFile with a hard deadline so a stuck
// write fails the test fast instead of consuming the full job budget.
// This is belt-and-braces around the 45-minute workflow timeout and
// makes write-buffer regressions surface as an actionable failure.
func writeWithTimeout(t *testing.T, path string, data []byte, timeout time.Duration) {
t.Helper()
done := make(chan error, 1)
go func() { done <- os.WriteFile(path, data, 0644) }()
select {
case err := <-done:
if err != nil {
// Dump mount goroutines on any write error, not just
// timeout — upload failures that surface via close()
// as EIO leave the mount process running but in an
// informative state (pending sealed chunks, error
// counters, etc).
t.Logf("write %s failed (%v) — dumping MOUNT goroutines:\n%s", path, err, fetchMountGoroutines())
}
require.NoError(t, err, "write %s", path)
case <-time.After(timeout):
t.Logf("write %s did not finish within %v — dumping TEST goroutines:\n%s", path, timeout, dumpAllGoroutines())
t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
t.Fatalf("write %s timed out — write buffer cap is likely leaking or deadlocking", path)
}
}
// runSubtestWithWatchdog runs body on the current (subtest main)
// goroutine and starts a watcher goroutine that logs diagnostics and
// fails the test if body doesn't return within timeout.
//
// body must run on the main goroutine because test helpers inside it
// (require.NoError, writeWithTimeout's own t.Fatalf on its internal
// timeout) need t.Fatal / t.FailNow, which Go's testing docs restrict
// to the goroutine running the test function. The watcher goroutine
// only calls goroutine-safe t methods (t.Log, t.Logf, t.Errorf) so it
// can mark the test failed and dump diagnostics without violating
// that contract. If body is stuck past timeout the watcher still
// surfaces the wedge (test + mount goroutine dumps + a FAIL mark);
// body itself gets unblocked either by its own inner writeWithTimeout
// firing t.Fatalf or by Go test's global -timeout.
func runSubtestWithWatchdog(t *testing.T, timeout time.Duration, body func(t *testing.T)) {
t.Helper()
stop := make(chan struct{})
defer close(stop)
go func() {
select {
case <-stop:
return
case <-time.After(timeout):
t.Logf("subtest exceeded %v watchdog — dumping TEST goroutines:\n%s", timeout, dumpAllGoroutines())
t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
t.Errorf("subtest exceeded %v watchdog — see goroutine dumps above", timeout)
}
}()
body(t)
}
// TestWriteBufferCap exercises the end-to-end write-buffer cap on a
// real FUSE mount. Without the cap, a volume-server stall would let
// the swap file grow without bound (issue #8777). With the cap, writers
// must serialize through a bounded budget while still producing correct
// output — that correctness (and the absence of deadlocks) is what
// this test verifies.
//
// Note: this test deliberately does not assert that Reserve *blocked*
// at some observed used-byte peak. The mount runs as a subprocess so
// its in-process WriteBufferAccountant state is not reachable from the
// test without adding a metrics/RPC surface to the mount binary. The
// deterministic peak-vs-cap assertion instead lives in the in-package
// unit test TestWriteBufferCap_SharedAcrossPipelines, which drives a
// controlled gated uploader and samples Used() throughout the run.
func TestWriteBufferCap(t *testing.T) {
mountDebugPort = freePort(t)
config := writeBufferCapConfig(mountDebugPort)
framework := NewFuseTestFramework(t, config)
defer framework.Cleanup()
require.NoError(t, framework.Setup(config))
const subtestTimeout = 3 * time.Minute
t.Run("ConcurrentWritesUnderCap", func(t *testing.T) {
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
testConcurrentWritesUnderCap(t, framework)
})
})
t.Run("LargeFileUnderCap", func(t *testing.T) {
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
testLargeFileUnderCap(t, framework)
})
})
t.Run("DoesNotDeadlockAfterPressure", func(t *testing.T) {
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
testWriteBufferNoDeadlockAfterPressure(t, framework)
})
})
}
// testConcurrentWritesUnderCap opens several files in parallel with
// aggregate demand that exceeds the 16 MiB write buffer cap, then
// verifies every byte survived the round trip.
func testConcurrentWritesUnderCap(t *testing.T, framework *FuseTestFramework) {
const (
numFiles = 4
fileSize = 8 * 1024 * 1024 // 8 MiB per file ⇒ 32 MiB total vs 16 MiB cap
)
dir := "write_buffer_cap_concurrent"
framework.CreateTestDir(dir)
payloads := make([][]byte, numFiles)
for i := range payloads {
buf := make([]byte, fileSize)
_, err := rand.Read(buf)
require.NoError(t, err)
payloads[i] = buf
}
start := time.Now()
var wg sync.WaitGroup
errs := make(chan error, numFiles)
timedOut := make(chan struct{}, numFiles)
for i := 0; i < numFiles; i++ {
i := i
wg.Add(1)
go func() {
defer wg.Done()
name := fmt.Sprintf("file_%02d.bin", i)
path := filepath.Join(framework.GetMountPoint(), dir, name)
done := make(chan error, 1)
go func() { done <- os.WriteFile(path, payloads[i], 0644) }()
select {
case err := <-done:
if err != nil {
errs <- fmt.Errorf("writer %d: %w", i, err)
}
case <-time.After(90 * time.Second):
timedOut <- struct{}{}
errs <- fmt.Errorf("writer %d: timed out after 90s", i)
}
}()
}
wg.Wait()
close(errs)
// If any writer timed out, dump every live goroutine so CI shows the
// wedge instead of just a walltime.
select {
case <-timedOut:
t.Logf("at least one concurrent writer timed out — dumping goroutines:\n%s", dumpAllGoroutines())
default:
}
for err := range errs {
t.Fatal(err)
}
t.Logf("wrote %d × %d MiB under 16 MiB cap in %v", numFiles, fileSize/(1024*1024), time.Since(start))
for i := 0; i < numFiles; i++ {
name := fmt.Sprintf("file_%02d.bin", i)
path := filepath.Join(framework.GetMountPoint(), dir, name)
got, err := os.ReadFile(path)
require.NoError(t, err, "read %s", name)
require.Equal(t, len(payloads[i]), len(got), "size mismatch for %s", name)
if !bytes.Equal(payloads[i], got) {
t.Fatalf("content mismatch for %s", name)
}
}
}
// testLargeFileUnderCap writes a single file whose size exceeds the
// 16 MiB cap through a single handle, verifying that the pipeline
// drains its own earlier chunks and makes forward progress rather than
// self-deadlocking when the global budget is already full of its own
// earlier sealed chunks.
func testLargeFileUnderCap(t *testing.T, framework *FuseTestFramework) {
const fileSize = 20 * 1024 * 1024 // 20 MiB ⇒ 10 chunks vs 8-slot budget
payload := make([]byte, fileSize)
_, err := rand.Read(payload)
require.NoError(t, err)
name := "write_buffer_cap_large.bin"
path := filepath.Join(framework.GetMountPoint(), name)
start := time.Now()
writeWithTimeout(t, path, payload, 90*time.Second)
t.Logf("wrote %d MiB through one handle under 16 MiB cap in %v", fileSize/(1024*1024), time.Since(start))
got, err := os.ReadFile(path)
require.NoError(t, err)
require.Equal(t, len(payload), len(got))
if !bytes.Equal(payload, got) {
t.Fatal("content mismatch on large single-handle write")
}
}
// testWriteBufferNoDeadlockAfterPressure verifies the mount is still
// healthy after being driven against the cap. A budget-slot leak would
// eventually cause every new chunk allocation to hang; a quick canary
// write catches that as a hard failure.
func testWriteBufferNoDeadlockAfterPressure(t *testing.T, framework *FuseTestFramework) {
name := "write_buffer_cap_canary.txt"
path := filepath.Join(framework.GetMountPoint(), name)
content := []byte("write buffer cap canary — mount still healthy")
writeWithTimeout(t, path, content, 30*time.Second)
got, err := os.ReadFile(path)
require.NoError(t, err)
require.Equal(t, content, got)
}
+4 -40
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"sync"
"sync/atomic"
"testing"
@@ -792,37 +791,18 @@ func TestWritebackCacheStressSmallFiles(t *testing.T) {
wg.Wait()
writeDuration := time.Since(start)
// Stream to stderr so the line is visible even if the test hangs and the
// 45m suite alarm panics before t.Logf flushes its buffer.
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: wrote %d files in %v (%.0f files/sec)\n",
totalFiles, writeDuration, float64(totalFiles)/writeDuration.Seconds())
t.Logf("wrote %d files in %v (%.0f files/sec)",
totalFiles, writeDuration, float64(totalFiles)/writeDuration.Seconds())
require.Empty(t, writeErrors, "write errors: %v", writeErrors)
assert.Equal(t, totalFiles, len(records))
// Verify all files are eventually readable with correct content.
// A whole-phase budget bounds total verify time so a stuck mount fails
// with diagnostics instead of being swallowed by the 45m suite alarm.
const verifyPhaseBudget = 10 * time.Minute
const perFileDeadline = 120 * time.Second
verifyStart := time.Now()
verifyPhaseDeadline := verifyStart.Add(verifyPhaseBudget)
// Verify all files are eventually readable with correct content
var verifyErrors []error
for i, rec := range records {
if time.Now().After(verifyPhaseDeadline) {
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: verify-phase budget %v exceeded at file %d/%d (%s); goroutine dump:\n%s\n",
verifyPhaseBudget, i, len(records), rec.path, buf[:n])
t.Fatalf("verify phase exceeded %v at file %d/%d (%s); accumulated errors: %v",
verifyPhaseBudget, i, len(records), rec.path, verifyErrors)
}
fileDeadline := time.Now().Add(perFileDeadline)
for _, rec := range records {
deadline := time.Now().Add(120 * time.Second)
var lastErr error
retries := 0
for time.Now().Before(fileDeadline) {
for time.Now().Before(deadline) {
actual, err := os.ReadFile(rec.path)
if err == nil && bytes.Equal(rec.content, actual) {
lastErr = nil
@@ -833,29 +813,13 @@ func TestWritebackCacheStressSmallFiles(t *testing.T) {
} else {
lastErr = fmt.Errorf("content mismatch for %s: got %d bytes, want %d", rec.path, len(actual), len(rec.content))
}
retries++
// Surface the in-flight error every ~5s of polling so a stuck file
// is visible in real time, not only after the test exits.
if retries%10 == 0 {
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: file %d/%d %s still failing after %d retries: %v\n",
i, len(records), rec.path, retries, lastErr)
}
time.Sleep(500 * time.Millisecond)
}
if lastErr != nil {
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: file %d/%d %s gave up after %v: %v\n",
i, len(records), rec.path, perFileDeadline, lastErr)
verifyErrors = append(verifyErrors, lastErr)
}
// Periodic progress so the test isn't silent during the verify phase.
if (i+1)%100 == 0 {
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: verified %d/%d (elapsed %v, %d errors)\n",
i+1, len(records), time.Since(verifyStart), len(verifyErrors))
}
}
require.Empty(t, verifyErrors, "verification errors after stress test: %v", verifyErrors)
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: all %d files verified in %v\n",
totalFiles, time.Since(verifyStart))
t.Logf("all %d files verified successfully", totalFiles)
}
-388
View File
@@ -1,388 +0,0 @@
//go:build linux || darwin
package fuse_p2p
import (
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"strconv"
"sync"
"syscall"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/stretchr/testify/require"
)
// p2pTestCluster manages a minimal SeaweedFS cluster exercising the peer
// chunk-sharing (p2p) read path: 1 master, 1 volume, 1 filer, and N FUSE
// mounts that all have -peer.enable set. Three mounts is the sweet spot
// for integration testing — with 2 mounts the HRW owner of a chunk is
// ~50% the reader itself (which short-circuits the peer path); with 3+
// mounts it's ≤ 1/3, so a multi-chunk file almost certainly exercises
// the remote-owner fan-out.
type p2pTestCluster struct {
t testing.TB
baseDir string
weedBinary string
masterPort int
masterGrpcPort int
volumePort int
volumeGrpcPort int
filerPort int
filerGrpcPort int
mountPeerPorts []int
mountPoints []string
masterCmd *exec.Cmd
volumeCmd *exec.Cmd
filerCmd *exec.Cmd
mountCmds []*exec.Cmd
logFiles []*os.File
cleanupOnce sync.Once
}
// startP2PTestCluster brings up a cluster with numMounts FUSE mounts,
// every one advertising on its own -peer.listen port. All mounts register
// with the same filer. Verbose logging (-v=4) is enabled on each mount so
// peer-read success/failure messages land in the log file and the test
// can grep them to verify the p2p path fired.
func startP2PTestCluster(t testing.TB, numMounts int) *p2pTestCluster {
require.GreaterOrEqual(t, numMounts, 2, "need at least 2 mounts to exercise p2p")
binary := findWeedBinary()
if binary == "" {
t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
}
baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_p2p_test_")
require.NoError(t, err)
c := &p2pTestCluster{
t: t,
baseDir: baseDir,
weedBinary: binary,
mountPeerPorts: make([]int, numMounts),
mountPoints: make([]string, numMounts),
mountCmds: make([]*exec.Cmd, numMounts),
}
t.Cleanup(c.Stop)
// master(2) + volume(2) + filer(2) + one peer port per mount.
// testutil.AllocatePorts holds all listeners open until every port
// is reserved, avoiding the brief close→bind race that would
// happen with a per-listener close loop.
ports, err := testutil.AllocatePorts(6 + numMounts)
require.NoError(t, err)
c.masterPort = ports[0]
c.masterGrpcPort = ports[1]
c.volumePort = ports[2]
c.volumeGrpcPort = ports[3]
c.filerPort = ports[4]
c.filerGrpcPort = ports[5]
for i := 0; i < numMounts; i++ {
c.mountPeerPorts[i] = ports[6+i]
}
configDir := filepath.Join(baseDir, "config")
require.NoError(t, os.MkdirAll(configDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
require.NoError(t, c.startMaster(configDir))
require.NoError(t, c.waitForTCP(c.masterCmd, "master",
fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second))
require.NoError(t, c.startVolume(configDir))
require.NoError(t, c.waitForTCP(c.volumeCmd, "volume",
fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second))
require.NoError(t, c.startFiler(configDir))
require.NoError(t, c.waitForTCP(c.filerCmd, "filer",
fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPort), 30*time.Second))
for i := 0; i < numMounts; i++ {
mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
require.NoError(t, os.MkdirAll(mp, 0755))
c.mountPoints[i] = mp
require.NoError(t, c.startMount(i, configDir))
require.NoError(t, c.waitForMount(mp, 30*time.Second),
"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
}
return c
}
func (c *p2pTestCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
for i := len(c.mountCmds) - 1; i >= 0; i-- {
stopCmd(c.mountCmds[i])
// Backup unmount in case the FUSE teardown didn't clean up.
exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
}
stopCmd(c.filerCmd)
stopCmd(c.volumeCmd)
stopCmd(c.masterCmd)
for _, f := range c.logFiles {
f.Close()
}
c.copyLogsForCI()
if !c.t.Failed() {
os.RemoveAll(c.baseDir)
}
time.Sleep(2 * time.Second) // let ports drain
})
}
// MountDir returns the filesystem path of the i-th mount.
func (c *p2pTestCluster) MountDir(i int) string { return c.mountPoints[i] }
// masterAddress / filerAddress return SeaweedFS-style addresses encoding
// both ports as "host:httpPort.grpcPort". Without this, downstream
// components fall back to the grpcPort = httpPort + 10000 default,
// which doesn't match the random port we allocate.
func (c *p2pTestCluster) masterAddress() string {
return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
}
func (c *p2pTestCluster) filerAddress() string {
return string(pb.NewServerAddress("127.0.0.1", c.filerPort, c.filerGrpcPort))
}
// MountLog returns the contents of mount i's log file.
func (c *p2pTestCluster) MountLog(i int) string {
return c.tailLogFull(fmt.Sprintf("mount%d", i))
}
func (c *p2pTestCluster) startMaster(configDir string) error {
c.masterCmd = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"master",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.masterPort),
"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
"-mdir="+filepath.Join(c.baseDir, "master"),
)
return c.startCmd(c.masterCmd, "master")
}
func (c *p2pTestCluster) startVolume(configDir string) error {
volDir := filepath.Join(c.baseDir, "volume")
if err := os.MkdirAll(volDir, 0755); err != nil {
return fmt.Errorf("create volume dir: %w", err)
}
c.volumeCmd = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"volume",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.volumePort),
"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
"-master="+c.masterAddress(),
"-dir="+volDir,
"-max=10",
)
return c.startCmd(c.volumeCmd, "volume")
}
func (c *p2pTestCluster) startFiler(configDir string) error {
filerDir := filepath.Join(c.baseDir, "filer")
if err := os.MkdirAll(filerDir, 0755); err != nil {
return fmt.Errorf("create filer dir: %w", err)
}
c.filerCmd = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"filer",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port="+strconv.Itoa(c.filerPort),
"-port.grpc="+strconv.Itoa(c.filerGrpcPort),
"-master="+c.masterAddress(),
"-defaultStoreDir="+filerDir,
)
return c.startCmd(c.filerCmd, "filer")
}
func (c *p2pTestCluster) startMount(idx int, configDir string) error {
cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return fmt.Errorf("create cache dir: %w", err)
}
c.mountCmds[idx] = exec.Command(c.weedBinary,
"-logdir="+filepath.Join(c.baseDir, "logs"),
"-v=4",
"mount",
"-filer="+c.filerAddress(),
"-dir="+c.mountPoints[idx],
"-filer.path=/",
"-dirAutoCreate",
"-allowOthers=false",
"-cacheDir="+cacheDir,
"-peer.enable=true",
fmt.Sprintf("-peer.listen=127.0.0.1:%d", c.mountPeerPorts[idx]),
fmt.Sprintf("-peer.advertise=127.0.0.1:%d", c.mountPeerPorts[idx]),
"-peer.dataCenter=dc1",
fmt.Sprintf("-peer.rack=rack%d", idx),
)
return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
}
func (c *p2pTestCluster) startCmd(cmd *exec.Cmd, name string) error {
logPath := filepath.Join(c.baseDir, "logs")
if err := os.MkdirAll(logPath, 0755); err != nil {
return fmt.Errorf("create log dir: %w", err)
}
logFile, err := os.Create(filepath.Join(logPath, name+".log"))
if err != nil {
return err
}
c.logFiles = append(c.logFiles, logFile)
cmd.Stdout = logFile
cmd.Stderr = logFile
return cmd.Start()
}
func (c *p2pTestCluster) tailLog(name string) string {
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
if err != nil {
return fmt.Sprintf("(log %s not available: %v)", name, err)
}
const maxTail = 8192
if len(data) > maxTail {
data = data[len(data)-maxTail:]
}
return string(data)
}
func (c *p2pTestCluster) tailLogFull(name string) string {
data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
if err != nil {
return ""
}
return string(data)
}
func (c *p2pTestCluster) copyLogsForCI() {
ciLogDir := "/tmp/seaweedfs-fuse-p2p-logs"
os.MkdirAll(ciLogDir, 0755)
logsDir := filepath.Join(c.baseDir, "logs")
entries, err := os.ReadDir(logsDir)
if err != nil {
return
}
for _, e := range entries {
data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
if err != nil {
continue
}
os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
}
}
// waitForTCP polls addr until it accepts a connection, OR the supplied
// subprocess exits — whichever comes first. Short-circuiting on child
// exit turns a 30 s-spin-on-dead-process into an immediate failure with
// the tail of its log. cmd may be nil for callers that don't track a
// process; in that case we fall back to pure polling.
//
// Liveness is checked with signal 0 (POSIX "is this process alive").
// We deliberately do NOT call cmd.Wait() in a goroutine here because
// stopCmd() later calls Wait() at shutdown, and only one Wait per
// process is allowed.
func (c *p2pTestCluster) waitForTCP(cmd *exec.Cmd, name, addr string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", addr, time.Second)
if err == nil {
conn.Close()
return nil
}
if cmd != nil && cmd.Process != nil {
// signal 0 doesn't actually send anything; it just asks
// the kernel whether the process exists. An error here
// (typically "process already finished" or ESRCH) means
// the child died before coming up.
if sigErr := cmd.Process.Signal(syscall.Signal(0)); sigErr != nil {
return fmt.Errorf("%s exited before listening on %s: %v\n%s",
name, addr, sigErr, c.tailLog(name))
}
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("service at %s not ready within timeout\n%s", addr, c.tailLog(name))
}
// waitForMount waits for a FUSE filesystem to actually be mounted by
// watching the device id flip (FUSE mounts have a different Dev than
// their parent dir).
func (c *p2pTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
parentDir := filepath.Dir(mountPoint)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
parentStat, err := os.Stat(parentDir)
if err != nil {
time.Sleep(200 * time.Millisecond)
continue
}
mountStat, err := os.Stat(mountPoint)
if err != nil {
time.Sleep(200 * time.Millisecond)
continue
}
parentSys := parentStat.Sys().(*syscall.Stat_t)
mountSys := mountStat.Sys().(*syscall.Stat_t)
if parentSys.Dev != mountSys.Dev {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
}
// --- utilities (binary discovery, process shutdown) ---
//
// Port allocation is delegated to testutil.MustAllocatePorts, which
// holds all listeners open until every port is reserved before
// returning them in a batch — safer than the per-listener
// close-then-reserve pattern fuse_dlm originally used.
func findWeedBinary() string {
if env := os.Getenv("WEED_BINARY"); env != "" {
if _, err := os.Stat(env); err == nil {
return env
}
}
if p, err := exec.LookPath("weed"); err == nil {
return p
}
return ""
}
// stopCmd kills a child process with SIGTERM and waits up to 5 s, then
// escalates to SIGKILL. Tolerates nil and already-exited processes.
func stopCmd(cmd *exec.Cmd) {
if cmd == nil || cmd.Process == nil {
return
}
cmd.Process.Signal(syscall.SIGTERM)
done := make(chan struct{})
go func() {
cmd.Wait()
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
cmd.Process.Signal(syscall.SIGKILL)
<-done
}
}

Some files were not shown because too many files have changed in this diff Show More