mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-06 22:55:51 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Chris Lu
|
08dbfce493 | ||
|
Chris Lu
|
d4261c2271 |
@@ -0,0 +1 @@
|
||||
{"sessionId":"d6574c47-eafc-4a94-9dce-f9ffea22b53c","pid":10111,"acquiredAt":1775248373916}
|
||||
@@ -4,10 +4,6 @@ on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: binaries-dev-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -22,7 +18,6 @@ jobs:
|
||||
|
||||
- name: Delete old release assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
name: "docker: build latest container"
|
||||
|
||||
# Manual fallback only. On tag push, container_release_unified.yml already
|
||||
# re-tags the released versioned image as `latest` / `latest_large_disk`,
|
||||
# so a full rebuild here is unnecessary. Run this manually if you need to
|
||||
# rebuild `latest` from an arbitrary ref.
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_ref:
|
||||
@@ -338,7 +337,7 @@ jobs:
|
||||
- name: Trivy report (published image)
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan amd64 only - OS packages are identical across architectures
|
||||
@@ -355,7 +354,7 @@ jobs:
|
||||
exit-code: '0'
|
||||
- name: Trivy report (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
@@ -375,7 +374,7 @@ jobs:
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
# Gate only on fixable high/critical vulnerabilities. Non-fixable
|
||||
# findings are still visible in the SARIF upload above.
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
scan-type: image
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
|
||||
@@ -388,7 +387,7 @@ jobs:
|
||||
skip-setup-trivy: true
|
||||
- name: Trivy gate (local tarball)
|
||||
if: needs.setup.outputs.publish != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
|
||||
with:
|
||||
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
|
||||
scanners: vuln
|
||||
|
||||
@@ -4,19 +4,11 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: 'Release tag to publish (e.g. 3.93)'
|
||||
required: true
|
||||
default: ''
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
|
||||
jobs:
|
||||
|
||||
build-large-release-container_foundationdb:
|
||||
@@ -34,7 +26,7 @@ jobs:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
tags: |
|
||||
type=raw,value=${{ env.RELEASE_TAG }}_large_disk_foundationdb
|
||||
type=ref,event=tag,suffix=_large_disk_foundationdb
|
||||
flavor: |
|
||||
latest=false
|
||||
labels: |
|
||||
|
||||
@@ -29,7 +29,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.release_tag || github.ref_name }}
|
||||
@@ -358,128 +357,7 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }} \
|
||||
chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.tag_suffix }}
|
||||
|
||||
echo "Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
# Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub
|
||||
# Security for visibility, but never blocks the release. Releases (including
|
||||
# `latest`) ship regardless — vulnerabilities are tracked, not gated, since
|
||||
# we sometimes need to publish through known findings (e.g. unfixed upstream
|
||||
# CVE, base-image lag).
|
||||
trivy-scan:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name == 'push'
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
variant: normal
|
||||
- source_suffix: _large_disk
|
||||
variant: large_disk
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Trivy report (${{ matrix.variant }})
|
||||
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
scan-type: image
|
||||
# Scan the multi-arch tag on GHCR (already pushed by the build job).
|
||||
# Trivy scans the runner's native platform; OS packages are identical
|
||||
# across architectures since they all share the same alpine base.
|
||||
image-ref: ghcr.io/chrislusf/seaweedfs:${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
limit-severities-for-sarif: true
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
# Point `latest` (and `latest_large_disk`) at the just-released versioned
|
||||
# image. crane tag adds an extra tag to an existing manifest — no rebuild,
|
||||
# no QEMU, no separate workflow. Replaces the old container_latest.yml
|
||||
# rebuild that often failed or lagged behind the release. Independent of
|
||||
# trivy-scan: vuln findings are reported but do not block `latest`.
|
||||
tag-latest:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [copy-to-dockerhub]
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- source_suffix: ""
|
||||
latest_tag: latest
|
||||
- source_suffix: _large_disk
|
||||
latest_tag: latest_large_disk
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
|
||||
- name: Install crane
|
||||
run: |
|
||||
cd $(mktemp -d)
|
||||
curl -sL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" | tar xz
|
||||
sudo mv crane /usr/local/bin/
|
||||
crane version
|
||||
|
||||
- name: Re-tag ${{ env.RELEASE_TAG }}${{ matrix.source_suffix }} as ${{ matrix.latest_tag }}
|
||||
run: |
|
||||
retry_with_backoff() {
|
||||
local max_attempts=5
|
||||
local timeout=1
|
||||
local attempt=1
|
||||
local exit_code=0
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
else
|
||||
exit_code=$?
|
||||
fi
|
||||
if [ $attempt -lt $max_attempts ]; then
|
||||
echo "Attempt $attempt failed. Retrying in ${timeout}s..." >&2
|
||||
sleep $timeout
|
||||
timeout=$((timeout * 2))
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
echo "Command failed after $max_attempts attempts" >&2
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
SRC_TAG="${{ env.RELEASE_TAG }}${{ matrix.source_suffix }}"
|
||||
DST_TAG="${{ matrix.latest_tag }}"
|
||||
|
||||
echo "Tagging ghcr.io/chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "ghcr.io/chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
|
||||
echo "Tagging chrislusf/seaweedfs:${SRC_TAG} as ${DST_TAG}"
|
||||
retry_with_backoff crane tag "chrislusf/seaweedfs:${SRC_TAG}" "${DST_TAG}"
|
||||
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v2
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
|
||||
|
||||
@@ -57,12 +57,12 @@ jobs:
|
||||
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
# Enable Docker buildkit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
|
||||
# Build with retry logic
|
||||
for i in {1..3}; do
|
||||
echo "Build attempt $i/3"
|
||||
@@ -77,18 +77,10 @@ jobs:
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Start services with wait
|
||||
docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Rotate buildx cache
|
||||
if: always()
|
||||
run: |
|
||||
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
|
||||
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
|
||||
rm -rf /tmp/.buildx-cache
|
||||
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
|
||||
|
||||
- name: Run FIO 4k
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
name: EC Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
|
||||
jobs:
|
||||
ec-integration-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o ../weed_bin
|
||||
|
||||
- name: Run EC integration tests
|
||||
run: |
|
||||
cd test/erasure_coding/admin_dockertest
|
||||
go test -v -timeout 15m ec_integration_test.go
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ec-test-logs
|
||||
path: test/erasure_coding/admin_dockertest/tmp/logs/
|
||||
retention-days: 7
|
||||
@@ -1,63 +0,0 @@
|
||||
name: "FUSE DLM Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-dlm-integration:
|
||||
name: FUSE DLM Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run DLM integration tests
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=20m ./test/fuse_dlm/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-dlm-test-logs
|
||||
path: /tmp/seaweedfs-fuse-dlm-logs/
|
||||
retention-days: 3
|
||||
@@ -1,69 +0,0 @@
|
||||
name: "FUSE P2P Peer Chunk Sharing Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/mount_peer_registry*.go'
|
||||
- 'weed/server/filer_grpc_server_mount_peer.go'
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
fuse-p2p-integration:
|
||||
name: FUSE P2P Peer Chunk Sharing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libfuse3-dev
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
- name: Run P2P integration tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: go test -v -count=1 -timeout=12m ./test/fuse_p2p/...
|
||||
|
||||
- name: Upload logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-p2p-test-logs
|
||||
path: /tmp/seaweedfs-fuse-p2p-logs/
|
||||
retention-days: 3
|
||||
@@ -68,135 +68,6 @@ jobs:
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "✓ Security configuration renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing IAM gRPC opt-in path ==="
|
||||
# Regression test: the filer registers the IAM gRPC service the
|
||||
# Admin UI Users tab calls only when jwt.filer_signing.key is in
|
||||
# security.toml. Operators must be able to enable that without
|
||||
# the cert-manager mTLS bundle.
|
||||
# Install PyYAML explicitly: this block runs before the later
|
||||
# security+S3 block that does the same install, and we don't
|
||||
# want to rely on the runner image shipping it.
|
||||
pip install pyyaml -q
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import subprocess, sys, yaml
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def docs(manifest):
|
||||
return [d for d in yaml.safe_load_all(manifest) if d]
|
||||
|
||||
def configmap(manifest, name):
|
||||
for d in docs(manifest):
|
||||
if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
|
||||
return d
|
||||
return None
|
||||
|
||||
def workload_mounts(manifest, name):
|
||||
for d in docs(manifest):
|
||||
if d.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
if d["metadata"]["name"] != name:
|
||||
continue
|
||||
pod = d["spec"]["template"]["spec"]
|
||||
vols = {v["name"] for v in pod.get("volumes", [])}
|
||||
mounts = set()
|
||||
for c in pod.get("containers", []):
|
||||
for vm in c.get("volumeMounts", []):
|
||||
mounts.add(vm["name"])
|
||||
return vols, mounts
|
||||
return None, None
|
||||
|
||||
failed = []
|
||||
|
||||
# Case 1: defaults. The chart historically rendered nothing
|
||||
# security-related; preserve that so this PR is non-breaking on
|
||||
# existing installs.
|
||||
out = render({})
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append("defaults: security ConfigMap should not render")
|
||||
else:
|
||||
print("✓ defaults: no security-config ConfigMap (unchanged)")
|
||||
|
||||
# Case 2: filerWrite=true alone is the documented opt-in for
|
||||
# the Admin UI Users tab. Configmap must render with
|
||||
# [jwt.filer_signing] and NO [grpc.*] sections (cert paths
|
||||
# only exist with mTLS).
|
||||
out = render({
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true",
|
||||
"admin.enabled": "true",
|
||||
})
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("filerWrite=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["data"]["security.toml"]
|
||||
if "[jwt.filer_signing]" not in toml:
|
||||
failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
|
||||
if "[grpc" in toml:
|
||||
failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
|
||||
if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
|
||||
print("✓ filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
|
||||
|
||||
# Case 3: filer + admin pods must MOUNT the security ConfigMap
|
||||
# under filerWrite=true so the JWT key reaches both processes.
|
||||
# Cert volumes must NOT be present (no mTLS).
|
||||
for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
|
||||
vols, mounts = workload_mounts(out, wl)
|
||||
if vols is None:
|
||||
failed.append(f"filerWrite=true: workload {wl} not found")
|
||||
continue
|
||||
if "security-config" not in vols or "security-config" not in mounts:
|
||||
failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
|
||||
else:
|
||||
print(f"✓ filerWrite=true: {wl} mounts security-config")
|
||||
cert_vols = {v for v in vols if v.endswith("-cert")}
|
||||
if cert_vols:
|
||||
failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")
|
||||
|
||||
# Case 4: enableSecurity=true must still render the full toml
|
||||
# with both [jwt.signing] and [grpc.*]. Guards against the
|
||||
# decoupling change accidentally regressing the mTLS path.
|
||||
out = render({"global.seaweedfs.enableSecurity": "true"})
|
||||
cm = configmap(out, "test-seaweedfs-security-config")
|
||||
if cm is None:
|
||||
failed.append("enableSecurity=true: security ConfigMap missing")
|
||||
else:
|
||||
toml = cm["data"]["security.toml"]
|
||||
missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
|
||||
if missing:
|
||||
failed.append(f"enableSecurity=true: security.toml missing {missing}")
|
||||
else:
|
||||
print("✓ enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
|
||||
|
||||
# Case 5: helper must tolerate explicit nulls (gemini-code-assist
|
||||
# PR review). securityConfig=null was the parens-pattern crash
|
||||
# the helper review caught.
|
||||
for null_path in ("global.seaweedfs.securityConfig",
|
||||
"global.seaweedfs.securityConfig.jwtSigning"):
|
||||
try:
|
||||
out = render({null_path: "null"})
|
||||
except subprocess.CalledProcessError as e:
|
||||
failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
|
||||
continue
|
||||
if configmap(out, "test-seaweedfs-security-config") is not None:
|
||||
failed.append(f"{null_path}=null: should not render configmap")
|
||||
else:
|
||||
print(f"✓ {null_path}=null: render tolerates explicit null")
|
||||
|
||||
if failed:
|
||||
print("\nFAIL:", file=sys.stderr)
|
||||
for f in failed:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
echo "✓ IAM gRPC decoupling tests passed"
|
||||
|
||||
echo "=== Testing with monitoring enabled ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.monitoring.enabled=true \
|
||||
@@ -346,106 +217,6 @@ jobs:
|
||||
print("✓ No blank lines in security+S3 command blocks")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
|
||||
# Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
|
||||
# the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
|
||||
# them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
|
||||
# makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
|
||||
# HTTP request to an HTTPS server" into the pod log.
|
||||
#
|
||||
# When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
|
||||
# together so the opt-in HTTPS listener actually has credentials.
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import subprocess, sys, yaml
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def script_of(manifest, kind_name):
|
||||
for doc in yaml.safe_load_all(manifest):
|
||||
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
|
||||
continue
|
||||
if doc["metadata"]["name"] != kind_name:
|
||||
continue
|
||||
for c in doc["spec"]["template"]["spec"]["containers"]:
|
||||
cmd = c.get("command", [])
|
||||
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
|
||||
return cmd[2]
|
||||
raise AssertionError(f"no container script for {kind_name}")
|
||||
|
||||
cases = [
|
||||
# (values, workload-name, httpsPort-set?, arg-prefix)
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true"},
|
||||
"test-seaweedfs-s3", False, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"s3.enabled": "true",
|
||||
"s3.httpsPort": "8443"},
|
||||
"test-seaweedfs-s3", True, ""),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true"},
|
||||
"test-seaweedfs-filer", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"filer.s3.enabled": "true",
|
||||
"filer.s3.httpsPort": "8444"},
|
||||
"test-seaweedfs-filer", True, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true"},
|
||||
"test-seaweedfs-all-in-one", False, "s3."),
|
||||
({"global.seaweedfs.enableSecurity": "true",
|
||||
"allInOne.enabled": "true",
|
||||
"allInOne.s3.enabled": "true",
|
||||
"allInOne.s3.httpsPort": "8445"},
|
||||
"test-seaweedfs-all-in-one", True, "s3."),
|
||||
]
|
||||
|
||||
failed = False
|
||||
for values, name, https_on, prefix in cases:
|
||||
script = script_of(render(values), name)
|
||||
cert_flag = f"-{prefix}cert.file="
|
||||
key_flag = f"-{prefix}key.file="
|
||||
https_flag = f"-{prefix}port.https="
|
||||
has_cert = cert_flag in script
|
||||
has_key = key_flag in script
|
||||
has_https = https_flag in script
|
||||
label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
|
||||
if https_on:
|
||||
if not (has_cert and has_key and has_https):
|
||||
print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: cert/key/https args emitted together")
|
||||
else:
|
||||
if has_cert or has_key or has_https:
|
||||
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
|
||||
f"main S3 -port would silently become HTTPS and break HTTP probes "
|
||||
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
|
||||
failed = True
|
||||
else:
|
||||
print(f"✓ {label}: no TLS args emitted, main -port stays HTTP")
|
||||
|
||||
# bash -n: pin down that the rendered script parses. Guards against
|
||||
# a future helper change that leaves a dangling `\` with nothing
|
||||
# after it (every current caller already exits cleanly because
|
||||
# bash treats trailing `\<newline><EOF>` as line-continuation to
|
||||
# an empty line — but keep the contract explicit).
|
||||
parse = subprocess.run(["bash", "-n"], input=script, text=True,
|
||||
capture_output=True)
|
||||
if parse.returncode != 0:
|
||||
print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
|
||||
file=sys.stderr)
|
||||
failed = True
|
||||
|
||||
sys.exit(1 if failed else 0)
|
||||
PYEOF
|
||||
|
||||
echo "✅ All template rendering tests passed!"
|
||||
|
||||
- name: Create kind cluster
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
name: "helm: release"
|
||||
name: "helm: manual release"
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -445,7 +445,7 @@ jobs:
|
||||
# Test consumer group functionality with explicit timeout
|
||||
ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
|
||||
ulimit -u 100 || echo "Warning: Could not set process limit"
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/...
|
||||
timeout 240s go test -v -run "^TestConsumerGroups" -timeout 180s ./integration/... || echo "Test execution timed out or failed"
|
||||
env:
|
||||
GOMAXPROCS: 1
|
||||
SEAWEEDFS_MASTERS: 127.0.0.1:9333
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
name: "Multi-Master Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/server/master_*.go'
|
||||
- 'weed/server/raft_*.go'
|
||||
- 'weed/topology/**'
|
||||
- 'test/multi_master/**'
|
||||
- 'test/testutil/**'
|
||||
- '.github/workflows/multi-master-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/multi-master-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
multi-master-failover-tests:
|
||||
name: Multi-Master Failover Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Run multi-master failover tests
|
||||
# The tests in test/multi_master spin up their own 3-node master raft
|
||||
# cluster (using the freshly-installed `weed` binary) and exercise
|
||||
# leader-election, failover and recovery scenarios. The shared
|
||||
# test/testutil port-allocator regression test runs alongside since it
|
||||
# is a prerequisite for the cluster fixtures.
|
||||
run: |
|
||||
go test -v -timeout=8m ./test/multi_master/... ./test/testutil/...
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
# test/multi_master/cluster.go creates per-test dirs via
|
||||
# os.MkdirTemp("", "seaweedfs_multi_master_it_") and writes each
|
||||
# node's log into <baseDir>/logs/master*.log.
|
||||
echo "Collecting per-node master logs from temp directories..."
|
||||
mkdir -p /tmp/multi-master-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_multi_master_it_*" 2>/dev/null | while read dir; do
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/multi-master-logs/ 2>/dev/null || true
|
||||
done
|
||||
find /tmp/multi-master-logs -type f -name "*.log" -print -exec tail -n 100 {} \; 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: multi-master-test-logs
|
||||
path: /tmp/multi-master-logs/
|
||||
retention-days: 7
|
||||
@@ -1,137 +0,0 @@
|
||||
name: "NFS Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/nfs/**'
|
||||
- 'weed/command/nfs.go'
|
||||
- 'weed/filer/filer_inode.go'
|
||||
- 'weed/filer/filer_inode_index.go'
|
||||
- 'weed/filer/filerstore_wrapper.go'
|
||||
- 'weed/server/filer_grpc_server_rename.go'
|
||||
- 'test/nfs/**'
|
||||
- '.github/workflows/nfs-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/nfs-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '15m'
|
||||
|
||||
jobs:
|
||||
nfs-integration:
|
||||
name: NFS Integration Testing
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Run NFS Integration Tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running NFS integration tests..."
|
||||
echo "============================================"
|
||||
|
||||
# Install test dependencies
|
||||
go mod download
|
||||
|
||||
# Run the protocol-layer tests. The kernel-mount tests require root
|
||||
# for mount(2) and are exercised in their own privileged step below;
|
||||
# skip them here so a "skipped because not root" line doesn't show
|
||||
# up as noise on every CI run.
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -skip '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "NFS integration tests completed"
|
||||
|
||||
- name: Install kernel NFS client
|
||||
run: |
|
||||
# nfs-common provides mount.nfs; netbase provides /etc/protocols
|
||||
# which mount.nfs's protocol-name lookups (`tcp`, `udp`) need.
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nfs-common netbase
|
||||
|
||||
- name: Run kernel-mount E2E tests
|
||||
run: |
|
||||
cd test/nfs
|
||||
|
||||
echo "Running kernel-mount end-to-end tests..."
|
||||
echo "These mount the running 'weed nfs' subprocess via the actual"
|
||||
echo "Linux NFS client to catch protocol regressions invisible to"
|
||||
echo "the go-nfs-client-based tests above."
|
||||
echo "============================================"
|
||||
|
||||
# mount(2) is privileged. Preserve PATH so 'go' (and the weed
|
||||
# binary that test/nfs/framework.go locates via $PATH) resolve
|
||||
# correctly under sudo, and pass through the Go module/cache dirs
|
||||
# so we don't redownload modules under root.
|
||||
sudo env "PATH=$PATH" \
|
||||
GOMODCACHE="$(go env GOMODCACHE)" \
|
||||
GOCACHE="$(go env GOCACHE)" \
|
||||
go test -v -timeout=${{ env.TEST_TIMEOUT }} -run '^TestKernelMount' ./...
|
||||
|
||||
echo "============================================"
|
||||
echo "Kernel-mount E2E tests completed"
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## NFS Integration Test Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Test Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Read/Write Round Trip**: Basic file create + read" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Directory Operations**: Mkdir, ReadDirPlus, RmDir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Nested Directories**: Deep tree creation and leaf I/O" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Rename**: Content preserved across rename" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Overwrite + Truncate**: Setattr(size=0) + shorter write" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Large Files**: 3 MiB binary round trip" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Edge Payloads**: All 256 byte values + empty files" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Symlinks**: Symlink + Lookup" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Missing Path**: Remove on missing entry errors cleanly" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **FSINFO**: Non-zero rtpref/wtpref advertised" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **Sequential Append**: Two-part concatenation" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **ReadDir After Remove**: Meta cache does not serve stale entries" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Kernel-Mount E2E Coverage" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 over TCP**: baseline NFSv3 mount + readdir" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V3 with mountproto=udp**: regression test for UDP MOUNT v3 responder" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- **V4 rejects cleanly**: regression test for the v4 PROG_MISMATCH path (#9262)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Most tests boot their own master + volume + filer + nfs subprocess" >> $GITHUB_STEP_SUMMARY
|
||||
echo "stack on loopback and drive it via the NFSv3 RPC protocol using" >> $GITHUB_STEP_SUMMARY
|
||||
echo "go-nfs-client. The kernel-mount E2E tests reuse the same harness" >> $GITHUB_STEP_SUMMARY
|
||||
echo "but mount the export through the in-tree Linux NFS client to" >> $GITHUB_STEP_SUMMARY
|
||||
echo "catch protocol regressions a Go-only client can't see; they run" >> $GITHUB_STEP_SUMMARY
|
||||
echo "in a separate privileged step (mount(2) requires root)." >> $GITHUB_STEP_SUMMARY
|
||||
@@ -1,118 +0,0 @@
|
||||
name: "pjdfstest POSIX Compliance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'test/pjdfstest/**'
|
||||
- '.github/workflows/pjdfstest.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: pjdfstest/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pjdfstest:
|
||||
name: pjdfstest
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-e2e
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build pjdfstest image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/pjdfstest
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
push: true
|
||||
cache-from: type=gha,scope=pjdfstest-harness
|
||||
cache-to: type=gha,mode=max,scope=pjdfstest-harness
|
||||
|
||||
- name: Tag pjdfstest image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:pjdfstest
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:pjdfstest chrislusf/seaweedfs:pjdfstest
|
||||
|
||||
- name: Start SeaweedFS cluster
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml up --wait
|
||||
|
||||
- name: Run pjdfstest
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/pjdfstest/docker-compose.yml exec -T mount \
|
||||
/run.sh 2>&1 | tee /tmp/pjdfstest-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/pjdfstest-docker-logs
|
||||
for svc in master volume filer mount; do
|
||||
docker compose -f test/pjdfstest/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/pjdfstest-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/pjdfstest/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pjdfstest-results
|
||||
path: |
|
||||
/tmp/pjdfstest-output.log
|
||||
/tmp/pjdfstest-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -19,7 +19,6 @@ jobs:
|
||||
steps:
|
||||
- name: Delete old Rust volume dev assets
|
||||
uses: mknejp/delete-release-assets@v1
|
||||
continue-on-error: true
|
||||
with:
|
||||
token: ${{ github.token }}
|
||||
tag: dev
|
||||
@@ -84,7 +83,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
@@ -155,7 +154,7 @@ jobs:
|
||||
rm weed-volume-normal
|
||||
|
||||
- name: Upload dev release assets
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: dev
|
||||
prerelease: true
|
||||
|
||||
@@ -89,7 +89,7 @@ jobs:
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
@@ -166,7 +166,7 @@ jobs:
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_${{ matrix.asset_suffix }}.tar.gz
|
||||
@@ -235,7 +235,7 @@ jobs:
|
||||
|
||||
- name: Upload release assets
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
weed-volume_large_disk_windows_amd64.zip
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
name: "S3 ETag and ACL Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/filer/etag*.go'
|
||||
- 'weed/server/filer_server_handlers_*.go'
|
||||
- 'test/s3/etag/**'
|
||||
- 'test/s3/acl/**'
|
||||
- '.github/workflows/s3-etag-acl-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-etag-acl-tests:
|
||||
name: S3 ETag + ACL Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-etag-acl
|
||||
# Minimal identity config so SSE-aware tests under acl/ can authenticate.
|
||||
cat > /tmp/seaweedfs-etag-acl-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-etag-acl \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-etag-acl-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
# Wait for the S3 endpoint to come up (returns 403 unauth before any
|
||||
# request is signed; that's fine — it means the server is listening).
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run ETag tests
|
||||
# Pins the regression for #7768: PutObject of an auto-chunked file (>8MB)
|
||||
# must return a pure MD5 hex ETag, not a `<md5>-N` composite — the AWS
|
||||
# SDK for Java v2 rejects the latter on the PutObject path.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/etag/...
|
||||
|
||||
- name: Run ACL versioning tests
|
||||
# Pins object-ACL behavior on a versioned bucket: GetObjectAcl /
|
||||
# PutObjectAcl with and without versionId, modifying ACLs on different
|
||||
# versions independently.
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/acl/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-etag-acl-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -245,132 +245,6 @@ jobs:
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# One job per test so each gets a fresh `weed mini` server, avoiding
|
||||
# the cross-test volume-pool exhaustion that surfaced when several
|
||||
# TTL-pinned bucket collections piled up in a single run.
|
||||
test:
|
||||
- TestLifecycleAbortIncompleteMultipartUpload
|
||||
- TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort
|
||||
- TestLifecycleBootstrapWalkOnExistingObjects
|
||||
- TestLifecycleConfigUpdateBetweenSweeps
|
||||
- TestLifecycleDeleteBucketLifecycleStopsDispatching
|
||||
- TestLifecycleDisabledRuleSkipsObject
|
||||
- TestLifecycleEmptyBucketSweepIsNoOp
|
||||
- TestLifecycleExpirationDateInThePast
|
||||
- TestLifecycleExpirationFiresOnBackdatedObject
|
||||
- TestLifecycleExpiredDeleteMarkerCleanup
|
||||
- TestLifecycleMultipleBucketsInOneSweep
|
||||
- TestLifecycleMultipleRulesInOneBucket
|
||||
- TestLifecycleNewerNoncurrentVersions
|
||||
- TestLifecycleNoncurrentVersionExpiration
|
||||
- TestLifecycleSizeFilterGreaterThan
|
||||
- TestLifecycleSkipsObjectLockedObjects
|
||||
- TestLifecycleSuspendedVersioningExpiration
|
||||
- TestLifecycleTagFilter
|
||||
- TestLifecycleVersionedBucketCreatesDeleteMarker
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run ${{ matrix.test }}
|
||||
timeout-minutes: 8
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server TEST_PATTERN='^${{ matrix.test }}$$'
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Last 200 lines of server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-lifecycle-test-logs-${{ matrix.test }}
|
||||
path: test/s3/lifecycle/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Checksum Tests
|
||||
timeout-minutes: 16
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Tests ==="
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/checksum
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-checksum-test-logs
|
||||
path: test/s3/checksum/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-cors-tests:
|
||||
name: S3 CORS Tests
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group", "sts"]
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -138,23 +138,6 @@ jobs:
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
"sts")
|
||||
echo "Running STS and service account tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
# SigV4-signed STS calls need admin credentials matching test_config.json.
|
||||
# Tests default to "admin"/"admin" when env vars are unset, which don't exist.
|
||||
export STS_TEST_ACCESS_KEY=test-access-key
|
||||
export STS_TEST_SECRET_KEY=test-secret-key
|
||||
# The use_service_account_credentials subtest is excluded because
|
||||
# newly-created service-account access keys are not currently
|
||||
# persisted to the filer after CreateServiceAccount — a
|
||||
# pre-existing sync issue tracked separately from the
|
||||
# GetFederationToken routing fix this PR addresses.
|
||||
go test -v -timeout 15m \
|
||||
-run "TestSTS|TestAssumeRoleWithWebIdentity|TestServiceAccount" \
|
||||
-skip "TestServiceAccountLifecycle/use_service_account_credentials" \
|
||||
./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
|
||||
@@ -71,10 +71,8 @@ jobs:
|
||||
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
|
||||
S3_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
# Check logs first for the readiness line. weed mini's progress
|
||||
# board prints " S3 ready (Xs)"; older builds and the
|
||||
# standalone S3 binary log "S3 (gateway|service) ... ready".
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)|S3[[:space:]]+ready"; then
|
||||
# Check logs first for startup message (weed mini says "S3 service is ready")
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)"; then
|
||||
echo "SeaweedFS S3 gateway is ready"
|
||||
S3_READY=1
|
||||
break
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
name: "S3 SDK V2 Route Disambiguation Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/s3api/**'
|
||||
- 'test/s3/sdk_v2_routing/**'
|
||||
- '.github/workflows/s3-sdk-v2-routing-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
s3-sdk-v2-routing-tests:
|
||||
name: S3 SDK V2 Routing Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
cd weed && go install -buildvcs=false
|
||||
|
||||
- name: Start weed mini (S3 on :8333)
|
||||
# Pins the regression for issue #9559: AWS SDK V2 / Hadoop s3a
|
||||
# listing a bucket literally named "buckets" must get an XML
|
||||
# ListObjectsV2 response, not the JSON ListTableBuckets body
|
||||
# served by the S3 Tables REST endpoint on the same path.
|
||||
run: |
|
||||
mkdir -p /tmp/seaweedfs-sdk-v2-routing
|
||||
cat > /tmp/seaweedfs-sdk-v2-routing-s3.json <<'JSON'
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{"accessKey": "some_access_key1", "secretKey": "some_secret_key1"}
|
||||
],
|
||||
"actions": ["Admin", "Read", "Write"]
|
||||
}
|
||||
]
|
||||
}
|
||||
JSON
|
||||
AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
weed mini \
|
||||
-dir=/tmp/seaweedfs-sdk-v2-routing \
|
||||
-s3.port=8333 \
|
||||
-s3.config=/tmp/seaweedfs-sdk-v2-routing-s3.json \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/weed-mini.log 2>&1 &
|
||||
echo $! > /tmp/weed-mini.pid
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8333/ | grep -qE "^(200|403)$"; then
|
||||
echo "weed mini is ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "weed mini failed to start within 30s"
|
||||
tail -50 /tmp/weed-mini.log
|
||||
exit 1
|
||||
|
||||
- name: Run SDK V2 routing tests
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
AWS_ACCESS_KEY_ID: some_access_key1
|
||||
AWS_SECRET_ACCESS_KEY: some_secret_key1
|
||||
AWS_REGION: us-east-1
|
||||
run: go test -v -timeout=5m ./test/s3/sdk_v2_routing/...
|
||||
|
||||
- name: Stop weed mini
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f /tmp/weed-mini.pid ]; then
|
||||
kill "$(cat /tmp/weed-mini.pid)" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
- name: Show server log on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== weed mini log (last 200 lines) ==="
|
||||
tail -n 200 /tmp/weed-mini.log 2>/dev/null || echo "no log available"
|
||||
|
||||
- name: Archive log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sdk-v2-routing-server-log
|
||||
path: /tmp/weed-mini.log
|
||||
retention-days: 3
|
||||
@@ -73,12 +73,8 @@ jobs:
|
||||
# Quick tests - basic SSE-C and SSE-KMS functionality + Range requests
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSEKMSIntegrationBasic|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
else
|
||||
# Comprehensive tests - SSE-C/KMS functionality plus cross-SSE copy.
|
||||
# The copy-operation tests (`.*ObjectCopyIntegration`, `TestCrossSSECopy`,
|
||||
# `TestSSEMultipartCopy`) were excluded for a long time as "pre-existing
|
||||
# SSE-C issues" (#9281); fixed and brought back into CI as part of the
|
||||
# same change that fixed them.
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior|.*ObjectCopyIntegration|TestCrossSSECopy|TestSSEMultipartCopy"
|
||||
# Comprehensive tests - SSE-C/KMS functionality, excluding copy operations (pre-existing SSE-C issues)
|
||||
make test-with-server TEST_PATTERN="TestSSECIntegrationBasic|TestSSECIntegrationVariousDataSizes|TestSSEKMSIntegrationBasic|TestSSEKMSIntegrationVariousDataSizes|.*Multipart.*Integration|TestSimpleSSECIntegration|.*RangeRequestsServerBehavior"
|
||||
fi
|
||||
|
||||
- name: Show server logs on failure
|
||||
|
||||
@@ -2,6 +2,10 @@ name: "S3 Tables Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-tables-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -190,148 +194,6 @@ jobs:
|
||||
path: test/s3tables/catalog_trino/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
dremio-iceberg-catalog-tests:
|
||||
name: Dremio Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Dremio image
|
||||
run: docker pull dremio/dremio-oss:25.2.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Dremio Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Dremio Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Dremio Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_dremio
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|dremio)" || true
|
||||
echo "=== Dremio containers ==="
|
||||
docker ps -a --filter "name=seaweed-dremio" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: dremio-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_dremio/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
doris-iceberg-catalog-tests:
|
||||
name: Doris Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Doris image
|
||||
run: docker pull apache/doris:doris-all-in-one-2.1.0
|
||||
|
||||
- name: Pre-pull Python image for PyIceberg writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Doris Iceberg Catalog Integration Tests
|
||||
timeout-minutes: 30
|
||||
working-directory: test/s3tables/catalog_doris
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Doris Iceberg Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 25m . 2>&1 | tee test-output.log || {
|
||||
echo "Doris Iceberg catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/catalog_doris
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|doris)" || true
|
||||
echo "=== Doris containers ==="
|
||||
docker ps -a --filter "name=seaweed-doris" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: doris-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_doris/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -662,77 +524,6 @@ jobs:
|
||||
path: test/s3tables/lakekeeper/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
unity-catalog-integration-tests:
|
||||
name: Unity Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Unity Catalog image
|
||||
run: docker pull unitycatalog/unitycatalog:v0.4.0
|
||||
|
||||
- name: Pre-pull Python image for delta-rs writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Unity Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Unity Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Unity Catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
|
||||
echo "=== Unity Catalog containers ==="
|
||||
docker ps -a --filter "name=seaweed-unity-catalog" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: unity-catalog-integration-test-logs
|
||||
path: test/s3tables/unity_catalog/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -37,8 +37,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -132,49 +130,7 @@ jobs:
|
||||
done
|
||||
|
||||
echo "✅ S3 server is responding, starting tests..."
|
||||
|
||||
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
|
||||
# something driving deletions. The s3tests build tag rescales one
|
||||
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
|
||||
# the upload's mtime; -dispatch / -checkpoint defaults are already
|
||||
# tightened under the same build tag.
|
||||
LC_LOG=/tmp/lifecycle-worker.log
|
||||
# -debug routes glog to stderr so the bootstrap walker's progress
|
||||
# shows up in $LC_LOG; without it weed shell silences glog.
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9333 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
# Aliveness check: a bad shell command exits in <1s and the suite
|
||||
# would otherwise just timeout the expiration tests with no signal.
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
# bash -e exits the step on the first tox failure, so move teardown
|
||||
# into a trap to guarantee the worker log + data dir reach the runner.
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -354,8 +310,11 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
@@ -380,8 +339,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -550,8 +507,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -775,8 +730,6 @@ jobs:
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
@@ -997,39 +950,6 @@ jobs:
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Spawn the lifecycle worker (see basic-tests block for context).
|
||||
LC_LOG=/tmp/lifecycle-worker-sql.log
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9337 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -1209,7 +1129,10 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
name: "Samba on FUSE Integration"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/filer/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/samba/**'
|
||||
- '.github/workflows/samba-integration.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: samba-integration/${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
samba-integration:
|
||||
name: samba-integration
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Start local Docker registry
|
||||
run: docker run -d --restart=always -p 5000:5000 --name registry registry:2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Build weed race binary
|
||||
run: |
|
||||
cd docker
|
||||
make binary_race
|
||||
|
||||
- name: Build SeaweedFS e2e image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: docker
|
||||
file: docker/Dockerfile.e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:e2e
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-e2e
|
||||
cache-to: type=gha,mode=max,scope=samba-e2e
|
||||
|
||||
- name: Tag e2e image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:e2e
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:e2e chrislusf/seaweedfs:e2e
|
||||
|
||||
- name: Build samba image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: test/samba
|
||||
build-contexts: |
|
||||
chrislusf/seaweedfs:e2e=docker-image://localhost:5000/chrislusf/seaweedfs:e2e
|
||||
tags: localhost:5000/chrislusf/seaweedfs:samba
|
||||
push: true
|
||||
cache-from: type=gha,scope=samba-harness
|
||||
cache-to: type=gha,mode=max,scope=samba-harness
|
||||
|
||||
- name: Tag samba image for docker compose
|
||||
run: |
|
||||
docker pull localhost:5000/chrislusf/seaweedfs:samba
|
||||
docker tag localhost:5000/chrislusf/seaweedfs:samba chrislusf/seaweedfs:samba
|
||||
|
||||
- name: Start SeaweedFS cluster and Samba
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml up --wait
|
||||
|
||||
- name: Run Samba test battery
|
||||
run: |
|
||||
set -o pipefail
|
||||
docker compose -f test/samba/docker-compose.yml exec -T samba \
|
||||
/run_inside_container.sh 2>&1 | tee /tmp/samba-output.log
|
||||
|
||||
- name: Collect logs
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p /tmp/samba-docker-logs
|
||||
for svc in master volume filer samba; do
|
||||
docker compose -f test/samba/docker-compose.yml logs "$svc" \
|
||||
> "/tmp/samba-docker-logs/${svc}.log" 2>&1 || true
|
||||
done
|
||||
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f test/samba/docker-compose.yml down -v
|
||||
|
||||
- name: Upload logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: samba-integration-results
|
||||
path: |
|
||||
/tmp/samba-output.log
|
||||
/tmp/samba-docker-logs/
|
||||
retention-days: 7
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "TLS Rotation Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
tls-rotation-tests:
|
||||
name: TLS Rotation Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run TLS Rotation Integration Tests
|
||||
working-directory: test/tls_rotation
|
||||
run: |
|
||||
go test -v -count=1 -timeout 5m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting master logs from temp directories..."
|
||||
mkdir -p /tmp/tls-rotation-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestMasterHTTPS*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/tls-rotation-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/tls-rotation-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tls-rotation-test-logs
|
||||
path: /tmp/tls-rotation-test-logs/
|
||||
retention-days: 14
|
||||
@@ -1,56 +0,0 @@
|
||||
name: "Vacuum Integration Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
vacuum-integration-tests:
|
||||
name: Vacuum Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed && go build -o weed .
|
||||
|
||||
- name: Run Vacuum Integration Tests
|
||||
working-directory: test/vacuum
|
||||
run: |
|
||||
go test -v -timeout 10m
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/vacuum-test-logs
|
||||
find /tmp -maxdepth 1 -type d -name "TestVacuum*" 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
cp -r "$dir" /tmp/vacuum-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
echo "Collected logs:"
|
||||
find /tmp/vacuum-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: vacuum-integration-test-logs
|
||||
path: /tmp/vacuum-test-logs/
|
||||
retention-days: 14
|
||||
@@ -2,7 +2,6 @@
|
||||
vendor
|
||||
tags
|
||||
*.swp
|
||||
.claude/
|
||||
### OSX template
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
|
||||
@@ -42,7 +42,8 @@ test: admin-generate
|
||||
|
||||
# Admin component targets
|
||||
admin-generate:
|
||||
@cd $(ADMIN_DIR) && $(MAKE) generate
|
||||
@echo "Generating admin component templates..."
|
||||
@cd $(ADMIN_DIR) && templ generate ./view
|
||||
|
||||
admin-build: admin-generate
|
||||
@echo "Building admin component..."
|
||||
|
||||
@@ -56,6 +56,7 @@ Table of Contents
|
||||
* [Quick Start](#quick-start)
|
||||
* [Quick Start with weed mini](#quick-start-with-weed-mini)
|
||||
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
|
||||
* [Quick Start with Single Binary](#quick-start-with-single-binary)
|
||||
* [Introduction](#introduction)
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
@@ -79,41 +80,39 @@ Table of Contents
|
||||
|
||||
|
||||
## Quick Start with weed mini ##
|
||||
The easiest way to get started with SeaweedFS for development and testing:
|
||||
|
||||
Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file, or run `go install github.com/seaweedfs/seaweedfs/weed@latest`. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
AWS_ACCESS_KEY_ID=admin \
|
||||
AWS_SECRET_ACCESS_KEY=secret \
|
||||
S3_BUCKET=my-bucket \
|
||||
# remove quarantine on macOS
|
||||
# xattr -d com.apple.quarantine ./weed
|
||||
|
||||
./weed mini -dir=/data
|
||||
```
|
||||
|
||||
That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
|
||||
|
||||
The same command starts everything else too:
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
This single command starts a complete SeaweedFS setup with:
|
||||
- **Master UI**: http://localhost:9333
|
||||
- **Volume Server**: http://localhost:9340
|
||||
- **Filer UI**: http://localhost:8888
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
- **WebDAV**: http://localhost:7333
|
||||
- **Admin UI**: http://localhost:23646
|
||||
|
||||
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
|
||||
|
||||
Perfect for development, testing, learning SeaweedFS, and single-node deployments. To scale out, add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, on another machine, or on thousands of machines.
|
||||
Perfect for development, testing, learning SeaweedFS, and single node deployments!
|
||||
|
||||
## Quick Start for S3 API on Docker ##
|
||||
|
||||
```bash
|
||||
docker run -p 8333:8333 \
|
||||
-e AWS_ACCESS_KEY_ID=admin \
|
||||
-e AWS_SECRET_ACCESS_KEY=secret \
|
||||
-e S3_BUCKET=my-bucket \
|
||||
chrislusf/seaweedfs
|
||||
```
|
||||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
|
||||
Same behavior as the `weed mini` command above — the S3 endpoint is at http://localhost:8333 with `my-bucket` pre-created. Drop the env vars to run anonymously for development.
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
|
||||
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
# Introduction #
|
||||
|
||||
@@ -146,11 +145,6 @@ SeaweedFS can achieve both fast local access time and elastic cloud storage capa
|
||||
What's more, the cloud storage access API cost is minimized.
|
||||
Faster and cheaper than direct cloud storage!
|
||||
|
||||
SeaweedFS also ships a built-in **Iceberg REST Catalog**, turning the same cluster into a self-contained lakehouse.
|
||||
Spark, Trino, Dremio, DuckDB, and RisingWave can query Iceberg tables directly — no Hive Metastore, Glue, or
|
||||
external catalog service required. Storage and table metadata live in one system, simplifying on-prem and
|
||||
small-team analytics stacks.
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
# Features #
|
||||
@@ -187,13 +181,6 @@ small-team analytics stacks.
|
||||
* [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
|
||||
* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
|
||||
|
||||
## Data Lakehouse Features ##
|
||||
* [S3 Table Buckets][S3TableBucket] expose a dedicated namespace for Iceberg tables with strict layout validation.
|
||||
* Built-in [Iceberg REST Catalog][IcebergCatalog] runs alongside the S3 endpoint — no external metastore needed.
|
||||
* Native integrations with [Apache Spark][SparkIceberg], [Trino][TrinoIceberg], [Dremio][DremioIceberg], [DuckDB][DuckDBIceberg], and [RisingWave][RisingWaveIceberg].
|
||||
* [Automated table maintenance][IcebergMaintenance]: compaction, snapshot expiration, orphan removal, manifest rewriting.
|
||||
* Granular IAM at the bucket, namespace, and table level via standard S3 bucket policies.
|
||||
|
||||
## Kubernetes ##
|
||||
* [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
|
||||
* [SeaweedFS Operator](https://github.com/seaweedfs/seaweedfs-operator)
|
||||
@@ -217,14 +204,6 @@ small-team analytics stacks.
|
||||
[KeyLargeValueStore]: https://github.com/seaweedfs/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
|
||||
[CloudDrive]: https://github.com/seaweedfs/seaweedfs/wiki/Cloud-Drive-Architecture
|
||||
[GatewayToRemoteObjectStore]: https://github.com/seaweedfs/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
|
||||
[S3TableBucket]: https://github.com/seaweedfs/seaweedfs/wiki/S3-Table-Bucket
|
||||
[IcebergCatalog]: https://github.com/seaweedfs/seaweedfs/wiki/SeaweedFS-Iceberg-Catalog
|
||||
[IcebergMaintenance]: https://github.com/seaweedfs/seaweedfs/wiki/Iceberg-Table-Maintenance
|
||||
[SparkIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Spark-Iceberg-Integration
|
||||
[TrinoIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Trino-Iceberg-Integration
|
||||
[DremioIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/Dremio-Iceberg-Integration
|
||||
[DuckDBIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/DuckDB-Iceberg-Integration
|
||||
[RisingWaveIceberg]: https://github.com/seaweedfs/seaweedfs/wiki/RisingWave-Iceberg-Integration
|
||||
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
@@ -509,22 +488,20 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Sqlite, Mong
|
||||
|
||||
### Compared to MinIO ###
|
||||
|
||||
Please note, as Apr 25, 2026 MinIO ceased developement. It's strongly discouraged to use that unmaintained software with multiple security bugs.
|
||||
MinIO follows AWS S3 closely and is ideal for testing for S3 API. It has good UI, policies, versionings, etc. SeaweedFS is trying to catch up here. It is also possible to put MinIO as a gateway in front of SeaweedFS later.
|
||||
|
||||
MinIO followed AWS S3 closely and was ideal for testing for S3 API. It had good UI, policies, versionings, etc. SeaweedFS is trying to catch up here.
|
||||
MinIO metadata are in simple files. Each file write will incur extra writes to corresponding meta file.
|
||||
|
||||
MinIO metadata were in simple files. Each file write will incur extra writes to corresponding meta file.
|
||||
|
||||
MinIO did not have optimization for lots of small files. The files were simply stored as is to local disks.
|
||||
MinIO does not have optimization for lots of small files. The files are simply stored as is to local disks.
|
||||
Plus the extra meta file and shards for erasure coding, it only amplifies the LOSF problem.
|
||||
|
||||
MinIO had multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
|
||||
MinIO has multiple disk IO to read one file. SeaweedFS has O(1) disk reads, even for erasure coded files.
|
||||
|
||||
MinIO had full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
|
||||
MinIO has full-time erasure coding. SeaweedFS uses replication on hot data for faster speed and optionally applies erasure coding on warm data.
|
||||
|
||||
MinIO did not have POSIX-like API support.
|
||||
MinIO does not have POSIX-like API support.
|
||||
|
||||
MinIO had specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
|
||||
MinIO has specific requirements on storage layout. It is not flexible to adjust capacity. In SeaweedFS, just start one volume server pointing to the master. That's all.
|
||||
|
||||
## Dev Plan ##
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
-24
@@ -1,24 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you find a security issue in SeaweedFS, please report it privately:
|
||||
|
||||
- Email: support@seaweedfs.com
|
||||
- Do not open a public GitHub issue
|
||||
|
||||
Please include:
|
||||
- A clear description of the issue
|
||||
- Steps to reproduce (if possible)
|
||||
- Affected versions
|
||||
|
||||
## Response
|
||||
|
||||
- We will respond as soon as possible (usually within 1 business day)
|
||||
- We will investigate and work on a fix
|
||||
- We may coordinate disclosure with you
|
||||
|
||||
## Notes
|
||||
|
||||
- Please allow time for a fix before public disclosure
|
||||
- If you’re unsure whether something is a security issue, feel free to reach out
|
||||
@@ -2,15 +2,13 @@ FROM ubuntu:22.04
|
||||
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
|
||||
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
|
||||
# Use faster mirrors and optimize package installation
|
||||
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
|
||||
apt-get -o Acquire::http::Timeout=15 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
|
||||
@@ -42,10 +42,6 @@ RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \
|
||||
echo "Skipping Rust build for $TARGETARCH (unsupported)" && \
|
||||
touch /weed-volume; \
|
||||
fi
|
||||
# Pre-built binaries arrive via GitHub Actions artifacts, which drop the
|
||||
# executable bit, so the copied file is 0644 and exec fails with "Permission
|
||||
# denied". Restore it (no-op for the empty placeholder, which stays size 0).
|
||||
RUN chmod 0755 /weed-volume
|
||||
|
||||
FROM alpine AS final
|
||||
LABEL author="Chris Lu"
|
||||
@@ -60,8 +56,8 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
|
||||
# To disable: docker run -e GODEBUG=fips140=off ...
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
apk add --no-cache fuse curl su-exec libgcc libcrypto3 libssl3 && \
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse curl su-exec libgcc && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
@@ -93,8 +89,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -7,7 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
|
||||
COPY ./entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
@@ -40,8 +40,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -34,7 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse snappy gflags curl su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
@@ -68,8 +68,3 @@ WORKDIR /data
|
||||
|
||||
# Entrypoint will handle permission fixes and user switching
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
# Default to a complete single-process cluster (master+volume+filer+S3+admin)
|
||||
# so the image is usable out of the box — including in environments like
|
||||
# GitHub Actions service containers that cannot pass arguments to the entrypoint.
|
||||
# Override with any other subcommand at `docker run` / compose time.
|
||||
CMD ["mini", "-dir=/data"]
|
||||
|
||||
@@ -17,7 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
|
||||
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Install dependencies and create non-root user
|
||||
RUN apk upgrade --no-cache && \
|
||||
RUN apk upgrade --no-cache zlib && \
|
||||
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
|
||||
addgroup -g 1000 seaweed && \
|
||||
adduser -D -u 1000 -G seaweed seaweed
|
||||
|
||||
+2
-15
@@ -37,21 +37,17 @@ if [ "$(id -u)" = "0" ]; then
|
||||
fi
|
||||
|
||||
isArgPassed() {
|
||||
# Match both `-flag` and `--flag` (and their `=value` forms): the Go fla9
|
||||
# library accepts both, and users may pick either form on the CLI.
|
||||
arg="$1"
|
||||
argWithEqualSign="$1="
|
||||
argDouble="-$1"
|
||||
argDoubleWithEqualSign="-$1="
|
||||
shift
|
||||
while [ $# -gt 0 ]; do
|
||||
passedArg="$1"
|
||||
shift
|
||||
case $passedArg in
|
||||
"$arg"|"$argDouble")
|
||||
"$arg")
|
||||
return 0
|
||||
;;
|
||||
"$argWithEqualSign"*|"$argDoubleWithEqualSign"*)
|
||||
"$argWithEqualSign"*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
@@ -99,15 +95,6 @@ case "$1" in
|
||||
exec /usr/bin/weed -logtostderr=true server $ARGS $@
|
||||
;;
|
||||
|
||||
'mini')
|
||||
ARGS="-dir=/data"
|
||||
if isArgPassed "-dir" "$@"; then
|
||||
ARGS=""
|
||||
fi
|
||||
shift
|
||||
exec /usr/bin/weed -logtostderr=true mini $ARGS $@
|
||||
;;
|
||||
|
||||
'filer')
|
||||
ARGS=""
|
||||
shift
|
||||
|
||||
@@ -4,8 +4,8 @@ go 1.25.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.2
|
||||
cloud.google.com/go/storage v1.62.1
|
||||
cloud.google.com/go/pubsub v1.50.1
|
||||
cloud.google.com/go/storage v1.60.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -15,6 +15,7 @@ require (
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/eapache/go-resiliency v1.6.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
|
||||
@@ -33,20 +34,20 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/jackc/pgx/v5 v5.8.0
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/klauspost/reedsolomon v1.14.0
|
||||
github.com/klauspost/compress v1.18.5
|
||||
github.com/klauspost/reedsolomon v1.13.3
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.7
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
@@ -60,14 +61,14 @@ require (
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/procfs v0.20.1
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.8
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/seaweedfs/raft v1.1.7
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/viper v1.21.0
|
||||
@@ -91,45 +92,44 @@ require (
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.51.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/image v0.39.0
|
||||
golang.org/x/net v0.54.0
|
||||
golang.org/x/crypto v0.49.0
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
|
||||
golang.org/x/image v0.38.0
|
||||
golang.org/x/net v0.52.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.44.0
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
golang.org/x/tools v0.44.0 // indirect
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.278.0
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/api v0.274.0
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.49.1
|
||||
modernc.org/sqlite v1.46.1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.31.0
|
||||
cloud.google.com/go/kms v1.26.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.1001
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0
|
||||
github.com/a-h/templ v0.3.977
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
|
||||
github.com/apache/iceberg-go v0.5.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0
|
||||
github.com/cognusion/imaging v1.0.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.9.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
@@ -140,23 +140,21 @@ require (
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.30.1
|
||||
github.com/parquet-go/parquet-go v0.28.0
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.11.0
|
||||
github.com/rclone/rclone v1.74.1
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.73.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.19.0
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.3
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2
|
||||
github.com/shirou/gopsutil/v4 v4.26.2
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.4
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.10
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.20.0
|
||||
@@ -170,18 +168,17 @@ require (
|
||||
atomicgo.dev/cursor v0.2.0 // indirect
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.9.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.8.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.23.0 // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
@@ -190,7 +187,8 @@ require (
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
@@ -211,6 +209,7 @@ require (
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
github.com/gin-gonic/gin v1.11.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
@@ -223,7 +222,7 @@ require (
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -256,10 +255,9 @@ require (
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
@@ -284,62 +282,62 @@ require (
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.35.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
|
||||
golang.org/x/mod v0.33.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.7.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
filippo.io/edwards25519 v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.10.3 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
|
||||
github.com/aws/smithy-go v1.25.1
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -347,13 +345,13 @@ require (
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
@@ -367,7 +365,7 @@ require (
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
@@ -375,19 +373,19 @@ require (
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/errors v0.22.4 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.28.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
@@ -404,7 +402,7 @@ require (
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
|
||||
github.com/jonboulle/clockwork v0.5.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
@@ -418,9 +416,9 @@ require (
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.22 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
@@ -435,8 +433,8 @@ require (
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.5 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
@@ -452,7 +450,7 @@ require (
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/rfjakob/eme v1.2.0 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
@@ -465,11 +463,11 @@ require (
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.6.3 // indirect
|
||||
github.com/tinylib/msgp v1.5.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
@@ -478,7 +476,7 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@@ -489,37 +487,32 @@ require (
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.42.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.43.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/libc v1.68.0 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
storj.io/picobuf v0.0.4 // indirect
|
||||
storj.io/uplink v1.14.0 // indirect
|
||||
storj.io/uplink v1.13.1 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
// apache/thrift v0.23.0 uses math.MaxUint32 as an untyped int constant in
|
||||
// lib/go/thrift/framed_transport.go, which overflows int on 32-bit GOARCHes
|
||||
// (e.g. openbsd/arm, linux/arm). Pin to v0.22.0 until upstream fixes it.
|
||||
replace github.com/apache/thrift => github.com/apache/thrift v0.22.0
|
||||
|
||||
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
|
||||
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
|
||||
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA=
|
||||
cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
|
||||
@@ -283,8 +283,8 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE
|
||||
cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
|
||||
cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY=
|
||||
cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
|
||||
cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U=
|
||||
cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY=
|
||||
cloud.google.com/go/iam v1.5.3 h1:+vMINPiDF2ognBJ97ABAYYwRgsaqxPbQDlMnbHMjolc=
|
||||
cloud.google.com/go/iam v1.5.3/go.mod h1:MR3v9oLkZCTlaqljW6Eb2d3HGDGK5/bDv93jhfISFvU=
|
||||
cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc=
|
||||
cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A=
|
||||
cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk=
|
||||
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
|
||||
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
|
||||
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
|
||||
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
|
||||
cloud.google.com/go/kms v1.31.0 h1:LS8N92OxFDgOLg5NCo3OmbvjtQAIVT5gUHVLKIDHaFE=
|
||||
cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U=
|
||||
cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
|
||||
cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
|
||||
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
|
||||
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
|
||||
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
|
||||
@@ -315,8 +315,8 @@ cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZC
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY=
|
||||
cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E=
|
||||
cloud.google.com/go/longrunning v0.8.0 h1:LiKK77J3bx5gDLi4SMViHixjD2ohlkwBi+mKA7EhfW8=
|
||||
cloud.google.com/go/longrunning v0.8.0/go.mod h1:UmErU2Onzi+fKDg2gR7dusz11Pe26aknR4kHmJJqIfk=
|
||||
cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE=
|
||||
cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM=
|
||||
cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA=
|
||||
@@ -391,10 +391,10 @@ cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjp
|
||||
cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI=
|
||||
cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0=
|
||||
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
|
||||
cloud.google.com/go/pubsub v1.50.2 h1:54Up97HnThdP4H8jjWJSSQ/mnYG2EKon7ZSNETRq0tM=
|
||||
cloud.google.com/go/pubsub v1.50.2/go.mod h1:jyCWeZdGFqd4mitSsBERnJcpqaHBsxQoPkNvjj4sp0w=
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0 h1:oMKNiBQpXImRWnHYla9uSU66ZzByZwBSCJOEs/pTKVg=
|
||||
cloud.google.com/go/pubsub/v2 v2.4.0/go.mod h1:2lS/XQKq5qtOMs6kHBK+WX1ytUC36kLl2ig3zqsGUx8=
|
||||
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
|
||||
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
|
||||
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
|
||||
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
|
||||
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
|
||||
@@ -485,8 +485,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq
|
||||
cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
|
||||
cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
|
||||
cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
|
||||
cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8=
|
||||
cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA=
|
||||
cloud.google.com/go/storage v1.60.0 h1:oBfZrSOCimggVNz9Y/bXY35uUcts7OViubeddTTVzQ8=
|
||||
cloud.google.com/go/storage v1.60.0/go.mod h1:q+5196hXfejkctrnx+VYU8RKQr/L3c0cBIlrjmiAKE0=
|
||||
cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
|
||||
cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I=
|
||||
cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4=
|
||||
@@ -553,36 +553,36 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum
|
||||
git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiIS75iTVAKIUxrST5X2dVHMTptL4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc=
|
||||
github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M=
|
||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A=
|
||||
github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/Codefor/geohash v0.0.0-20140723084247-1b41c28e3a9d h1:iG9B49Q218F/XxXNRM7k/vWf7MKmLIS8AcJV9cGN4nA=
|
||||
@@ -593,20 +593,20 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 h1:2RfP0d2QgkFH64BjZSWd59aMsc28IyWsUuHqU0txBtY=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82/go.mod h1:IPk80dOmc7VFC0DJ85xMTPmre+8xoXX6kGHAkf5jRRw=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37 h1:W8S9TrAyZ4//3PXsU6+Bi+fe/6uIL986GyS7PVzIDL4=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.37/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 h1:lMHTplAYI9FtmCo/QOcpRxmPA5REVAct1r2riQmDQKw=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264/go.mod h1:wGqkOzRu/ClJibvDgcfuJNAqI2nLhe8g91tPlDKRCdE=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 h1:mJ5QbLPOm4g5qhZiVB6wbSllfpeUExftGoyPek2hk4M=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2/go.mod h1:ngpMgwkjur1VNUjqn11LPk3o5eCyOCRbcfg/0YAY7Hc=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 h1:DUnYhvC4SoC8T84rx5omnhY3+xcQg/Whyoa3mDPIMkk=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0/go.mod h1:Q3BYO6iDA2zweQPDGbNTtqft5tDcEpm6RTuqMlPcvbw=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1/go.mod h1:HbOjEdu/yzCJ/mjTF6wEOJNbAUpHfU2UOA2hVD4CNFg=
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
|
||||
@@ -633,16 +633,16 @@ github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk=
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
|
||||
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
|
||||
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
|
||||
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
|
||||
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
|
||||
github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI=
|
||||
github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s=
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
|
||||
github.com/Sereal/Sereal/Go/sereal v0.0.0-20231009093132-b9187f1a92c6/go.mod h1:JwrycNnC8+sZPDyzM3MQ86LvaGzSpfxg885KOOwFRW4=
|
||||
github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
|
||||
github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
|
||||
@@ -652,18 +652,16 @@ github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfih
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4=
|
||||
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb h1:wumPkzt4zaxO4rHPBrjDK8iZMR41C1qs7njNqlacwQg=
|
||||
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb/go.mod h1:QiYsIBRQEO+Z4Rz7GoI+dsHVneZNONvhczuA+llOZNM=
|
||||
github.com/a-h/templ v0.3.1001 h1:yHDTgexACdJttyiyamcTHXr2QkIeVF1MukLy44EAhMY=
|
||||
github.com/a-h/templ v0.3.1001/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
|
||||
github.com/a-h/templ v0.3.977 h1:kiKAPXTZE2Iaf8JbtM21r54A8bCNsncrfnokZZSrSDg=
|
||||
github.com/a-h/templ v0.3.977/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 h1:iLDOF0rdGTrol/q8OfPIIs5kLD8XvA2q75o6Uq/tgak=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0/go.mod h1:DrEWcQJjz7t5iF2duaiyhg4jyoF0kxOD6LtECNGkZ/Q=
|
||||
github.com/aalpar/deheap v1.1.2 h1:MABHLcnjqsffb8GLkUFDigqpBBxOMz0DoKM9QfELeTw=
|
||||
github.com/aalpar/deheap v1.1.2/go.mod h1:A+nfkD4JbS05sewV0he/MYgR/90vfqyMoNNROgs+rmA=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 h1:hhdWprfSpFbN7lz3W1gM40vOgvSh1WCSMxYD6gGB4Hs=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3/go.mod h1:XaUnRxSCYgL3kkgX0QHIV0D+znljPIDImxlv2kbGv0Y=
|
||||
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
|
||||
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
|
||||
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
|
||||
github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
|
||||
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
|
||||
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
|
||||
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
|
||||
@@ -694,10 +692,11 @@ github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmO
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 h1:r/gg2gzUXiXoy72VU3jnODh8l/5rL0aslnTAbtmai/U=
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87/go.mod h1:IJTMBTlHe7cDOhRh0ioGuEKBl5iTR6xPfl5BN4AgirU=
|
||||
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq/RrkjCcxCnqUxhwf8=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0 h1:Omnzb1Z/P90Dr2TbVNu54ICQL7TKVIIsJO231w484HU=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
|
||||
github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
|
||||
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
|
||||
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
|
||||
@@ -715,36 +714,36 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0 h1:foqo/ocQ7WqKwy3FojGtZQJo0FR4vto9qnz9VaumbCo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
@@ -757,8 +756,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -828,13 +827,15 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 h1:iLoIwb7BJECHTbNcmIhYDsQhoZiACWGNvEpyqQy97Dk=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 h1:ugiQwb7DwpWQnete2AZkTh94MonZKmxD7hDGy1qTzDs=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=
|
||||
@@ -866,8 +867,8 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 h1:8Vfw2iNEpYIV6aLtMwT5UOGuPmp9MKlEKWKFTuB+MPU=
|
||||
github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2/go.mod h1:P9WiZOdQ1R/ZZDL0WzF5wlyRvrjtfhNOwMZymFpBwjE=
|
||||
github.com/cognusion/imaging v1.0.3 h1:nHyIeEVDV8JkBbuhgx8iSBW72W8rHbDEyruA0Jh7Lnk=
|
||||
github.com/cognusion/imaging v1.0.3/go.mod h1:38tFLFhGK81ORThZG8dVXPtp1uhd+3xM1WXuMn+unOA=
|
||||
github.com/cognusion/imaging v1.0.2 h1:BQwBV8V8eF3+dwffp8Udl9xF1JKh5Z0z5JkJwAi98Mc=
|
||||
github.com/cognusion/imaging v1.0.2/go.mod h1:mj7FvH7cT2dlFogQOSUQRtotBxJ4gFQ2ySMSmBm5dSk=
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 h1:v6R8oBx/Wu9fHpdPoJJjpGSUxo8NhHIwrwsfhFvU9W0=
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0/go.mod h1:0NAO+/3knbMx6+5pCv+Hcbaz4xn/Zzbn9+WIib2rKVI=
|
||||
github.com/compose-spec/compose-go/v2 v2.9.0 h1:UHSv/QHlo6QJtrT4igF1rdORgIUhDo1gWuyJUoiNNIM=
|
||||
@@ -904,9 +905,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
|
||||
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
|
||||
github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
|
||||
github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=
|
||||
github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 h1:tLWBZgPg6TV67oe76W4p+aUQEWIa52wbcuiz8GFd3vo=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
|
||||
github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
|
||||
@@ -1047,8 +1047,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
|
||||
github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
|
||||
@@ -1071,8 +1071,8 @@ github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3
|
||||
github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
|
||||
github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA=
|
||||
github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw=
|
||||
github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM=
|
||||
github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@@ -1096,8 +1096,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
|
||||
github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM=
|
||||
github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
|
||||
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
|
||||
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
|
||||
@@ -1116,8 +1116,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
|
||||
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
|
||||
github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
|
||||
github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
|
||||
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
|
||||
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
|
||||
@@ -1126,8 +1126,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
|
||||
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
|
||||
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
|
||||
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
|
||||
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
@@ -1136,8 +1136,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
|
||||
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
|
||||
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
|
||||
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
|
||||
@@ -1397,14 +1397,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 h1:PV6ipOJN0JIek4dqPqsTtenQmjI1SZntCUgPzhfk9gM=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160/go.mod h1:yRuPDnHgGmsbvopF0amMqXxr4n32GynzX5GTwFYDHaw=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c h1:r+KtxPyrhsYeNbsfeqTfEM8xRdwgV6LuNhLZxpXecb4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260213125353-6f59c89fcb7c/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
|
||||
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jaegertracing/jaeger v1.47.0 h1:XXxTMO+GxX930gxKWsg90rFr6RswkCRIW0AgWFnTYsg=
|
||||
@@ -1429,8 +1429,8 @@ github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
|
||||
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb h1:6vkM8gO+zFV2m21QzGYyUSq5TP0VQgP2Xz3UQyCN2kI=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb/go.mod h1:H1+whwD0Qe3YOunlXIWhh3rlvzW5cZfkMDYGQPg+KAM=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
@@ -1475,15 +1475,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
|
||||
github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
|
||||
github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
|
||||
@@ -1523,8 +1523,8 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
|
||||
github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
|
||||
github.com/lpar/date v1.0.0/go.mod h1:KjYe0dDyMQTgpqcUz4LEIeM5VZwhggjVx/V2dtc8NSo=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
|
||||
@@ -1544,8 +1544,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk=
|
||||
github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
|
||||
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
@@ -1657,18 +1657,18 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
|
||||
github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 h1:eDkWg6ZN0uKwWzSekoFcQJhR+C+F/aVdTwr+lGHU9Qk=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0/go.mod h1:8ZzvzuEG/cFLFZhxg/Mg1w19KqyXBKO3c17QIc5PkGs=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 h1:l9awEvzWvxmYhy/97A0hZ87pa7BncYXmcO/S8+rvgK0=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0/go.mod h1:oB8jFGVc/7/zJ+DbleE8MzGHjhs2ioCz5stRTdZdIcY=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
||||
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
|
||||
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
|
||||
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
|
||||
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
|
||||
github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
|
||||
github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
|
||||
github.com/parquet-go/parquet-go v0.30.1 h1:Oy6ganNrAdFiVwy7wNmWagfPTWA2X9Z3tVHBc7JtuX8=
|
||||
github.com/parquet-go/parquet-go v0.30.1/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
|
||||
github.com/parquet-go/parquet-go v0.28.0 h1:ECyksyv8T2pOrlLsN7aWJIoQakyk/HtxQ2lchgS4els=
|
||||
github.com/parquet-go/parquet-go v0.28.0/go.mod h1:navtkAYr2LGoJVp141oXPlO/sxLvaOe3la2JEoD8+rg=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
@@ -1755,8 +1755,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8=
|
||||
github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
@@ -1779,24 +1779,22 @@ github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
|
||||
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
|
||||
github.com/rclone/go-proton-api v1.0.2/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.74.1 h1:QQI//tl4O5uooTowdKwNKcv5qCQvJ3LIBG00+TKeXw0=
|
||||
github.com/rclone/rclone v1.74.1/go.mod h1:44epaKf5w/1ZQounefr76wx4+y9NZmgfMKle9T3pzj0=
|
||||
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.73.1 h1:FCGZUd7sXO8+nIpbuXGPkd+f23o/ZACefrHtRGDAiDM=
|
||||
github.com/rclone/rclone v1.73.1/go.mod h1:T9HeSXv/SI9qJFkYLqsjY/5ljIFWJTVmTrG7H/4JrPY=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k=
|
||||
github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
|
||||
github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
|
||||
@@ -1809,8 +1807,8 @@ github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=
|
||||
github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
@@ -1840,12 +1838,12 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3 h1:rJufGrHImTx7yoGUmetUi+To4LrmTQJROJnBHWt92ic=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/seaweedfs/raft v1.1.8 h1:a5f+wSnriRjINLqLqMY7DDNNfU1+kv6TKBuOZa0KDwU=
|
||||
github.com/seaweedfs/raft v1.1.8/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/seaweedfs/raft v1.1.7 h1:3mVJZ2p4rdvBtbbrHROPjYKtH+q5qjMBc56G6VRu1kA=
|
||||
github.com/seaweedfs/raft v1.1.7/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
@@ -1856,15 +1854,15 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
|
||||
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
|
||||
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
|
||||
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
|
||||
@@ -1887,8 +1885,8 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
@@ -1930,8 +1928,8 @@ github.com/substrait-io/substrait-protobuf/go v0.81.0 h1:/qC1XYKuO4oPdTwLYySuVZ6
|
||||
github.com/substrait-io/substrait-protobuf/go v0.81.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c h1:dtcOwRimeiBFrlutmF6K94l0rxYFARNFMA+lSQ41C+M=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 h1:rrGZv6xYk37hx0tW2sYfgbO0PqStbHqz6Bq6oc9Hurg=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/tarantool/go-iproto v1.1.0 h1:HULVOIHsiehI+FnHfM7wMDntuzUddO09DKqu2WnFQ5A=
|
||||
github.com/tarantool/go-iproto v1.1.0/go.mod h1:LNCtdyZxojUed8SbOiYHoc3v9NvaZTB7p96hUySMlIo=
|
||||
@@ -1961,8 +1959,8 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GR
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA=
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g=
|
||||
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
|
||||
github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s=
|
||||
github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tinylib/msgp v1.5.0 h1:GWnqAE54wmnlFazjq2+vgr736Akg58iiHImh+kPY2pc=
|
||||
github.com/tinylib/msgp v1.5.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
@@ -2028,10 +2026,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
|
||||
github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
@@ -2064,14 +2058,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180 h1:avIdi8eGXjKbn1WLokNR1Ofnz1k8t7tJ88YQLD/iCi8=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260311095541-ebbf792c1180/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2 h1:Bzra5hUQIWrxknQ0oV6HE5q/eZU1VhHMKSZ/XWpqMgM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.134.2/go.mod h1:VYUUkRJkKuQPkIpgtZJj6+58Fa2g8ccAqdmaaK6HP5k=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3 h1:hTwpF+PdbuR7vcixN+4AC6yu4asaUIAljQCxL51eDII=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3/go.mod h1:stS1mQYjbJvwwYaYzKyFY9eMiuVXWWXQA6T+SpOLg9c=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
|
||||
@@ -2098,15 +2092,13 @@ github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
|
||||
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
|
||||
github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
|
||||
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/mwc v0.0.7 h1:0NerGhCww6ZQx+/xCx5iwznftveokvto1KILpYfENZk=
|
||||
github.com/zeebo/mwc v0.0.7/go.mod h1:0B32or6moOig1YGuqMoimBpU9QK9uYaGG2bBOuddqtE=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
|
||||
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
|
||||
go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
|
||||
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 h1:jlwjtELjA8yi2VWpOFH+0w0lGr3K6mVDyn0RDB9aaAY=
|
||||
@@ -2130,14 +2122,14 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 h1:0tY123n7CdWMem7MOVdKOt0YfshufLCwfE5Bob+hQuM=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.35.0 h1:0NIXxOCFx+SKbhCVxwl3ETG8ClLPAa0KuKV6p3yhxP8=
|
||||
@@ -2148,18 +2140,18 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4D
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 h1:5gn2urDL/FBnK8OkCfD1j3/ER79rUuTYmCvlXBKeYL8=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0/go.mod h1:0fBG6ZJxhqByfFZDwSwpZGzJU671HkwpWaNe2t4VUPI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
@@ -2216,8 +2208,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -2233,8 +2225,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
@@ -2248,8 +2240,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
|
||||
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2283,8 +2275,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
|
||||
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
|
||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2352,8 +2344,8 @@ golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
|
||||
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -2511,11 +2503,11 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4=
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0=
|
||||
golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2531,8 +2523,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
||||
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
|
||||
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -2553,8 +2545,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -2632,8 +2624,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
|
||||
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -2711,8 +2703,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E=
|
||||
google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -2846,12 +2838,12 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -2892,8 +2884,8 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
|
||||
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
|
||||
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
|
||||
google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
|
||||
@@ -2983,19 +2975,19 @@ modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
|
||||
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
|
||||
modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=
|
||||
modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8=
|
||||
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
|
||||
modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
|
||||
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
|
||||
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
|
||||
modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
|
||||
modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
|
||||
modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU=
|
||||
modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0=
|
||||
modernc.org/ccgo/v4 v4.30.2 h1:4yPaaq9dXYXZ2V8s1UgrC3KIj580l2N4ClrLwnbv2so=
|
||||
modernc.org/ccgo/v4 v4.30.2/go.mod h1:yZMnhWEdW0qw3EtCndG1+ldRrVGS+bIwyWmAWzS0XEw=
|
||||
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
|
||||
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
|
||||
@@ -3010,8 +3002,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
|
||||
modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
|
||||
modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
|
||||
modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
|
||||
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
|
||||
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
|
||||
modernc.org/libc v1.68.0 h1:PJ5ikFOV5pwpW+VqCK1hKJuEWsonkIJhhIXyuF/91pQ=
|
||||
modernc.org/libc v1.68.0/go.mod h1:NnKCYeoYgsEqnY3PgvNgAeaJnso968ygU8Z0DxjoEc0=
|
||||
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
@@ -3030,8 +3022,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
|
||||
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
|
||||
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU=
|
||||
modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA=
|
||||
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
@@ -3054,8 +3046,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aN
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
|
||||
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
|
||||
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a h1:7gSBQY3vhQMIqi3vfaEXR7mreRjcBLfVsYY0rHIN7P0=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a/go.mod h1:kyxwKwlfH4paBZCZt/szvRB770ieAIJF73iDy2DpEHw=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c h1:UDXSrdeLJe3QFouavSW10fYdpclK0YNu3KvQHzqq2+k=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c/go.mod h1:XNX7uykja6aco92y2y8RuqaXIDRPpt1YA2OQDKlKEUk=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 h1:8OE12DvUnB9lfZcHe7IDGsuhjrY9GBAr964PVHmhsro=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 h1:5MZ0CyMbG6Pi0rRzUWVG6dvpXjbBYEX2oyXuj+tT+sk=
|
||||
@@ -3064,7 +3056,7 @@ storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q=
|
||||
storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs=
|
||||
storj.io/picobuf v0.0.4 h1:qswHDla+YZ2TovGtMnU4astjvrADSIz84FXRn0qgP6o=
|
||||
storj.io/picobuf v0.0.4/go.mod h1:hSMxmZc58MS/2qSLy1I0idovlO7+6K47wIGUyRZa6mg=
|
||||
storj.io/uplink v1.14.0 h1:J1yXlt0aRr6kgLTHWXOWosNCFVfbamlcyd+CSxyIczo=
|
||||
storj.io/uplink v1.14.0/go.mod h1:2ysmjzd/1Xtz4VKoErNcSqBQz3UC9WKTVuLMV1cNu6E=
|
||||
storj.io/uplink v1.13.1 h1:C8RdW/upALoCyuF16Lod9XGCXEdbJAS+ABQy9JO/0pA=
|
||||
storj.io/uplink v1.13.1/go.mod h1:x0MQr4UfFsQBwgVWZAtEsLpuwAn6dg7G0Mpne1r516E=
|
||||
tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc=
|
||||
tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.28"
|
||||
appVersion: "4.18"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.28.0
|
||||
version: 4.18.0
|
||||
|
||||
@@ -49,35 +49,6 @@ CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
#### RocksDB variant
|
||||
|
||||
The `_large_disk_rocksdb` image tag ships with RocksDB pre-configured as the filer backend.
|
||||
To use this image with the Helm chart, override the image on all three components and disable
|
||||
the chart's default `WEED_LEVELDB2_ENABLED`, which would otherwise re-enable LevelDB2 and
|
||||
override the image's built-in RocksDB configuration:
|
||||
|
||||
```yaml
|
||||
# Replace <VERSION> with the desired seaweedfs version, e.g. 3.80_large_disk_rocksdb.
|
||||
master:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
volume:
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
|
||||
filer:
|
||||
enablePVC: true
|
||||
imageOverride: chrislusf/seaweedfs:<VERSION>_large_disk_rocksdb
|
||||
extraEnvironmentVars:
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
* `master` and `volume` use the same image tag so that all components share a consistent
|
||||
SeaweedFS build; RocksDB itself is only used by the filer.
|
||||
* `filer.enablePVC: true` (or another form of persistent storage for the filer) is required
|
||||
so that the RocksDB metadata store survives pod restarts — otherwise metadata will be lost.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.admin.image" . }}
|
||||
image: {{ template "admin.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
{{- $adminAuthEnabled := or .Values.admin.secret.existingSecret .Values.admin.secret.adminPassword }}
|
||||
{{- $urlPrefix := .Values.admin.urlPrefix }}
|
||||
@@ -135,14 +135,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- $secretExtraEnvironmentVars := .Values.admin.secretExtraEnvironmentVars }}
|
||||
{{- if $secretExtraEnvironmentVars }}
|
||||
{{- range $key := keys $secretExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $secretExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
valueFrom: {{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -188,13 +180,11 @@ spec:
|
||||
- name: admin-logs
|
||||
mountPath: /logs
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -252,7 +242,7 @@ spec:
|
||||
securityContext: {{- omit .Values.admin.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.admin.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.admin.data.type "hostPath" }}
|
||||
@@ -285,12 +275,10 @@ spec:
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.admin.logs.claimName }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
@@ -315,7 +303,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.admin.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.admin.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "admin.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.admin.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
{{- /* Determine default cluster alias and the corresponding env var keys to avoid conflicts */}}
|
||||
@@ -247,8 +247,8 @@ spec:
|
||||
{{- $httpsPort := .Values.allInOne.s3.httpsPort | default .Values.s3.httpsPort }}
|
||||
{{- if $httpsPort }}
|
||||
-s3.port.https={{ $httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
|
||||
@@ -330,13 +330,11 @@ spec:
|
||||
mountPath: /etc/seaweedfs/master.toml
|
||||
subPath: master.toml
|
||||
readOnly: true
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
readOnly: true
|
||||
@@ -420,7 +418,7 @@ spec:
|
||||
{{- omit .Values.allInOne.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.allInOne.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: data
|
||||
@@ -463,12 +461,10 @@ spec:
|
||||
- name: master-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master-config
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -117,13 +117,11 @@ spec:
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -166,7 +164,7 @@ spec:
|
||||
securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.cosi.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: socket
|
||||
@@ -181,12 +179,10 @@ spec:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-s3-secret
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -86,7 +86,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.filer.image" . }}
|
||||
image: {{ template "filer.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -200,8 +200,8 @@ spec:
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- if .Values.filer.s3.httpsPort }}
|
||||
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/seaweedfs_s3_config \
|
||||
@@ -234,13 +234,11 @@ spec:
|
||||
mountPath: /etc/seaweedfs/notification.toml
|
||||
subPath: notification.toml
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -276,8 +274,7 @@ spec:
|
||||
name: swfs-s3-tls
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $jwt := (.Values.global.seaweedfs.securityConfig).jwtSigning | default dict }}
|
||||
{{- $isJwtEnabled := or $jwt.filerWrite $jwt.filerRead }}
|
||||
{{- $isJwtEnabled := or .Values.global.seaweedfs.securityConfig.jwtSigning.filerWrite .Values.global.seaweedfs.securityConfig.jwtSigning.filerRead }}
|
||||
{{- if .Values.filer.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
{{- if or $isJwtEnabled .Values.filer.readinessProbe.tcpSocket }}
|
||||
@@ -320,7 +317,7 @@ spec:
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
@@ -371,12 +368,10 @@ spec:
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-notification-config
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
@@ -418,7 +413,7 @@ spec:
|
||||
storageClassName: {{ .Values.filer.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.filer.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "filer.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
|
||||
|
||||
@@ -80,7 +80,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -188,13 +188,11 @@ spec:
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/master.toml
|
||||
subPath: master.toml
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -253,7 +251,7 @@ spec:
|
||||
securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.master.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
@@ -289,12 +287,10 @@ spec:
|
||||
- name: master-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master-config
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
@@ -316,7 +312,7 @@ spec:
|
||||
nodeSelector:
|
||||
{{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- $pvc_exists := include "seaweedfs.master.pvc_exists" . -}}
|
||||
{{- $pvc_exists := include "master.pvc_exists" . -}}
|
||||
{{- if $pvc_exists }}
|
||||
volumeClaimTemplates:
|
||||
{{- if eq .Values.master.data.type "persistentVolumeClaim"}}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.s3.image" . }}
|
||||
image: {{ template "s3.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -127,8 +127,8 @@ spec:
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
-port.https={{ .Values.s3.httpsPort }} \
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.domainName }}
|
||||
-domainName={{ .Values.s3.domainName }} \
|
||||
@@ -156,13 +156,11 @@ spec:
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -228,7 +226,7 @@ spec:
|
||||
securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.s3.enableAuth }}
|
||||
@@ -251,12 +249,10 @@ spec:
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
{{- $access_key_admin := $adminCreds.accessKey -}}
|
||||
{{- $secret_key_admin := $adminCreds.secretKey -}}
|
||||
{{- if not (and $access_key_admin $secret_key_admin) -}}
|
||||
{{- $access_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_admin = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
{{- $readCreds := $creds.read | default dict -}}
|
||||
{{- $access_key_read := $readCreds.accessKey -}}
|
||||
{{- $secret_key_read := $readCreds.secretKey -}}
|
||||
{{- if not (and $access_key_read $secret_key_read) -}}
|
||||
{{- $access_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $access_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_access_key_id" "length" 20 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- $secret_key_read = include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "read_secret_access_key" "length" 40 "existingSecret" (ternary $existingSecret nil $reuse)) -}}
|
||||
{{- end -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- /*
|
||||
The seaweedfs-s3 Service only gains a "metrics" port when the standalone S3
|
||||
gateway is enabled. With only the embedded filer S3 gateway, metrics live on
|
||||
the filer process and are already scraped by the filer ServiceMonitor.
|
||||
*/ -}}
|
||||
{{- if and .Values.s3.enabled .Values.s3.metricsPort .Values.global.seaweedfs.monitoring.enabled }}
|
||||
{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
|
||||
{{- if .Values.s3.metricsPort }}
|
||||
{{- if .Values.global.seaweedfs.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
@@ -34,3 +31,5 @@ spec:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -74,7 +74,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.sftp.image" . }}
|
||||
image: {{ template "sftp.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -176,13 +176,11 @@ spec:
|
||||
- mountPath: /etc/sw/ssh
|
||||
name: config-ssh
|
||||
readOnly: true
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -235,7 +233,7 @@ spec:
|
||||
securityContext: {{- omit .Values.sftp.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.sftp.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if .Values.sftp.enableAuth }}
|
||||
@@ -266,12 +264,10 @@ spec:
|
||||
- name: logs
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
{{- if or .Values.sftp.enabled .Values.allInOne.enabled }}
|
||||
{{- $secretName := printf "%s-sftp-secret" (include "seaweedfs.fullname" .) }}
|
||||
{{- $admin_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" "length" 20) -}}
|
||||
{{- $read_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" "length" 20) -}}
|
||||
{{- $public_user_pwd := include "seaweedfs.getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" "length" 20) -}}
|
||||
{{- $admin_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "admin_password" 20) -}}
|
||||
{{- $read_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "readonly_password" 20) -}}
|
||||
{{- $public_user_pwd := include "getOrGeneratePassword" (dict "namespace" .Release.Namespace "secretName" $secretName "key" "public_user_password" 20) -}}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
type: Opaque
|
||||
|
||||
@@ -72,77 +72,77 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper filer image */}}
|
||||
{{- define "seaweedfs.filer.image" -}}
|
||||
{{- define "filer.image" -}}
|
||||
{{- if .Values.filer.imageOverride -}}
|
||||
{{- $imageOverride := .Values.filer.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper master image */}}
|
||||
{{- define "seaweedfs.master.image" -}}
|
||||
{{- define "master.image" -}}
|
||||
{{- if .Values.master.imageOverride -}}
|
||||
{{- $imageOverride := .Values.master.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper s3 image */}}
|
||||
{{- define "seaweedfs.s3.image" -}}
|
||||
{{- define "s3.image" -}}
|
||||
{{- if .Values.s3.imageOverride -}}
|
||||
{{- $imageOverride := .Values.s3.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper sftp image */}}
|
||||
{{- define "seaweedfs.sftp.image" -}}
|
||||
{{- define "sftp.image" -}}
|
||||
{{- if .Values.sftp.imageOverride -}}
|
||||
{{- $imageOverride := .Values.sftp.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper admin image */}}
|
||||
{{- define "seaweedfs.admin.image" -}}
|
||||
{{- define "admin.image" -}}
|
||||
{{- if .Values.admin.imageOverride -}}
|
||||
{{- $imageOverride := .Values.admin.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper worker image */}}
|
||||
{{- define "seaweedfs.worker.image" -}}
|
||||
{{- define "worker.image" -}}
|
||||
{{- if .Values.worker.imageOverride -}}
|
||||
{{- $imageOverride := .Values.worker.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper volume image */}}
|
||||
{{- define "seaweedfs.volume.image" -}}
|
||||
{{- define "volume.image" -}}
|
||||
{{- if .Values.volume.imageOverride -}}
|
||||
{{- $imageOverride := .Values.volume.imageOverride -}}
|
||||
{{- printf "%s" $imageOverride -}}
|
||||
{{- else -}}
|
||||
{{- include "seaweedfs.image" . }}
|
||||
{{- include "common.image" . }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Computes the container image name for all components (if they are not overridden) */}}
|
||||
{{- define "seaweedfs.image" -}}
|
||||
{{- define "common.image" -}}
|
||||
{{- $registryName := default .Values.image.registry .Values.global.imageRegistry | toString -}}
|
||||
{{- $repositoryName := default .Values.image.repository .Values.global.seaweedfs.image.repository | toString -}}
|
||||
{{- $name := .Values.global.seaweedfs.image.name | toString -}}
|
||||
@@ -160,7 +160,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Volume PVC exists */}}
|
||||
{{- define "seaweedfs.volume.pvc_exists" -}}
|
||||
{{- define "volume.pvc_exists" -}}
|
||||
{{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -169,7 +169,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Filer PVC exists */}}
|
||||
{{- define "seaweedfs.filer.pvc_exists" -}}
|
||||
{{- define "filer.pvc_exists" -}}
|
||||
{{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -178,7 +178,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Master PVC exists */}}
|
||||
{{- define "seaweedfs.master.pvc_exists" -}}
|
||||
{{- define "master.pvc_exists" -}}
|
||||
{{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -187,7 +187,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any Admin PVC exists */}}
|
||||
{{- define "seaweedfs.admin.pvc_exists" -}}
|
||||
{{- define "admin.pvc_exists" -}}
|
||||
{{- if or (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.logs.type "persistentVolumeClaim") -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -196,7 +196,7 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
|
||||
{{/* check if any InitContainers exist for Volumes */}}
|
||||
{{- define "seaweedfs.volume.initContainers_exists" -}}
|
||||
{{- define "volume.initContainers_exists" -}}
|
||||
{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
|
||||
{{- printf "true" -}}
|
||||
{{- else -}}
|
||||
@@ -225,10 +225,10 @@ imagePullSecrets:
|
||||
{{/*
|
||||
Renders a value that contains template perhaps with scope if the scope is present.
|
||||
Usage:
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "seaweedfs.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
|
||||
{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.tplvalues.render" -}}
|
||||
{{- define "common.tplvalues.render" -}}
|
||||
{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
|
||||
{{- if contains "{{" (toJson .value) }}
|
||||
{{- if .scope }}
|
||||
@@ -245,9 +245,9 @@ Usage:
|
||||
Converts a Kubernetes quantity like "256Mi" or "2G" to a float64 in base units,
|
||||
handling both binary (Ki, Mi, Gi) and decimal (m, k, M) suffixes; numeric inputs
|
||||
Usage:
|
||||
{{ include "seaweedfs.resource-quantity" "10Gi" }}
|
||||
{{ include "common.resource-quantity" "10Gi" }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.resource-quantity" -}}
|
||||
{{- define "common.resource-quantity" -}}
|
||||
{{- $value := . -}}
|
||||
{{- $unit := 1.0 -}}
|
||||
{{- if typeIs "string" . -}}
|
||||
@@ -267,7 +267,7 @@ Usage:
|
||||
getOrGeneratePassword will check if a password exists in a secret and return it,
|
||||
or generate a new random password if it doesn't exist.
|
||||
*/}}
|
||||
{{- define "seaweedfs.getOrGeneratePassword" -}}
|
||||
{{- define "getOrGeneratePassword" -}}
|
||||
{{- $params := . -}}
|
||||
{{- $namespace := $params.namespace -}}
|
||||
{{- $secretName := $params.secretName -}}
|
||||
@@ -332,16 +332,6 @@ Create the name of the service account to use
|
||||
{{- .Values.global.seaweedfs.serviceAccountName | default "seaweedfs" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* True when security.toml should be rendered and mounted. volumeWrite is
|
||||
excluded since it defaults to true. */}}
|
||||
{{- define "seaweedfs.securityConfigEnabled" -}}
|
||||
{{- $sec := (.Values.global.seaweedfs).securityConfig | default dict -}}
|
||||
{{- $jwt := $sec.jwtSigning | default dict -}}
|
||||
{{- if or .Values.global.seaweedfs.enableSecurity $jwt.volumeRead $jwt.filerWrite $jwt.filerRead -}}
|
||||
true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}}
|
||||
{{- define "seaweedfs.s3.tlsArgs" -}}
|
||||
{{- $prefix := .prefix -}}
|
||||
|
||||
@@ -68,7 +68,7 @@ spec:
|
||||
{{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
|
||||
containers:
|
||||
- name: post-install-job
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
image: {{ template "master.image" . }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
env:
|
||||
- name: WEED_CLUSTER_DEFAULT
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
@@ -55,7 +55,6 @@ data:
|
||||
key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
# all grpc tls authentications are mutual
|
||||
# the values for the following ca, cert, and key are paths to the PERM files.
|
||||
[grpc]
|
||||
@@ -95,5 +94,4 @@ data:
|
||||
[https.volume]
|
||||
cert = ""
|
||||
key = ""
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -35,8 +35,8 @@
|
||||
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
|
||||
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
|
||||
{{- if and $currentPVC }}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
|
||||
{{- $oldSize := include "common.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "common.resource-quantity" $desiredSize }}
|
||||
{{- if gt $newSize $oldSize }}
|
||||
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
|
||||
{{- end }}
|
||||
|
||||
@@ -71,12 +71,12 @@ spec:
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
|
||||
{{- $initContainers_exists := include "seaweedfs.volume.initContainers_exists" $ -}}
|
||||
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
|
||||
{{- if $initContainers_exists }}
|
||||
initContainers:
|
||||
{{- if $volume.idx }}
|
||||
- name: seaweedfs-vol-move-idx
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
command: [ '/bin/sh', '-c' ]
|
||||
args: [ '{{range $dir := $volume.dataDirs }}if ls /{{$dir.name}}/*.idx >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
|
||||
@@ -104,7 +104,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.volume.image" $ }}
|
||||
image: {{ template "volume.image" $ }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" $.Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_NAME
|
||||
@@ -137,9 +137,6 @@ spec:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
{{- if $volume.rust }}
|
||||
exec /usr/bin/weed-volume \
|
||||
{{- else }}
|
||||
exec /usr/bin/weed \
|
||||
{{- if $volume.logs }}
|
||||
-logdir=/logs \
|
||||
@@ -152,7 +149,6 @@ spec:
|
||||
-v={{ $.Values.global.seaweedfs.loggingLevel }} \
|
||||
{{- end }}
|
||||
volume \
|
||||
{{- end }}
|
||||
-port={{ $volume.port }} \
|
||||
{{- if $volume.metricsPort }}
|
||||
-metricsPort={{ $volume.metricsPort }} \
|
||||
@@ -184,7 +180,7 @@ spec:
|
||||
{{- if $volume.imagesFixOrientation }}
|
||||
-images.fix.orientation \
|
||||
{{- end }}
|
||||
{{- if and $volume.pulseSeconds (not $volume.rust) }}
|
||||
{{- if $volume.pulseSeconds }}
|
||||
-pulseSeconds={{ $volume.pulseSeconds }} \
|
||||
{{- end }}
|
||||
{{- if $volume.index }}
|
||||
@@ -215,13 +211,11 @@ spec:
|
||||
- name: idx
|
||||
mountPath: "/idx/"
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" $ }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -280,7 +274,7 @@ spec:
|
||||
securityContext: {{- omit $volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if $volume.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
|
||||
@@ -339,12 +333,10 @@ spec:
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" $ }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" $ }}-security-config
|
||||
{{- end }}
|
||||
{{- if $.Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" $ }}-ca-cert
|
||||
|
||||
@@ -77,7 +77,7 @@ spec:
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: seaweedfs
|
||||
image: {{ template "seaweedfs.worker.image" . }}
|
||||
image: {{ template "worker.image" . }}
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
@@ -149,13 +149,11 @@ spec:
|
||||
- name: worker-logs
|
||||
mountPath: /logs
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
@@ -221,7 +219,7 @@ spec:
|
||||
securityContext: {{- omit .Values.worker.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.sidecars }}
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- include "common.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- if eq .Values.worker.data.type "hostPath" }}
|
||||
@@ -254,12 +252,10 @@ spec:
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.worker.logs.claimName }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
|
||||
@@ -18,8 +18,6 @@ global:
|
||||
loggingLevel: 1
|
||||
enableSecurity: false
|
||||
masterServer: null
|
||||
# filerWrite: true mounts security.toml on filer + admin without needing
|
||||
# enableSecurity (mTLS); required for the Admin UI Users tab.
|
||||
securityConfig:
|
||||
jwtSigning:
|
||||
volumeWrite: true
|
||||
@@ -305,11 +303,6 @@ volume:
|
||||
enabled: true
|
||||
imageOverride: null
|
||||
restartPolicy: null
|
||||
# Run the Rust volume server (/usr/bin/weed-volume) instead of the Go one.
|
||||
# Requires an image that ships the Rust binary (amd64/arm64). The Go-only
|
||||
# log flags (-logtostderr/-logdir/-v) and -pulseSeconds are dropped; set log
|
||||
# level via the RUST_LOG env var in extraEnvironmentVars if needed.
|
||||
rust: false
|
||||
port: 8080
|
||||
grpcPort: 18080
|
||||
metricsPort: 9327
|
||||
@@ -1293,13 +1286,6 @@ admin:
|
||||
|
||||
extraEnvironmentVars: {}
|
||||
|
||||
# secret env variables (e.g. for injecting OIDC client secret from a Kubernetes Secret)
|
||||
secretExtraEnvironmentVars: {}
|
||||
# WEED_ADMIN_OIDC_CLIENT_SECRET:
|
||||
# secretKeyRef:
|
||||
# name: seaweedfs-admin-oidc
|
||||
# key: client_secret
|
||||
|
||||
# Health checks
|
||||
livenessProbe:
|
||||
enabled: true
|
||||
@@ -1353,14 +1339,9 @@ worker:
|
||||
# Admin server to connect to
|
||||
adminServer: ""
|
||||
|
||||
# Worker job types - categories or comma-separated list of names/aliases
|
||||
# Categories: all, default, heavy
|
||||
# default: vacuum, volume_balance, ec_balance, admin_script
|
||||
# heavy: erasure_coding, iceberg_maintenance
|
||||
# all: every registered job type
|
||||
# Examples: "all", "default", "heavy", "default,iceberg" (default+iceberg), "vacuum,volume_balance"
|
||||
# Refer: https://github.com/seaweedfs/seaweedfs/wiki/Worker
|
||||
jobType: "all"
|
||||
# Worker job types - comma-separated list
|
||||
# Available: vacuum, volume_balance, erasure_coding
|
||||
jobType: "vacuum,volume_balance,erasure_coding"
|
||||
|
||||
# Maximum number of concurrent detection requests
|
||||
maxDetect: 1
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
MARP ?= npx @marp-team/marp-cli
|
||||
SRC := seaweedfs-production-setup.md
|
||||
PDF := $(SRC:.md=.pdf)
|
||||
HTML := $(SRC:.md=.html)
|
||||
PPTX := $(SRC:.md=.pptx)
|
||||
FLAGS := --allow-local-files
|
||||
|
||||
.PHONY: all pdf html pptx watch clean
|
||||
|
||||
all: pdf
|
||||
|
||||
pdf: $(PDF)
|
||||
|
||||
html: $(HTML)
|
||||
|
||||
pptx: $(PPTX)
|
||||
|
||||
$(PDF): $(SRC)
|
||||
$(MARP) $(SRC) --pdf $(FLAGS) -o $@
|
||||
|
||||
$(HTML): $(SRC)
|
||||
$(MARP) $(SRC) --html $(FLAGS) -o $@
|
||||
|
||||
$(PPTX): $(SRC)
|
||||
$(MARP) $(SRC) --pptx $(FLAGS) -o $@
|
||||
|
||||
watch:
|
||||
$(MARP) $(SRC) --watch --html $(FLAGS)
|
||||
|
||||
clean:
|
||||
rm -f $(PDF) $(HTML) $(PPTX)
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 738 KiB |
@@ -1,118 +0,0 @@
|
||||
---
|
||||
marp: true
|
||||
theme: default
|
||||
paginate: true
|
||||
---
|
||||
|
||||
# SeaweedFS Production Setup
|
||||
|
||||
A walkthrough of the recommended deployment topology
|
||||
|
||||
- Storage layer
|
||||
- File access layer
|
||||
- Backend operations
|
||||
- Erasure coding for durability
|
||||
|
||||
---
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
# Storage Layer
|
||||
|
||||
**3 Masters + Volume Servers**
|
||||
|
||||
- **Masters (run 3)**
|
||||
- Form a Raft quorum; tolerate 1 failure
|
||||
- Track volume locations and cluster topology
|
||||
- Lightweight; not on the data path for reads/writes after lookup
|
||||
|
||||
- **Volume Servers (scale out)**
|
||||
- Hold the actual needle data on local disks
|
||||
- Add more servers for capacity and throughput
|
||||
- Heartbeat to masters; clients talk to them directly for I/O
|
||||
|
||||
Rule of thumb: start with 3 masters, then grow volume servers with data.
|
||||
|
||||
---
|
||||
|
||||
# File Access Layer
|
||||
|
||||
**Filer + DB for metadata**
|
||||
|
||||
- Filer serves POSIX-like directory and file metadata
|
||||
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
|
||||
- Run **3 filers** for HA
|
||||
- Filers are stateless relative to each other — state lives in the DB
|
||||
|
||||
**S3 Servers**
|
||||
|
||||
- Expose the S3 API on top of the filer
|
||||
- Scale the count based on throughput requirements
|
||||
- Can be co-located with filers or run independently
|
||||
|
||||
---
|
||||
|
||||
# Backend Operations
|
||||
|
||||
**1 Admin Server + a few Workers**
|
||||
|
||||
- **Admin server (1 is enough)**
|
||||
- Coordinates background jobs: balancing, EC encoding, vacuum, replication fixes
|
||||
- **Not on the data path** — a restart does not affect object store reads/writes
|
||||
- No HA required for normal operation
|
||||
|
||||
- **Workers (a few)**
|
||||
- Execute long-running tasks issued by the admin server
|
||||
- Horizontally scalable based on maintenance workload
|
||||
|
||||
Keeps housekeeping off the hot path.
|
||||
|
||||
---
|
||||
|
||||
# Production Topology Summary
|
||||
|
||||
| Component | Count | Role |
|
||||
|----------------|----------------------|-----------------------------------|
|
||||
| Master | 3 | Raft quorum, topology, volume map |
|
||||
| Volume Server | N (scale out) | Stores data needles |
|
||||
| Filer | 3 | Metadata gateway |
|
||||
| Metadata DB | HA cluster | Persists filer metadata |
|
||||
| S3 Server | N (by throughput) | S3 API frontend |
|
||||
| Admin | 1 | Background job coordinator |
|
||||
| Worker | A few | Executes admin-scheduled tasks |
|
||||
|
||||
---
|
||||
|
||||
# Erasure Coding for Durability
|
||||
|
||||
Prefer **many volume servers** so EC shards spread across failure domains.
|
||||
|
||||
**Example: 8 volume servers, EC 5+3**
|
||||
|
||||
- Each volume becomes 5 data shards + 3 parity shards
|
||||
- Shards are placed on 8 distinct volume servers
|
||||
- **Tolerates up to 3 simultaneous volume server failures**
|
||||
- Storage overhead: 1.6x vs 3x for full replication
|
||||
|
||||
```
|
||||
Volume -> [D1][D2][D3][D4][D5] + [P1][P2][P3]
|
||||
| | | | | | | |
|
||||
VS1 VS2 VS3 VS4 VS5 VS6 VS7 VS8
|
||||
```
|
||||
|
||||
More volume servers = more EC layouts available (e.g. 10+4, 6+3).
|
||||
|
||||
---
|
||||
|
||||
# Getting Started Checklist
|
||||
|
||||
1. Provision 3 master nodes (small boxes are fine)
|
||||
2. Provision volume servers sized to your data footprint
|
||||
3. Stand up an HA metadata DB, then 3 filers pointing at it
|
||||
4. Add S3 servers sized to required throughput
|
||||
5. Run 1 admin + a few workers for background maintenance
|
||||
6. Once you have enough volume servers, enable EC (e.g. 5+3 on 8 servers)
|
||||
|
||||
Scale each layer independently as usage grows.
|
||||
@@ -22,9 +22,6 @@ service SeaweedFiler {
|
||||
rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
|
||||
}
|
||||
|
||||
rpc TouchAccessTime (TouchAccessTimeRequest) returns (TouchAccessTimeResponse) {
|
||||
}
|
||||
|
||||
rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
|
||||
}
|
||||
|
||||
@@ -89,13 +86,6 @@ service SeaweedFiler {
|
||||
}
|
||||
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
|
||||
}
|
||||
|
||||
// Peer chunk sharing — tier 1: mount-server registry.
|
||||
// See design-weed-mount-peer-chunk-sharing.md for details.
|
||||
rpc MountRegister (MountRegisterRequest) returns (MountRegisterResponse) {
|
||||
}
|
||||
rpc MountList (MountListRequest) returns (MountListResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
@@ -207,12 +197,6 @@ message FuseAttributes {
|
||||
bytes md5 = 14;
|
||||
uint32 rdev = 16;
|
||||
uint64 inode = 17;
|
||||
int64 ctime = 18; // unix time in seconds, inode change time
|
||||
int32 mtime_ns = 19; // nanosecond component of mtime (0-999999999)
|
||||
int32 ctime_ns = 20; // nanosecond component of ctime (0-999999999)
|
||||
int32 crtime_ns = 21; // nanosecond component of crtime (0-999999999)
|
||||
int64 atime = 22; // unix time in seconds, last access time
|
||||
int32 atime_ns = 23; // nanosecond component of atime (0-999999999)
|
||||
}
|
||||
|
||||
message CreateEntryRequest {
|
||||
@@ -252,16 +236,6 @@ message UpdateEntryResponse {
|
||||
SubscribeMetadataResponse metadata_event = 1;
|
||||
}
|
||||
|
||||
message TouchAccessTimeRequest {
|
||||
string directory = 1;
|
||||
string name = 2;
|
||||
int64 client_atime_ns = 3; // nanoseconds since epoch; filer may override with relatime
|
||||
}
|
||||
message TouchAccessTimeResponse {
|
||||
int64 persisted_atime_ns = 1; // nanoseconds since epoch; 0 if no update was performed
|
||||
bool updated = 2;
|
||||
}
|
||||
|
||||
message AppendToEntryRequest {
|
||||
string directory = 1;
|
||||
string entry_name = 2;
|
||||
@@ -320,7 +294,6 @@ message AssignVolumeRequest {
|
||||
string rack = 7;
|
||||
string data_node = 9;
|
||||
string disk_type = 8;
|
||||
uint64 expected_data_size = 10; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message AssignVolumeResponse {
|
||||
@@ -623,31 +596,3 @@ message StreamMutateEntryResponse {
|
||||
string error = 7; // human-readable error message when the operation failed
|
||||
int32 errno = 8; // POSIX errno (e.g. ENOENT=2, ENOTEMPTY=66) for direct FUSE status mapping
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Peer chunk sharing — mount-server registry
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
message MountRegisterRequest {
|
||||
string peer_addr = 1; // host:port where this mount serves peer chunk requests
|
||||
string rack = 2; // locality label (rack); used for peer ranking
|
||||
int32 ttl_seconds = 3; // how long the filer should keep this entry without a heartbeat
|
||||
string data_center = 4; // locality label (data center); coarser than rack
|
||||
}
|
||||
|
||||
message MountRegisterResponse {
|
||||
}
|
||||
|
||||
message MountListRequest {
|
||||
}
|
||||
|
||||
message MountListResponse {
|
||||
repeated MountInfo mounts = 1;
|
||||
}
|
||||
|
||||
message MountInfo {
|
||||
string peer_addr = 1;
|
||||
string rack = 2;
|
||||
int64 last_seen_ns = 3;
|
||||
string data_center = 4;
|
||||
}
|
||||
|
||||
Generated
+2
-2
@@ -3350,9 +3350,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.13"
|
||||
version = "0.103.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
|
||||
@@ -145,11 +145,6 @@ message VolumeEcShardInformationMessage {
|
||||
uint64 expire_at_sec = 5; // used to record the destruction time of ec volume
|
||||
uint32 disk_id = 6;
|
||||
repeated int64 shard_sizes = 7; // optimized: sizes for shards in order of set bits in ec_index_bits
|
||||
uint64 file_count = 8; // total needles in the .ecx index (live + tombstoned)
|
||||
uint64 delete_count = 9; // node-local tombstones in the .ecj deletion journal
|
||||
// fields 10-19 reserved for future upstream open-source additions.
|
||||
// fields 20+ are owned by the enterprise fork (e.g. data_shards/parity_shards)
|
||||
// and must not be used here without coordination.
|
||||
}
|
||||
|
||||
message StorageBackend {
|
||||
@@ -236,7 +231,6 @@ message AssignRequest {
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
uint32 writable_volume_count = 9;
|
||||
string disk_type = 10;
|
||||
uint64 expected_data_size = 11; // hint for size-aware volume selection
|
||||
}
|
||||
|
||||
message VolumeGrowRequest {
|
||||
|
||||
@@ -247,9 +247,6 @@ message VolumeUnmountResponse {
|
||||
message VolumeDeleteRequest {
|
||||
uint32 volume_id = 1;
|
||||
bool only_empty = 2;
|
||||
// when true, do not remove the cloud-tier object backing the volume.
|
||||
// used for moves where another server is taking over the same .vif.
|
||||
bool keep_remote_data = 3;
|
||||
}
|
||||
message VolumeDeleteResponse {
|
||||
}
|
||||
@@ -342,7 +339,6 @@ message ReceiveFileInfo {
|
||||
bool is_ec_volume = 4;
|
||||
uint32 shard_id = 5;
|
||||
uint64 file_size = 6;
|
||||
uint32 disk_id = 7; // EC shard disk; 0 = auto-select (see VolumeEcShardsCopyRequest.disk_id)
|
||||
}
|
||||
|
||||
message ReceiveFileResponse {
|
||||
@@ -458,7 +454,6 @@ message VolumeEcShardsMountRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
|
||||
}
|
||||
message VolumeEcShardsMountResponse {
|
||||
}
|
||||
@@ -641,7 +636,6 @@ message FetchAndWriteNeedleRequest {
|
||||
}
|
||||
repeated Replica replicas = 6;
|
||||
string auth = 7;
|
||||
int32 download_concurrency = 8; // multipart download concurrency if supported by the remote storage client; for S3, 0 = default (5)
|
||||
// remote conf
|
||||
remote_pb.RemoteConf remote_conf = 15;
|
||||
remote_pb.RemoteStorageLocation remote_location = 16;
|
||||
|
||||
@@ -63,9 +63,7 @@ pub struct Cli {
|
||||
#[arg(long = "rack", default_value = "")]
|
||||
pub rack: String,
|
||||
|
||||
/// Choose [memory|redb|redbMedium|redbLarge] mode for memory~performance balance.
|
||||
/// `leveldb`/`leveldbMedium`/`leveldbLarge` are accepted as aliases for the
|
||||
/// corresponding redb backends (Rust volume server uses redb under the hood).
|
||||
/// Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.
|
||||
#[arg(long = "index", default_value = "memory")]
|
||||
pub index: String,
|
||||
|
||||
@@ -703,16 +701,14 @@ fn resolve_config(cli: Cli) -> VolumeServerConfig {
|
||||
ip.clone()
|
||||
};
|
||||
|
||||
// Parse index type. Accept both `redb*` (preferred — what the volume server
|
||||
// actually uses) and the legacy `leveldb*` names as aliases.
|
||||
// Parse index type
|
||||
let index_type = match cli.index.as_str() {
|
||||
"memory" => NeedleMapKind::InMemory,
|
||||
"redb" | "leveldb" => NeedleMapKind::Redb,
|
||||
"redbMedium" | "leveldbMedium" => NeedleMapKind::RedbMedium,
|
||||
"redbLarge" | "leveldbLarge" => NeedleMapKind::RedbLarge,
|
||||
"leveldb" => NeedleMapKind::LevelDb,
|
||||
"leveldbMedium" => NeedleMapKind::LevelDbMedium,
|
||||
"leveldbLarge" => NeedleMapKind::LevelDbLarge,
|
||||
other => panic!(
|
||||
"Unknown index type: {}. Use memory|redb|redbMedium|redbLarge \
|
||||
(leveldb/leveldbMedium/leveldbLarge accepted as aliases)",
|
||||
"Unknown index type: {}. Use memory|leveldb|leveldbMedium|leveldbLarge",
|
||||
other
|
||||
),
|
||||
};
|
||||
@@ -1378,23 +1374,6 @@ mod tests {
|
||||
assert_eq!(cfg.folders, vec![default_volume_dir()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_config_index_accepts_redb_and_leveldb_aliases() {
|
||||
let pairs = [
|
||||
("memory", NeedleMapKind::InMemory),
|
||||
("redb", NeedleMapKind::Redb),
|
||||
("leveldb", NeedleMapKind::Redb),
|
||||
("redbMedium", NeedleMapKind::RedbMedium),
|
||||
("leveldbMedium", NeedleMapKind::RedbMedium),
|
||||
("redbLarge", NeedleMapKind::RedbLarge),
|
||||
("leveldbLarge", NeedleMapKind::RedbLarge),
|
||||
];
|
||||
for (input, expected) in pairs {
|
||||
let cfg = resolve_config(Cli::parse_from(["bin", "--index", input]));
|
||||
assert_eq!(cfg.index_type, expected, "input={}", input);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_security_config_access_ui() {
|
||||
let _guard = process_state_lock();
|
||||
|
||||
@@ -288,10 +288,6 @@ async fn run(
|
||||
config.jwt_read_signing_expires_seconds,
|
||||
);
|
||||
let master_url = config.masters.first().cloned().unwrap_or_default();
|
||||
// Defensive-copy the configured seed masters before freezing the lookup
|
||||
// set, so any later mutation of config.masters cannot desync them.
|
||||
let master_urls: Vec<String> = config.masters.clone();
|
||||
let seed_master_set = VolumeServerState::build_seed_master_set(&master_urls);
|
||||
let self_url = format!("{}:{}", config.ip, config.port);
|
||||
let (http_client, outgoing_http_scheme) = build_outgoing_http_client(&config)?;
|
||||
let outgoing_grpc_tls = load_outgoing_grpc_tls(&config)?;
|
||||
@@ -328,9 +324,7 @@ async fn run(
|
||||
),
|
||||
read_mode: config.read_mode,
|
||||
master_url,
|
||||
master_urls,
|
||||
seed_master_set,
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
master_urls: config.masters.clone(),
|
||||
self_url,
|
||||
http_client,
|
||||
outgoing_http_scheme,
|
||||
|
||||
@@ -142,35 +142,6 @@ lazy_static::lazy_static! {
|
||||
&["code"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Scrubbing metrics (Go: VolumeServerScrub*) ----
|
||||
|
||||
/// Last scrub execution time, as seconds since UNIX epoch, with label `mode`.
|
||||
pub static ref SCRUB_LAST_TIME_SECONDS: GaugeVec = GaugeVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_last_time_seconds",
|
||||
"Last scrub execution time, as seconds since UNIX epoch.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Counter of overall volumes with issues detected during scrubbing, with label `mode`.
|
||||
pub static ref SCRUB_VOLUME_FAILURES: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_volume_failures",
|
||||
"Counter of overall volumes with issues detected during scrubbing.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
/// Counter of overall EC shards with issues detected during scrubbing, with label `mode`.
|
||||
pub static ref SCRUB_SHARD_FAILURES: IntCounterVec = IntCounterVec::new(
|
||||
Opts::new(
|
||||
"SeaweedFS_volumeServer_scrub_shard_failures",
|
||||
"Counter of overall EC shards with issues detected during scrubbing.",
|
||||
),
|
||||
&["mode"],
|
||||
).expect("metric can be created");
|
||||
|
||||
// ---- Legacy aliases for backward compat with existing code ----
|
||||
|
||||
/// Total number of volumes on this server (flat gauge).
|
||||
@@ -280,9 +251,6 @@ pub fn register_metrics() {
|
||||
Box::new(INFLIGHT_DOWNLOAD_SIZE.clone()),
|
||||
Box::new(INFLIGHT_UPLOAD_SIZE.clone()),
|
||||
Box::new(UPLOAD_ERROR_COUNTER.clone()),
|
||||
Box::new(SCRUB_LAST_TIME_SECONDS.clone()),
|
||||
Box::new(SCRUB_VOLUME_FAILURES.clone()),
|
||||
Box::new(SCRUB_SHARD_FAILURES.clone()),
|
||||
// Legacy metrics
|
||||
Box::new(VOLUMES_TOTAL.clone()),
|
||||
Box::new(DISK_SIZE_BYTES.clone()),
|
||||
|
||||
@@ -486,11 +486,6 @@ impl S3TierRegistry {
|
||||
self.backends.keys().cloned().collect()
|
||||
}
|
||||
|
||||
/// Remove a backend by name.
|
||||
pub fn remove(&mut self, name: &str) {
|
||||
self.backends.remove(name);
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.backends.clear();
|
||||
}
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.6 KiB After Width: | Height: | Size: 70 B |
@@ -107,57 +107,6 @@ pub fn build_grpc_endpoint(
|
||||
Ok(endpoint)
|
||||
}
|
||||
|
||||
/// Parse a SeaweedFS server address (`"ip:port.grpcPort"` or
|
||||
/// `"ip:port"`) into the `host:grpcPort` form `build_grpc_endpoint`
|
||||
/// expects. With the trailing `.grpcPort` segment, that segment IS
|
||||
/// the gRPC port; without it, the gRPC port is `port + 10000`
|
||||
/// (SeaweedFS's HTTP↔gRPC port-offset convention).
|
||||
///
|
||||
/// Shared between `grpc_server.rs` and the distributed-EC-read path
|
||||
/// in `store_ec.rs` — keep this as the single source of truth so the
|
||||
/// HTTP↔gRPC port translation can't drift between callers.
|
||||
pub fn parse_grpc_address(source: &str) -> Result<String, String> {
|
||||
let colon_idx = source
|
||||
.rfind(':')
|
||||
.ok_or_else(|| format!("cannot parse address: {}", source))?;
|
||||
let host = &source[..colon_idx];
|
||||
let port_part = &source[colon_idx + 1..];
|
||||
|
||||
if let Some(dot_idx) = port_part.rfind('.') {
|
||||
// Format: "ip:port.grpcPort". Validate BOTH ports as u16
|
||||
// so a malformed HTTP port (e.g. `host:abc.18080`) is
|
||||
// rejected here rather than tripping a downstream
|
||||
// `build_grpc_endpoint` URI parse failure with a less
|
||||
// useful error.
|
||||
let http_port = &port_part[..dot_idx];
|
||||
let grpc_port = &port_part[dot_idx + 1..];
|
||||
http_port
|
||||
.parse::<u16>()
|
||||
.map_err(|e| format!("invalid http port {:?}: {}", http_port, e))?;
|
||||
grpc_port
|
||||
.parse::<u16>()
|
||||
.map_err(|e| format!("invalid grpc port {:?}: {}", grpc_port, e))?;
|
||||
return Ok(format!("{}:{}", host, grpc_port));
|
||||
}
|
||||
|
||||
// Format: "ip:port" → grpc = port + 10000. Reject inputs whose
|
||||
// implicit grpc port would overflow the TCP port range (e.g.
|
||||
// `host:60000` produces 70000 — invalid). Without this check
|
||||
// the cast silently wraps and the endpoint call later fails
|
||||
// with an opaque connection error.
|
||||
let port: u16 = port_part
|
||||
.parse()
|
||||
.map_err(|e| format!("invalid port {:?}: {}", port_part, e))?;
|
||||
let grpc_port = port as u32 + 10000;
|
||||
if grpc_port > u16::MAX as u32 {
|
||||
return Err(format!(
|
||||
"implicit grpc port out of range: {} + 10000 = {}",
|
||||
port, grpc_port
|
||||
));
|
||||
}
|
||||
Ok(format!("{}:{}", host, grpc_port))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{build_grpc_endpoint, grpc_endpoint_uri, load_outgoing_grpc_tls};
|
||||
@@ -254,50 +203,4 @@ mod tests {
|
||||
let endpoint = build_grpc_endpoint("127.0.0.1:19333", None).unwrap();
|
||||
assert_eq!(endpoint.uri().scheme_str(), Some("http"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_dotted_form() {
|
||||
use super::parse_grpc_address;
|
||||
assert_eq!(
|
||||
parse_grpc_address("127.0.0.1:8080.18080").unwrap(),
|
||||
"127.0.0.1:18080"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_implicit_form_adds_10000() {
|
||||
use super::parse_grpc_address;
|
||||
assert_eq!(
|
||||
parse_grpc_address("127.0.0.1:8080").unwrap(),
|
||||
"127.0.0.1:18080"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_non_numeric_http_port_in_dotted_form() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("host:abc.18080").unwrap_err();
|
||||
assert!(err.contains("invalid http port"), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_non_numeric_grpc_port_in_dotted_form() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("host:8080.xyz").unwrap_err();
|
||||
assert!(err.contains("invalid grpc port"), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_implicit_port_that_overflows() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("127.0.0.1:60000").unwrap_err();
|
||||
assert!(err.contains("out of range"), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_grpc_address_rejects_input_without_colon() {
|
||||
use super::parse_grpc_address;
|
||||
let err = parse_grpc_address("hostname").unwrap_err();
|
||||
assert!(err.contains("cannot parse"), "{}", err);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ use crate::pb::master_pb;
|
||||
use crate::pb::master_pb::seaweed_client::SeaweedClient;
|
||||
use crate::pb::volume_server_pb;
|
||||
use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
use crate::storage::needle::needle::{self, Needle};
|
||||
use crate::storage::types::*;
|
||||
|
||||
@@ -29,42 +28,6 @@ fn volume_is_remote_only(dat_path: &str, has_remote_file: bool) -> bool {
|
||||
has_remote_file && !std::path::Path::new(dat_path).exists()
|
||||
}
|
||||
|
||||
/// Map a numeric `VolumeScrubMode` to its proto enum name, matching Go's
|
||||
/// `req.GetMode().String()` used for the Prometheus `mode` label.
|
||||
fn scrub_mode_label(mode: i32) -> &'static str {
|
||||
match mode {
|
||||
0 => "UNKNOWN",
|
||||
1 => "INDEX",
|
||||
2 => "FULL",
|
||||
3 => "LOCAL",
|
||||
_ => "UNKNOWN",
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_now_seconds() -> f64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as f64)
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// Record scrub metrics. `broken_shards` is `Some` only for EC scrubs so the
|
||||
/// shard-failures family stays untouched on regular volume scrubs (matching Go).
|
||||
fn emit_scrub_metrics(mode: i32, broken_volumes: usize, broken_shards: Option<usize>) {
|
||||
let mode_label = scrub_mode_label(mode);
|
||||
crate::metrics::SCRUB_LAST_TIME_SECONDS
|
||||
.with_label_values(&[mode_label])
|
||||
.set(unix_now_seconds());
|
||||
crate::metrics::SCRUB_VOLUME_FAILURES
|
||||
.with_label_values(&[mode_label])
|
||||
.inc_by(broken_volumes as u64);
|
||||
if let Some(n) = broken_shards {
|
||||
crate::metrics::SCRUB_SHARD_FAILURES
|
||||
.with_label_values(&[mode_label])
|
||||
.inc_by(n as u64);
|
||||
}
|
||||
}
|
||||
|
||||
/// Persist VolumeServerState to a state.pb file (matches Go's State.save).
|
||||
fn save_state_file(
|
||||
path: &str,
|
||||
@@ -145,37 +108,6 @@ pub struct VolumeGrpcService {
|
||||
}
|
||||
|
||||
impl VolumeGrpcService {
|
||||
/// Verifies the gRPC caller is allowed to invoke a destructive admin
|
||||
/// operation. Mirrors the Go side's checkGrpcAdminAuth: an empty
|
||||
/// whitelist accepts everyone (insecure-by-default for tests and
|
||||
/// upgrades), a populated whitelist accepts only matching peer IPs.
|
||||
///
|
||||
/// `remote_addr()` on a real gRPC connection always yields the peer's
|
||||
/// SocketAddr; if it is somehow None we deny, matching "if we don't
|
||||
/// know who the caller is, refuse."
|
||||
fn check_grpc_admin_auth<T>(&self, request: &Request<T>) -> Result<(), Status> {
|
||||
let remote = match request.remote_addr() {
|
||||
Some(addr) => addr,
|
||||
None => {
|
||||
tracing::warn!("gRPC admin auth failed: no peer info");
|
||||
return Err(Status::permission_denied("no peer info"));
|
||||
}
|
||||
};
|
||||
let host = remote.ip().to_string();
|
||||
let guard = self.state.guard.read().unwrap();
|
||||
if !guard.check_whitelist(&host) {
|
||||
tracing::warn!(
|
||||
"gRPC admin auth failed: {} is not whitelisted (remote: {})",
|
||||
host,
|
||||
remote,
|
||||
);
|
||||
return Err(Status::permission_denied(format!(
|
||||
"not authorized: {host}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn notify_master_volume_readonly(
|
||||
&self,
|
||||
info: &MasterVolumeInfo,
|
||||
@@ -490,7 +422,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCompactRequest>,
|
||||
) -> Result<Response<Self::VacuumVolumeCompactStream>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -550,7 +481,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCommitRequest>,
|
||||
) -> Result<Response<volume_server_pb::VacuumVolumeCommitResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
|
||||
@@ -586,7 +516,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCleanupRequest>,
|
||||
) -> Result<Response<volume_server_pb::VacuumVolumeCleanupResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
@@ -602,7 +531,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::DeleteCollectionRequest>,
|
||||
) -> Result<Response<volume_server_pb::DeleteCollectionResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let collection = &request.into_inner().collection;
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
store
|
||||
@@ -615,7 +543,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::AllocateVolumeRequest>,
|
||||
) -> Result<Response<volume_server_pb::AllocateVolumeResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -764,7 +691,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMountRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMountResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
@@ -781,7 +707,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeUnmountRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeUnmountResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
// Go returns nil when volume is not found (idempotent unmount)
|
||||
@@ -795,7 +720,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeDeleteRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeDeleteResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -809,7 +733,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
store
|
||||
.delete_volume(vid, req.only_empty, req.keep_remote_data)
|
||||
.delete_volume(vid, req.only_empty)
|
||||
.map_err(|e| Status::internal(e.to_string()))?;
|
||||
self.state.volume_state_notify.notify_one();
|
||||
Ok(Response::new(volume_server_pb::VolumeDeleteResponse {}))
|
||||
@@ -819,7 +743,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMarkReadonlyRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMarkReadonlyResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
// Go: volume lookup (L239-241) happens before maintenance check (L166 in makeVolumeReadonly)
|
||||
@@ -839,7 +762,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMarkWritableRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMarkWritableResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
let info = {
|
||||
@@ -889,7 +811,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeConfigureRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeConfigureResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -1043,10 +964,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
if store.find_volume(vid).is_some() {
|
||||
drop(store);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
// keep remote data: the inbound copy carries a .vif that may
|
||||
// point at the same cloud-tier object the existing volume
|
||||
// references.
|
||||
store.delete_volume(vid, false, true).map_err(|e| {
|
||||
store.delete_volume(vid, false).map_err(|e| {
|
||||
Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
|
||||
})?;
|
||||
self.state.volume_state_notify.notify_one();
|
||||
@@ -1507,47 +1425,13 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// Determine file path
|
||||
let path = if info.is_ec_volume {
|
||||
let store = self.state.store.read().unwrap();
|
||||
// std::fs::File::create truncates in place; a mounted
|
||||
// EcVolume holds fds on the same inodes, so overwriting
|
||||
// corrupts live readers.
|
||||
if store.has_ec_volume(VolumeId(info.volume_id)) {
|
||||
let mounted_disks =
|
||||
store.find_ec_volume_disk_ids(VolumeId(info.volume_id));
|
||||
resp_error = Some(format!(
|
||||
"ec volume {} is mounted on disk_ids:{:?}; unmount before ReceiveFile",
|
||||
info.volume_id, mounted_disks
|
||||
));
|
||||
break;
|
||||
}
|
||||
// disk_id=0 means "unset" (protobuf default), so auto-select
|
||||
// using the same primitive as volume_ec_shards_copy: prefer
|
||||
// a disk that has the EC volume mounted, then a disk that
|
||||
// owns the .ecx on disk (volume not yet mounted — relevant
|
||||
// when shards stream in mid-rebuild before any
|
||||
// VolumeEcShardsMount has happened; see #9212), then any
|
||||
// HDD, then any disk. Pass the build's default data-shard
|
||||
// count for free-slot maths; the helper takes it as a
|
||||
// parameter so custom-ratio builds can swap it.
|
||||
let vid = VolumeId(info.volume_id);
|
||||
let dir = if info.disk_id > 0 {
|
||||
let count = store.locations.len();
|
||||
if (info.disk_id as usize) >= count {
|
||||
resp_error = Some(format!(
|
||||
"invalid disk_id {}: only have {} disks",
|
||||
info.disk_id, count
|
||||
));
|
||||
break;
|
||||
}
|
||||
Some(store.locations[info.disk_id as usize].directory.clone())
|
||||
} else {
|
||||
store
|
||||
.find_ec_shard_target_location(
|
||||
&info.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
)
|
||||
.map(|i| store.locations[i].directory.clone())
|
||||
};
|
||||
// Go prefers a HardDriveType location, then falls back to first
|
||||
let dir = store
|
||||
.locations
|
||||
.iter()
|
||||
.find(|loc| loc.disk_type == DiskType::HardDrive)
|
||||
.or_else(|| store.locations.first())
|
||||
.map(|loc| loc.directory.clone());
|
||||
drop(store);
|
||||
let dir = match dir {
|
||||
Some(d) => d,
|
||||
@@ -2099,9 +1983,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
// Check existing .vif for EC shard config (matching Go's MaybeLoadVolumeInfo)
|
||||
let (data_shards, parity_shards) =
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
|
||||
&dir, &idx_dir, collection, vid,
|
||||
);
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(&dir, collection, vid);
|
||||
|
||||
if let Err(e) = crate::storage::erasure_coding::ec_encoder::write_ec_files(
|
||||
&dir,
|
||||
@@ -2255,7 +2137,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let (data_shards, parity_shards) =
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
|
||||
&rebuild_dir,
|
||||
&rebuild_idx_dir,
|
||||
collection,
|
||||
vid,
|
||||
);
|
||||
@@ -2339,17 +2220,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
// Select target location:
|
||||
// When disk_id > 0: use that specific location.
|
||||
// When disk_id == 0 (unset): auto-select via
|
||||
// find_ec_shard_target_location, which prefers a disk that
|
||||
// already has the EC volume mounted, then a disk that owns the
|
||||
// .ecx on disk (volume not yet mounted — relevant for
|
||||
// ec.rebuild, where only the first shard carries .ecx and
|
||||
// subsequent shards must land on the same disk; see #9212),
|
||||
// then any HDD, then any disk. Pass the build's default
|
||||
// data-shard count; the helper takes it as a parameter so
|
||||
// custom-ratio builds can swap it.
|
||||
// Select target location matching Go's 3-tier fallback:
|
||||
// When disk_id > 0: use that specific location
|
||||
// When disk_id == 0 (unset): (1) location with existing EC shards, (2) any HDD, (3) any
|
||||
let (dest_dir, dest_idx_dir) = {
|
||||
let store = self.state.store.read().unwrap();
|
||||
let count = store.locations.len();
|
||||
@@ -2365,11 +2238,20 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let loc = &store.locations[req.disk_id as usize];
|
||||
(loc.directory.clone(), loc.idx_directory.clone())
|
||||
} else {
|
||||
match store.find_ec_shard_target_location(
|
||||
&req.collection,
|
||||
vid,
|
||||
DATA_SHARDS_COUNT as u32,
|
||||
) {
|
||||
// Auto-select: prefer location with existing EC shards for this volume
|
||||
let loc_idx = store
|
||||
.find_free_location_predicate(|loc| loc.has_ec_volume(vid))
|
||||
.or_else(|| {
|
||||
// Fall back to any HDD location
|
||||
store.find_free_location_predicate(|loc| {
|
||||
loc.disk_type == DiskType::HardDrive
|
||||
})
|
||||
})
|
||||
.or_else(|| {
|
||||
// Fall back to any location
|
||||
store.find_free_location_predicate(|_| true)
|
||||
});
|
||||
match loc_idx {
|
||||
Some(i) => {
|
||||
let loc = &store.locations[i];
|
||||
(loc.directory.clone(), loc.idx_directory.clone())
|
||||
@@ -2589,7 +2471,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeEcShardsDeleteRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeEcShardsDeleteResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -2614,7 +2495,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
for &shard_id in &req.shard_ids {
|
||||
store
|
||||
.mount_ec_shard(vid, &req.collection, shard_id, &req.source_disk_type)
|
||||
.mount_ec_shard(vid, &req.collection, shard_id)
|
||||
.map_err(|e| {
|
||||
Status::internal(format!("mount {}.{}: {}", req.volume_id, shard_id, e))
|
||||
})?;
|
||||
@@ -2658,19 +2539,12 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
let store = self.state.store.read().unwrap();
|
||||
// Reconciled EC volumes can have their shards split across
|
||||
// disks (e.g. shards 0/12 on disk 0, shard 1 on disk 1), so
|
||||
// resolve the EcVolume from the *shard*'s home location
|
||||
// rather than first-match `find_ec_volume(vid)` which would
|
||||
// miss shards that live on a sibling. Mirrors Go's findEcShard.
|
||||
let ec_vol = store
|
||||
.find_ec_volume_with_shard(vid, req.shard_id)
|
||||
.ok_or_else(|| {
|
||||
Status::not_found(format!(
|
||||
"ec volume {} shard {} not found",
|
||||
req.volume_id, req.shard_id
|
||||
))
|
||||
})?;
|
||||
let ec_vol = store.find_ec_volume(vid).ok_or_else(|| {
|
||||
Status::not_found(format!(
|
||||
"ec volume {} shard {} not found",
|
||||
req.volume_id, req.shard_id
|
||||
))
|
||||
})?;
|
||||
|
||||
// Check if the requested needle is deleted (via .ecx index, matching Go)
|
||||
if req.file_key > 0 {
|
||||
@@ -2689,8 +2563,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
|
||||
// Read from the shard. Guaranteed to be present because
|
||||
// find_ec_volume_with_shard already verified it.
|
||||
// Read from the shard
|
||||
let shard = ec_vol
|
||||
.shards
|
||||
.get(req.shard_id as usize)
|
||||
@@ -2780,13 +2653,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
let store = self.state.store.read().unwrap();
|
||||
// Aggregate per-shard data dirs across all locations so the
|
||||
// shard-presence check + decoder both see the union for
|
||||
// cross-disk reconciled volumes (#9252). Mirrors Go's
|
||||
// CollectEcShards.
|
||||
let max_shard_count = crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT;
|
||||
let (ec_vol, shard_dirs) = store
|
||||
.collect_ec_shard_dirs(vid, max_shard_count)
|
||||
let ec_vol = store
|
||||
.find_ec_volume(vid)
|
||||
.ok_or_else(|| Status::not_found(format!("ec volume {} not found", req.volume_id)))?;
|
||||
|
||||
if ec_vol.collection != req.collection {
|
||||
@@ -2800,6 +2668,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let data_shards = ec_vol.data_shards as usize;
|
||||
|
||||
// Validate data shard count range (matches Go's VolumeEcShardsToVolume)
|
||||
let max_shard_count = crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT;
|
||||
if data_shards == 0 || data_shards > max_shard_count {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"invalid data shard count {} for volume {} (must be 1..{})",
|
||||
@@ -2807,9 +2676,14 @@ impl VolumeServer for VolumeGrpcService {
|
||||
)));
|
||||
}
|
||||
|
||||
// Check that all data shards are present somewhere on this server.
|
||||
// Check that all data shards are present
|
||||
for shard_id in 0..data_shards {
|
||||
if shard_dirs[shard_id].is_none() {
|
||||
if ec_vol
|
||||
.shards
|
||||
.get(shard_id)
|
||||
.map(|s| s.is_none())
|
||||
.unwrap_or(true)
|
||||
{
|
||||
return Err(Status::internal(format!(
|
||||
"ec volume {} missing shard {}",
|
||||
req.volume_id, shard_id
|
||||
@@ -2841,46 +2715,29 @@ impl VolumeServer for VolumeGrpcService {
|
||||
)));
|
||||
}
|
||||
|
||||
// Reconstruct the volume from EC shards. Use the EcVolume's
|
||||
// own dir for the produced .dat (matches the volume's home
|
||||
// disk) and its `ecx_actual_dir` for the .ecx lookup, while
|
||||
// reading each shard from its real on-disk location.
|
||||
let dat_dir = ec_vol.dir.clone();
|
||||
let ecx_dir = ec_vol.ecx_actual_dir().to_string();
|
||||
// Reconstruct the volume from EC shards
|
||||
let dir = ec_vol.dir.clone();
|
||||
let collection = ec_vol.collection.clone();
|
||||
// shard_dirs[i] is guaranteed Some for i in 0..data_shards by
|
||||
// the check above; collect concrete dirs for the decoder.
|
||||
let per_shard_dirs: Vec<String> = shard_dirs[..data_shards]
|
||||
.iter()
|
||||
.map(|d| d.clone().unwrap())
|
||||
.collect();
|
||||
drop(store);
|
||||
|
||||
// Calculate .dat file size from .ecx entries (.ec00 lives on
|
||||
// its own disk, .ecx on the index disk).
|
||||
// Calculate .dat file size from .ecx entries
|
||||
let dat_file_size =
|
||||
crate::storage::erasure_coding::ec_decoder::find_dat_file_size_with_dirs(
|
||||
&per_shard_dirs[0],
|
||||
&ecx_dir,
|
||||
&collection,
|
||||
vid,
|
||||
)
|
||||
.map_err(|e| Status::internal(format!("FindDatFileSize: {}", e)))?;
|
||||
crate::storage::erasure_coding::ec_decoder::find_dat_file_size(&dir, &collection, vid)
|
||||
.map_err(|e| Status::internal(format!("FindDatFileSize: {}", e)))?;
|
||||
|
||||
// Write .dat file using block-interleaved reading from shards.
|
||||
crate::storage::erasure_coding::ec_decoder::write_dat_file_from_shards_with_dirs(
|
||||
&dat_dir,
|
||||
// Write .dat file using block-interleaved reading from shards
|
||||
crate::storage::erasure_coding::ec_decoder::write_dat_file_from_shards(
|
||||
&dir,
|
||||
&collection,
|
||||
vid,
|
||||
dat_file_size,
|
||||
data_shards,
|
||||
&per_shard_dirs,
|
||||
)
|
||||
.map_err(|e| Status::internal(format!("WriteDatFile: {}", e)))?;
|
||||
|
||||
// Write .idx file from .ecx and .ecj files (lives on idx dir).
|
||||
// Write .idx file from .ecx and .ecj files
|
||||
crate::storage::erasure_coding::ec_decoder::write_idx_file_from_ec_index(
|
||||
&dat_dir,
|
||||
&dir,
|
||||
&collection,
|
||||
vid,
|
||||
)
|
||||
@@ -3288,9 +3145,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
async fn volume_server_leave(
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeServerLeaveRequest>,
|
||||
_request: Request<volume_server_pb::VolumeServerLeaveRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeServerLeaveResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
*self.state.is_stopping.write().unwrap() = true;
|
||||
self.state.is_heartbeating.store(false, Ordering::Relaxed);
|
||||
// Wake heartbeat loop to send deregistration.
|
||||
@@ -3440,7 +3296,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::ScrubVolumeRequest>,
|
||||
) -> Result<Response<volume_server_pb::ScrubVolumeResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
|
||||
// Validate mode
|
||||
@@ -3505,8 +3360,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
// Match Go: if mark_broken_volumes_readonly, call makeVolumeReadonly on each broken volume.
|
||||
// Collect errors via errors.Join semantics (return joined error if any fail).
|
||||
let mut errs: Vec<String> = Vec::new();
|
||||
if req.mark_broken_volumes_readonly {
|
||||
let mut errs: Vec<String> = Vec::new();
|
||||
for vid in &broken_vids {
|
||||
match self.make_volume_readonly(*vid, true).await {
|
||||
Ok(()) => {
|
||||
@@ -3518,14 +3373,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Record metrics before the post-scrub error check so scrub failures are
|
||||
// persisted even when a follow-up admin action (mark-readonly) fails.
|
||||
emit_scrub_metrics(mode, broken_vids.len(), None);
|
||||
|
||||
if !errs.is_empty() {
|
||||
return Err(Status::internal(errs.join("\n")));
|
||||
if !errs.is_empty() {
|
||||
return Err(Status::internal(errs.join("\n")));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Response::new(volume_server_pb::ScrubVolumeResponse {
|
||||
@@ -3595,15 +3445,9 @@ impl VolumeServer for VolumeGrpcService {
|
||||
// LOCAL (2) / FULL (3): verify EC shard data
|
||||
let files = ecv.walk_ecx_stats().map(|(f, _, _)| f).unwrap_or(0);
|
||||
|
||||
// After cross-disk reconciliation, an EcVolume can
|
||||
// legitimately have ecv.dir != ecv.dir_idx (shards
|
||||
// on one disk, .ecx / .ecj / .vif on a sibling).
|
||||
// Use the EcVolume's own dirs rather than collapsing
|
||||
// both args to find_ec_dir's single answer, otherwise
|
||||
// read_ec_shard_config falls back to the wrong .vif
|
||||
// location for split-disk volumes (#9252).
|
||||
let dir = ecv.dir.clone();
|
||||
let idx_dir = ecv.dir_idx.clone();
|
||||
let dir = store
|
||||
.find_ec_dir(*vid, &collection)
|
||||
.unwrap_or_else(|| String::from(""));
|
||||
if dir.is_empty() {
|
||||
continue;
|
||||
}
|
||||
@@ -3613,7 +3457,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let (data_shards, parity_shards) =
|
||||
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
|
||||
&dir,
|
||||
&idx_dir,
|
||||
&collection,
|
||||
*vid,
|
||||
);
|
||||
@@ -3651,12 +3494,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
|
||||
emit_scrub_metrics(
|
||||
mode,
|
||||
broken_volume_ids.len(),
|
||||
Some(broken_shard_infos.len()),
|
||||
);
|
||||
|
||||
Ok(Response::new(volume_server_pb::ScrubEcVolumeResponse {
|
||||
total_volumes,
|
||||
total_files,
|
||||
@@ -3900,24 +3737,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
let start = now_ns();
|
||||
|
||||
// Empty target is a self-liveness probe and stays unauthenticated.
|
||||
// Otherwise gate the dial on cluster membership: volume servers only
|
||||
// know masters, so any other target type is refused. Mirrors Go's
|
||||
// volume_grpc_admin.go Ping admission check. tonic forbids returning
|
||||
// a body alongside an error, so we surface the InvalidArgument status
|
||||
// alone — behaviour-identical to Go's status.Errorf return.
|
||||
if !req.target.is_empty()
|
||||
&& !self
|
||||
.state
|
||||
.is_known_ping_target(&req.target, &req.target_type)
|
||||
.await
|
||||
{
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"unknown ping target {} of type {}",
|
||||
req.target, req.target_type
|
||||
)));
|
||||
}
|
||||
|
||||
// Route ping based on target type (matches Go's volume_grpc_admin.go Ping)
|
||||
let remote_time_ns = if req.target_type == "volumeServer" {
|
||||
match ping_volume_server_target(&req.target, self.state.outgoing_grpc_tls.as_ref())
|
||||
@@ -4053,12 +3872,29 @@ async fn ping_filer_target(
|
||||
Ok(resp.into_inner().start_time_ns)
|
||||
}
|
||||
|
||||
// parse_grpc_address moved to super::grpc_client::parse_grpc_address
|
||||
// for sharing with the distributed-EC-read path in server/store_ec.rs.
|
||||
// In-file callers below still write `parse_grpc_address(...)`; this
|
||||
// `use` makes them resolve to the new home without churning every
|
||||
// call site.
|
||||
use super::grpc_client::parse_grpc_address;
|
||||
/// Parse a SeaweedFS server address ("ip:port.grpcPort" or "ip:port") into a gRPC address.
|
||||
fn parse_grpc_address(source: &str) -> Result<String, String> {
|
||||
if let Some(colon_idx) = source.rfind(':') {
|
||||
let port_part = &source[colon_idx + 1..];
|
||||
if let Some(dot_idx) = port_part.rfind('.') {
|
||||
// Format: "ip:port.grpcPort"
|
||||
let host = &source[..colon_idx];
|
||||
let grpc_port = &port_part[dot_idx + 1..];
|
||||
grpc_port
|
||||
.parse::<u16>()
|
||||
.map_err(|e| format!("invalid grpc port: {}", e))?;
|
||||
return Ok(format!("{}:{}", host, grpc_port));
|
||||
}
|
||||
// Format: "ip:port" → grpc = port + 10000
|
||||
let port: u16 = port_part
|
||||
.parse()
|
||||
.map_err(|e| format!("invalid port: {}", e))?;
|
||||
let grpc_port = port as u32 + 10000;
|
||||
let host = &source[..colon_idx];
|
||||
return Ok(format!("{}:{}", host, grpc_port));
|
||||
}
|
||||
Err(format!("cannot parse address: {}", source))
|
||||
}
|
||||
|
||||
/// Set the modification time of a file from nanoseconds since Unix epoch.
|
||||
fn set_file_mtime(path: &str, modified_ts_ns: i64) {
|
||||
@@ -4356,7 +4192,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -4418,7 +4253,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -4428,8 +4262,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -4475,9 +4307,7 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
let tier_config = S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4489,16 +4319,14 @@ mod tests {
|
||||
};
|
||||
{
|
||||
let mut registry = global_s3_tier_registry().write().unwrap();
|
||||
registry.register(
|
||||
"s3.incr_copy_test".to_string(),
|
||||
S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register("s3.default".to_string(), S3TierBackend::new(&tier_config));
|
||||
registry.register("s3".to_string(), S3TierBackend::new(&tier_config));
|
||||
}
|
||||
|
||||
let vif = crate::storage::volume::VifVolumeInfo {
|
||||
files: vec![crate::storage::volume::VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "incr_copy_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4563,8 +4391,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -4667,8 +4493,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -4711,210 +4535,7 @@ mod tests {
|
||||
assert_eq!(copied, dat_bytes[super_block_size as usize..]);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.incr_copy_test");
|
||||
}
|
||||
|
||||
/// Build a bare-bones service with no on-disk store but a configurable
|
||||
/// seed master set, for Ping admission tests. Matches the structure of
|
||||
/// `make_local_service_with_volume` minus the volume bits.
|
||||
fn make_service_with_seed_masters(seeds: &[&str]) -> (VolumeGrpcService, TempDir) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let master_urls: Vec<String> = seeds.iter().map(|s| (*s).to_string()).collect();
|
||||
let seed_master_set =
|
||||
crate::server::volume_server::VolumeServerState::build_seed_master_set(&master_urls);
|
||||
|
||||
let state = Arc::new(VolumeServerState {
|
||||
store: RwLock::new(store),
|
||||
guard: RwLock::new(Guard::new(
|
||||
&[],
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
)),
|
||||
is_stopping: RwLock::new(false),
|
||||
maintenance: std::sync::atomic::AtomicBool::new(false),
|
||||
state_version: std::sync::atomic::AtomicU32::new(0),
|
||||
concurrent_upload_limit: 0,
|
||||
concurrent_download_limit: 0,
|
||||
inflight_upload_data_timeout: std::time::Duration::from_secs(60),
|
||||
inflight_download_data_timeout: std::time::Duration::from_secs(60),
|
||||
inflight_upload_bytes: std::sync::atomic::AtomicI64::new(0),
|
||||
inflight_download_bytes: std::sync::atomic::AtomicI64::new(0),
|
||||
upload_notify: tokio::sync::Notify::new(),
|
||||
download_notify: tokio::sync::Notify::new(),
|
||||
data_center: String::new(),
|
||||
rack: String::new(),
|
||||
file_size_limit_bytes: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
|
||||
has_master: false,
|
||||
pre_stop_seconds: 0,
|
||||
volume_state_notify: tokio::sync::Notify::new(),
|
||||
write_queue: std::sync::OnceLock::new(),
|
||||
s3_tier_registry: std::sync::RwLock::new(
|
||||
crate::remote_storage::s3_tier::S3TierRegistry::new(),
|
||||
),
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: master_urls.first().cloned().unwrap_or_default(),
|
||||
master_urls,
|
||||
seed_master_set,
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
outgoing_grpc_tls: None,
|
||||
metrics_runtime: std::sync::RwLock::new(
|
||||
crate::server::volume_server::RuntimeMetricsConfig::default(),
|
||||
),
|
||||
metrics_notify: tokio::sync::Notify::new(),
|
||||
fix_jpg_orientation: false,
|
||||
has_slow_read: false,
|
||||
read_buffer_size_bytes: 1024 * 1024,
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
});
|
||||
|
||||
(VolumeGrpcService { state }, tmp)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_empty_target_is_self_probe() {
|
||||
// Empty target stays unauthenticated and returns Ok with timing fields
|
||||
// populated — it is the local liveness probe path.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&[]);
|
||||
let response = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: String::new(),
|
||||
target_type: String::new(),
|
||||
}))
|
||||
.await
|
||||
.expect("empty target ping must succeed");
|
||||
let inner = response.into_inner();
|
||||
assert!(inner.start_time_ns > 0);
|
||||
assert!(inner.stop_time_ns >= inner.start_time_ns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_seed_master_target_passes_admission() {
|
||||
// A target that matches a configured seed master clears admission.
|
||||
// The dial itself may or may not succeed depending on what's listening
|
||||
// on the loopback; either way, the response must not be the
|
||||
// InvalidArgument the gate would surface.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9333".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
assert!(
|
||||
!err.message().contains("unknown ping target"),
|
||||
"admission gate should have allowed this target: {}",
|
||||
err.message()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_unknown_master_target_rejected() {
|
||||
// A master-type target not in the seed list and not the current
|
||||
// master is refused with InvalidArgument.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let err = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9999".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await
|
||||
.expect_err("unknown master target must be rejected");
|
||||
assert_eq!(err.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
"unknown ping target localhost:9999 of type master"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_volume_server_target_always_rejected() {
|
||||
// Volume servers do not maintain a peer-volume list, so volumeServer
|
||||
// pings are refused regardless of address.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let err = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:8080".to_string(),
|
||||
target_type: "volumeServer".to_string(),
|
||||
}))
|
||||
.await
|
||||
.expect_err("volumeServer target must be rejected");
|
||||
assert_eq!(err.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
"unknown ping target localhost:8080 of type volumeServer"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_current_master_target_passes_admission() {
|
||||
// A target that matches the current (post-leader-change) master also
|
||||
// clears admission, even if it is not in the seed list. Pick a port
|
||||
// that is extremely unlikely to be live so the test does not flake on
|
||||
// a developer machine that happens to be running a real master.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
// Simulate the heartbeat goroutine having moved to a new leader.
|
||||
*service.state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "127.0.0.1:1".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
assert!(
|
||||
!err.message().contains("unknown ping target"),
|
||||
"leader-change master should be admitted: {}",
|
||||
err.message()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_target_with_grpc_port_suffix_is_normalised() {
|
||||
// Seed masters in pb.ServerAddress form (`host:port.grpcPort`) must
|
||||
// match a Ping target sent in plain `host:port` form, since the gate
|
||||
// normalises both sides through to_http_address.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333.19333"]);
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9333".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
}
|
||||
global_s3_tier_registry().write().unwrap().clear();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -16,7 +16,7 @@ use axum::response::{IntoResponse, Response};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::grpc_client::{build_grpc_endpoint, GRPC_MAX_MESSAGE_SIZE};
|
||||
use super::volume_server::{normalize_outgoing_http_url, to_http_address, VolumeServerState};
|
||||
use super::volume_server::{normalize_outgoing_http_url, VolumeServerState};
|
||||
use crate::config::ReadMode;
|
||||
use crate::metrics;
|
||||
use crate::pb::volume_server_pb;
|
||||
@@ -357,12 +357,9 @@ async fn lookup_volume(
|
||||
master_url: &str,
|
||||
volume_id: u32,
|
||||
) -> Result<Vec<VolumeLocation>, String> {
|
||||
// master_url may be in SeaweedFS's "host:httpPort.grpcPort" form (mirrors
|
||||
// Go's pb.ServerAddress); strip the gRPC port before building the HTTP URL.
|
||||
let master_http = to_http_address(master_url);
|
||||
let url = normalize_outgoing_http_url(
|
||||
scheme,
|
||||
&format!("{}/dir/lookup?volumeId={}", master_http, volume_id),
|
||||
&format!("{}/dir/lookup?volumeId={}", master_url, volume_id),
|
||||
)?;
|
||||
let resp = client
|
||||
.get(&url)
|
||||
@@ -531,12 +528,9 @@ async fn do_replicated_request(
|
||||
|
||||
let mut futures = Vec::new();
|
||||
for loc in remote_locations {
|
||||
// Defensively strip a `.grpcPort` suffix in case loc.url carries it
|
||||
// (e.g. from an older master); see lookup_volume for the same fix.
|
||||
let peer_http = to_http_address(&loc.url);
|
||||
let url = normalize_outgoing_http_url(
|
||||
&state.outgoing_http_scheme,
|
||||
&format!("{}{}?{}", peer_http, path, new_query),
|
||||
&format!("{}{}?{}", loc.url, path, new_query),
|
||||
)?;
|
||||
let client = state.http_client.clone();
|
||||
|
||||
@@ -704,15 +698,13 @@ async fn proxy_request(
|
||||
) -> Response {
|
||||
// Build target URL, adding proxied=true query param
|
||||
let path = info.path.trim_start_matches('/');
|
||||
// Defensively strip a `.grpcPort` suffix in case target.url carries it.
|
||||
let target_http = to_http_address(&target.url);
|
||||
|
||||
let raw_target = if info.original_query.is_empty() {
|
||||
format!("{}/{}?proxied=true", target_http, path)
|
||||
format!("{}/{}?proxied=true", target.url, path)
|
||||
} else {
|
||||
format!(
|
||||
"{}/{}?{}&proxied=true",
|
||||
target_http, path, info.original_query
|
||||
target.url, path, info.original_query
|
||||
)
|
||||
};
|
||||
let target_url = match normalize_outgoing_http_url(&state.outgoing_http_scheme, &raw_target) {
|
||||
@@ -779,11 +771,9 @@ fn redirect_request(info: &ProxyRequestInfo, target: &VolumeLocation, scheme: &s
|
||||
query_params.push("proxied=true".to_string());
|
||||
let query = query_params.join("&");
|
||||
|
||||
// Defensively strip a `.grpcPort` suffix in case target.url carries it.
|
||||
let target_http = to_http_address(&target.url);
|
||||
let raw_target = format!(
|
||||
"{}/{},{}?{}",
|
||||
target_http, &info.vid_str, &info.fid_str, query
|
||||
target.url, &info.vid_str, &info.fid_str, query
|
||||
);
|
||||
let location = match normalize_outgoing_http_url(scheme, &raw_target) {
|
||||
Ok(url) => url,
|
||||
@@ -1019,44 +1009,43 @@ async fn get_or_head_handler_inner(
|
||||
|
||||
if has_ec_volume && !has_volume {
|
||||
// ---- EC volume read path (always full read, no streaming) ----
|
||||
//
|
||||
// The distributed read path already does a local-first pass
|
||||
// in its Snapshot phase under the same store read lock the
|
||||
// legacy code would have taken — so calling it directly
|
||||
// serves both the "all shards local" fast case and the
|
||||
// "some intervals need peer fetch + reconstruct" general
|
||||
// case without paying for the local interval reads twice.
|
||||
match crate::server::store_ec::read_ec_shard_needle_distributed(
|
||||
&state, vid, needle_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(ec_needle)) => {
|
||||
n = ec_needle;
|
||||
}
|
||||
Ok(None) => {
|
||||
let store = state.store.read().unwrap();
|
||||
match store.find_ec_volume(vid) {
|
||||
Some(ecv) => match ecv.read_ec_shard_needle(needle_id) {
|
||||
Ok(Some(ec_needle)) => {
|
||||
n = ec_needle;
|
||||
}
|
||||
Ok(None) => {
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
|
||||
.inc();
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
Err(e) => {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
|
||||
.inc();
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_INTERNAL])
|
||||
.inc();
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("read ec error: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
},
|
||||
None => {
|
||||
metrics::HANDLER_COUNTER
|
||||
.with_label_values(&[metrics::ERROR_GET_NOT_FOUND])
|
||||
.inc();
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
Err(e) => {
|
||||
let kind = if e.kind() == std::io::ErrorKind::NotFound {
|
||||
metrics::ERROR_GET_NOT_FOUND
|
||||
} else {
|
||||
metrics::ERROR_GET_INTERNAL
|
||||
};
|
||||
metrics::HANDLER_COUNTER.with_label_values(&[kind]).inc();
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
return StatusCode::NOT_FOUND.into_response();
|
||||
}
|
||||
return (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("ec read: {}", e),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
drop(store);
|
||||
|
||||
// Validate cookie (matches Go behavior after ReadEcShardNeedle)
|
||||
if n.cookie != cookie {
|
||||
@@ -3931,81 +3920,4 @@ mod tests {
|
||||
"https://volume.internal:8080/3,01637037d6?collection=photos&proxied=true"
|
||||
);
|
||||
}
|
||||
|
||||
/// Defensive coverage for peer-to-peer redirects: if a master ever returns
|
||||
/// `Location.url` already encoded in `host:httpPort.grpcPort` form,
|
||||
/// `redirect_request` must still emit a valid HTTP Location header.
|
||||
#[test]
|
||||
fn test_redirect_request_strips_grpc_port_from_target_url() {
|
||||
let info = ProxyRequestInfo {
|
||||
original_headers: HeaderMap::new(),
|
||||
original_query: String::new(),
|
||||
path: "/3,01637037d6".to_string(),
|
||||
vid_str: "3".to_string(),
|
||||
fid_str: "01637037d6".to_string(),
|
||||
};
|
||||
let target = VolumeLocation {
|
||||
url: "volume.internal:8080.18080".to_string(),
|
||||
public_url: "volume.public:8080.18080".to_string(),
|
||||
grpc_port: 18080,
|
||||
};
|
||||
|
||||
let response = redirect_request(&info, &target, "http");
|
||||
assert_eq!(response.status(), StatusCode::MOVED_PERMANENTLY);
|
||||
assert_eq!(
|
||||
response.headers().get(header::LOCATION).unwrap(),
|
||||
"http://volume.internal:8080/3,01637037d6?proxied=true"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression test for issue #9274.
|
||||
///
|
||||
/// SeaweedFS encodes a server's gRPC port by appending `.grpcPort` to
|
||||
/// the HTTP port: e.g. `master.local:9333.19333` (mirrors Go's
|
||||
/// `pb.ServerAddress`). When the volume server's `--mserver` flag is
|
||||
/// configured this way, replication writes look up the peer locations
|
||||
/// from the master over HTTP. Prior to the fix, the lookup URL became
|
||||
/// `http://host:9333.19333/dir/lookup?volumeId=...` — an invalid port
|
||||
/// that reqwest rejects with a "builder error", causing every
|
||||
/// replicated write to fail.
|
||||
#[tokio::test]
|
||||
async fn test_lookup_volume_strips_grpc_port_from_master_url() {
|
||||
use axum::{routing::get, Router};
|
||||
|
||||
let app = Router::new().route(
|
||||
"/dir/lookup",
|
||||
get(|axum::extract::Query(params): axum::extract::Query<std::collections::HashMap<String, String>>| async move {
|
||||
assert_eq!(params.get("volumeId").map(String::as_str), Some("31"));
|
||||
axum::Json(serde_json::json!({
|
||||
"volumeOrFileId": "31",
|
||||
"locations": [
|
||||
{"url": "10.0.0.2:5301", "publicUrl": "10.0.0.2:5301", "grpcPort": 5311}
|
||||
]
|
||||
}))
|
||||
}),
|
||||
);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
|
||||
// Mirrors how main.rs receives masters: a `host:httpPort.grpcPort` token.
|
||||
// The grpc-port part is fictitious here (the master is HTTP-only in
|
||||
// this test) — what matters is that the volume server must not pass
|
||||
// it through to reqwest.
|
||||
let master_url = format!("127.0.0.1:{}.{}", addr.port(), 19999);
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let locations = super::lookup_volume(&client, "http", &master_url, 31)
|
||||
.await
|
||||
.expect("lookup_volume should succeed even when master_url carries a .grpcPort suffix");
|
||||
|
||||
assert_eq!(locations.len(), 1);
|
||||
assert_eq!(locations[0].url, "10.0.0.2:5301");
|
||||
assert_eq!(locations[0].grpc_port, 5311);
|
||||
|
||||
server.abort();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,20 +169,10 @@ pub async fn run_heartbeat_with_state(
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a master address to a gRPC `host:port` target.
|
||||
///
|
||||
/// Mirrors Go's `pb.ServerToGrpcAddress()`:
|
||||
/// - `host:port.grpcPort` returns `host:grpcPort` (explicit gRPC port).
|
||||
/// - `host:port` returns `host:(port+10000)` (Go's default offset).
|
||||
/// - Anything that fails to parse is returned unchanged.
|
||||
/// Convert a master address "host:port" to a gRPC host:port target.
|
||||
/// The Go master uses port + 10000 for gRPC by default.
|
||||
pub fn to_grpc_address(master_addr: &str) -> String {
|
||||
if let Some((host, port_str)) = master_addr.rsplit_once(':') {
|
||||
// "host:port.grpcPort" — the part after the last '.' is the gRPC port.
|
||||
if let Some((_, grpc_port)) = port_str.rsplit_once('.') {
|
||||
if grpc_port.parse::<u16>().is_ok() {
|
||||
return format!("{}:{}", host, grpc_port);
|
||||
}
|
||||
}
|
||||
if let Ok(port) = port_str.parse::<u16>() {
|
||||
let grpc_port = port + 10000;
|
||||
return format!("{}:{}", host, grpc_port);
|
||||
@@ -344,12 +334,6 @@ fn diff_ec_shard_delta_messages(
|
||||
if !current.contains_key(key) {
|
||||
let mut deleted = message.clone();
|
||||
deleted.shard_sizes = vec![0];
|
||||
tracing::info!(
|
||||
volume_id = deleted.id,
|
||||
disk_id = deleted.disk_id,
|
||||
ec_index_bits = deleted.ec_index_bits,
|
||||
"deletes ec shards"
|
||||
);
|
||||
deleted_ec_shards.push(deleted);
|
||||
}
|
||||
}
|
||||
@@ -403,15 +387,6 @@ async fn do_heartbeat(
|
||||
let mut response_stream = client.send_heartbeat(stream).await?.into_inner();
|
||||
|
||||
info!("Heartbeat stream established with {}", grpc_addr);
|
||||
// Publish the master we're now talking to in canonical http host:port
|
||||
// form so Ping admission can recognise it once a leader change moves us
|
||||
// off the seed list. Mirrors Go's vs.setCurrentMaster(masterAddress).
|
||||
{
|
||||
let normalised =
|
||||
super::volume_server::to_http_address(current_master).into_owned();
|
||||
let mut guard = state.current_master_url.write().await;
|
||||
*guard = normalised;
|
||||
}
|
||||
if is_stopping(state) {
|
||||
state.is_heartbeating.store(false, Ordering::Relaxed);
|
||||
send_deregister_heartbeat(config, state, &tx).await;
|
||||
@@ -875,7 +850,7 @@ fn build_heartbeat_with_ec_status(
|
||||
}
|
||||
|
||||
for vid in delete_vids {
|
||||
let _ = loc.delete_volume(vid, false, false);
|
||||
let _ = loc.delete_volume(vid, false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1048,8 +1023,6 @@ mod tests {
|
||||
read_mode: ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -1065,26 +1038,6 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_grpc_address_default_offset() {
|
||||
assert_eq!(to_grpc_address("10.0.0.1:9333"), "10.0.0.1:19333");
|
||||
assert_eq!(to_grpc_address("localhost:9333"), "localhost:19333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_grpc_address_explicit_grpc_port() {
|
||||
// host:port.grpcPort form — gRPC port is what's after the dot.
|
||||
assert_eq!(to_grpc_address("10.85.183.6:5300.6300"), "10.85.183.6:6300");
|
||||
assert_eq!(to_grpc_address("master.local:9333.19333"), "master.local:19333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_grpc_address_returns_input_when_unparseable() {
|
||||
assert_eq!(to_grpc_address(""), "");
|
||||
assert_eq!(to_grpc_address("no-port"), "no-port");
|
||||
assert_eq!(to_grpc_address("host:not-a-port"), "host:not-a-port");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_heartbeat_includes_store_identity_and_disk_metadata() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
@@ -1258,7 +1211,7 @@ mod tests {
|
||||
let shard_path = format!("{}/ec_metrics_case_27.ec00", dir);
|
||||
std::fs::write(&shard_path, b"ec-shard").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0])
|
||||
.unwrap();
|
||||
|
||||
let state = test_state_with_store(store);
|
||||
@@ -1300,7 +1253,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/expired_heartbeat_ec_31.ec00", dir), b"expired").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0], "")
|
||||
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0])
|
||||
.unwrap();
|
||||
store
|
||||
.find_ec_volume_mut(VolumeId(31))
|
||||
@@ -1455,8 +1408,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_registers_s3_default_aliases() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently. Just register our entries and verify them.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1488,8 +1443,10 @@ mod tests {
|
||||
#[test]
|
||||
fn test_apply_storage_backends_ignores_unsupported_types() {
|
||||
let state = test_state_with_store(Store::new(NeedleMapKind::InMemory));
|
||||
// Do not call clear() on the global registry — other tests may be
|
||||
// running concurrently.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
|
||||
apply_storage_backends(
|
||||
&state,
|
||||
@@ -1500,17 +1457,12 @@ mod tests {
|
||||
}],
|
||||
);
|
||||
|
||||
// The per-state registry is freshly created and should have no entries
|
||||
// since "rclone" is unsupported.
|
||||
let registry = state.s3_tier_registry.read().unwrap();
|
||||
assert!(registry.names().is_empty());
|
||||
// Only check that the unsupported type was not added to the global
|
||||
// registry. Other tests may have their own entries present.
|
||||
let global_registry = crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.read()
|
||||
.unwrap();
|
||||
assert!(global_registry.get("rclone.default").is_none());
|
||||
assert!(global_registry.get("rclone").is_none());
|
||||
assert!(global_registry.names().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1603,7 +1555,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/ec_delta_case_81.ec00", dir), b"delta").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0])
|
||||
.unwrap();
|
||||
let current = collect_ec_shard_delta_messages(&store);
|
||||
let (new_ec_shards, deleted_ec_shards) =
|
||||
|
||||
@@ -9,7 +9,6 @@ pub mod memory_status;
|
||||
pub mod profiling;
|
||||
pub mod request_id;
|
||||
pub mod server_stats;
|
||||
pub mod store_ec;
|
||||
pub mod ui;
|
||||
pub mod volume_server;
|
||||
pub mod write_queue;
|
||||
|
||||
@@ -1,678 +0,0 @@
|
||||
//! Distributed EC read path. Mirror of `weed/storage/store_ec.go`'s
|
||||
//! `readEcShardIntervals` → `readOneEcShardInterval` →
|
||||
//! `readRemoteEcShardInterval` → `recoverOneRemoteEcShardInterval`
|
||||
//! chain.
|
||||
//!
|
||||
//! The existing `EcVolume::read_ec_shard_needle` reads only locally-
|
||||
//! mounted shards and returns `NotFound` if any interval requires a
|
||||
//! shard held on a peer server. In a standard RS(10,4)-across-14
|
||||
//! deployment each server holds one shard, so every read needs >=9
|
||||
//! peer fetches. This module fills the gap by:
|
||||
//!
|
||||
//! 1. Locating the needle in `.ecx` (under the Store read lock) and
|
||||
//! computing the per-interval (shard_id, shard_offset, size).
|
||||
//! 2. Reading the local-resident intervals while still holding the
|
||||
//! lock — same path the local-only helper uses.
|
||||
//! 3. Dropping the lock and, for any remaining intervals, fetching
|
||||
//! from peer volume servers via `VolumeEcShardRead`. If the
|
||||
//! direct peer read fails, fan-out reads to other shards at the
|
||||
//! same (shard_offset, size) and rebuild the missing shard via
|
||||
//! Reed-Solomon — exactly Go's flow.
|
||||
//! 4. Refreshing the per-EcVolume `shard_locations` cache from the
|
||||
//! master's `LookupEcVolume` RPC when the cached map is stale.
|
||||
//!
|
||||
//! All gRPC IO is async; the file IO portion runs under the sync
|
||||
//! Store read lock, matching Go's `readLocalEcShardInterval`. The
|
||||
//! cache write-back briefly reacquires the EcVolume's internal
|
||||
//! `RwLock` so we do not contend with the Store-level lock at all.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures::future::join_all;
|
||||
use reed_solomon_erasure::galois_8::ReedSolomon;
|
||||
use tonic::Request;
|
||||
|
||||
use crate::pb::master_pb::{self, seaweed_client::SeaweedClient, LookupEcVolumeRequest};
|
||||
use crate::pb::volume_server_pb::{
|
||||
volume_server_client::VolumeServerClient, VolumeEcShardReadRequest,
|
||||
};
|
||||
use crate::server::grpc_client::{build_grpc_endpoint, parse_grpc_address, GRPC_MAX_MESSAGE_SIZE};
|
||||
use crate::server::request_id::outgoing_request_id_interceptor;
|
||||
use crate::server::volume_server::VolumeServerState;
|
||||
use crate::storage::erasure_coding::ec_shard::ShardId;
|
||||
use crate::storage::needle::needle::{get_actual_size, Needle};
|
||||
use crate::storage::types::*;
|
||||
|
||||
/// One interval's data after Phase A.
|
||||
enum IntervalResult {
|
||||
/// Already read from a locally-mounted shard.
|
||||
Local(Vec<u8>),
|
||||
/// Shard not local (or local read failed). Must be fetched.
|
||||
NeedRemote {
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
},
|
||||
}
|
||||
|
||||
/// Snapshot extracted under the Store read lock so Phases B/C can run
|
||||
/// without holding any sync lock across `.await`.
|
||||
struct Snapshot {
|
||||
data_shards: u32,
|
||||
parity_shards: u32,
|
||||
version: Version,
|
||||
actual_size: usize,
|
||||
offset: Offset,
|
||||
size_for_parse: Size,
|
||||
intervals: Vec<IntervalResult>,
|
||||
cached_locations: HashMap<ShardId, Vec<String>>,
|
||||
cache_refreshed_at: Option<Instant>,
|
||||
}
|
||||
|
||||
/// Top-level entry point. Returns `Ok(None)` for "not found" (matches
|
||||
/// Go's `ReadEcShardNeedle`); errors propagate as `io::Error`.
|
||||
pub async fn read_ec_shard_needle_distributed(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<Option<Needle>> {
|
||||
// Phase A — under the Store read lock, locate the needle, compute
|
||||
// intervals, and read any locally-mounted shard intervals. We must
|
||||
// not `.await` while holding this guard (std::sync::RwLockReadGuard
|
||||
// is !Send).
|
||||
let snapshot = match snapshot_under_lock(state, vid, needle_id)? {
|
||||
Some(s) => s,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
// Phase B — refresh the shard_locations cache from the master if
|
||||
// it is stale. Do this lazily: if every needed interval was read
|
||||
// locally we can skip the master RPC entirely.
|
||||
let any_remote = snapshot
|
||||
.intervals
|
||||
.iter()
|
||||
.any(|r| matches!(r, IntervalResult::NeedRemote { .. }));
|
||||
let total_shards = (snapshot.data_shards + snapshot.parity_shards) as usize;
|
||||
|
||||
let mut shard_locations = snapshot.cached_locations.clone();
|
||||
if any_remote
|
||||
&& needs_refresh(
|
||||
&shard_locations,
|
||||
snapshot.cache_refreshed_at,
|
||||
snapshot.data_shards as usize,
|
||||
total_shards,
|
||||
)
|
||||
{
|
||||
match cached_lookup_ec_shard_locations(state, vid).await {
|
||||
Ok(fresh) => {
|
||||
shard_locations = fresh.clone();
|
||||
write_back_shard_locations(state, vid, fresh);
|
||||
}
|
||||
Err(e) => {
|
||||
// Lookup failed — proceed with cached values. If cache
|
||||
// is empty, the remote fetch below will fail and we
|
||||
// surface a NotFound (matching Go's behavior when no
|
||||
// locations are known).
|
||||
tracing::warn!(
|
||||
"ec lookup failed for volume {}: {} — using cached locations ({} entries)",
|
||||
vid.0,
|
||||
e,
|
||||
shard_locations.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase C — fetch missing intervals, reconstructing when the
|
||||
// direct peer read fails.
|
||||
let mut assembled: Vec<Vec<u8>> = Vec::with_capacity(snapshot.intervals.len());
|
||||
for res in snapshot.intervals {
|
||||
match res {
|
||||
IntervalResult::Local(buf) => assembled.push(buf),
|
||||
IntervalResult::NeedRemote {
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
} => {
|
||||
let buf = fetch_one_interval(
|
||||
state,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
&shard_locations,
|
||||
snapshot.data_shards as usize,
|
||||
snapshot.parity_shards as usize,
|
||||
)
|
||||
.await?;
|
||||
assembled.push(buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase D — assemble and parse the Needle. Mirrors the tail of
|
||||
// `EcVolume::read_ec_shard_needle`.
|
||||
let mut bytes = Vec::with_capacity(snapshot.actual_size);
|
||||
for chunk in assembled {
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
bytes.truncate(snapshot.actual_size);
|
||||
if bytes.len() < snapshot.actual_size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
format!(
|
||||
"read {} bytes but need {} for needle {}",
|
||||
bytes.len(),
|
||||
snapshot.actual_size,
|
||||
needle_id
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
let mut n = Needle::default();
|
||||
n.id = needle_id;
|
||||
n.read_bytes(
|
||||
&bytes,
|
||||
snapshot.offset.to_actual_offset(),
|
||||
snapshot.size_for_parse,
|
||||
snapshot.version,
|
||||
)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))?;
|
||||
Ok(Some(n))
|
||||
}
|
||||
|
||||
fn snapshot_under_lock(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<Option<Snapshot>> {
|
||||
let store = state.store.read().unwrap();
|
||||
let ecv = match store.find_ec_volume(vid) {
|
||||
Some(v) => v,
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
// Reuse EcVolume::locate_needle for offset/size resolution AND
|
||||
// the per-needle shard-interval math — it's the same routine the
|
||||
// local-only read path uses, so we stay byte-identical on the
|
||||
// shard-size + interval boundaries.
|
||||
let (offset, size, intervals) = match ecv.locate_needle(needle_id)? {
|
||||
Some(v) => v,
|
||||
None => return Ok(None),
|
||||
};
|
||||
if intervals.is_empty() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"no intervals for needle",
|
||||
));
|
||||
}
|
||||
let actual = get_actual_size(size, ecv.version);
|
||||
|
||||
// Phase A.local: for each interval read the local shard if we
|
||||
// hold it; otherwise fall through to remote. We accumulate the
|
||||
// results in interval order so the assembly step is just a
|
||||
// concat.
|
||||
let mut interval_results = Vec::with_capacity(intervals.len());
|
||||
for interval in &intervals {
|
||||
let (shard_id, shard_offset) = interval.to_shard_id_and_offset(ecv.data_shards);
|
||||
let buf_size = interval.size as usize;
|
||||
let local = ecv
|
||||
.shards
|
||||
.get(shard_id as usize)
|
||||
.and_then(|s| s.as_ref());
|
||||
match local {
|
||||
Some(shard) => {
|
||||
let mut buf = vec![0u8; buf_size];
|
||||
match shard.read_at(&mut buf, shard_offset as u64) {
|
||||
Ok(n) if n == buf_size => {
|
||||
interval_results.push(IntervalResult::Local(buf));
|
||||
}
|
||||
_ => interval_results.push(IntervalResult::NeedRemote {
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size: buf_size,
|
||||
}),
|
||||
}
|
||||
}
|
||||
None => interval_results.push(IntervalResult::NeedRemote {
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size: buf_size,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
let cached_locations = ecv.shard_locations.read().unwrap().clone();
|
||||
let cache_refreshed_at = *ecv.shard_locations_refresh_time.lock().unwrap();
|
||||
|
||||
Ok(Some(Snapshot {
|
||||
data_shards: ecv.data_shards,
|
||||
parity_shards: ecv.parity_shards,
|
||||
version: ecv.version,
|
||||
actual_size: actual as usize,
|
||||
offset,
|
||||
size_for_parse: size,
|
||||
intervals: interval_results,
|
||||
cached_locations,
|
||||
cache_refreshed_at,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Master `LookupEcVolume` freshness rules — match Go's
|
||||
/// `cachedLookupEcShardLocations` thresholds in store_ec.go.
|
||||
fn needs_refresh(
|
||||
locations: &HashMap<ShardId, Vec<String>>,
|
||||
refreshed_at: Option<Instant>,
|
||||
data_shards: usize,
|
||||
total_shards: usize,
|
||||
) -> bool {
|
||||
let now = Instant::now();
|
||||
let age = match refreshed_at {
|
||||
Some(t) => now.saturating_duration_since(t),
|
||||
None => return true,
|
||||
};
|
||||
let shard_count = locations.len();
|
||||
if shard_count < data_shards && age < Duration::from_secs(11) {
|
||||
return false;
|
||||
}
|
||||
if shard_count == total_shards && age < Duration::from_secs(37 * 60) {
|
||||
return false;
|
||||
}
|
||||
if shard_count >= data_shards && age < Duration::from_secs(7 * 60) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn cached_lookup_ec_shard_locations(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<HashMap<ShardId, Vec<String>>> {
|
||||
let master = {
|
||||
let live = state.current_master_url.read().await.clone();
|
||||
if !live.is_empty() {
|
||||
live
|
||||
} else {
|
||||
state.master_url.clone()
|
||||
}
|
||||
};
|
||||
if master.is_empty() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"no master configured for ec shard lookup",
|
||||
));
|
||||
}
|
||||
|
||||
let grpc_addr = parse_grpc_address(&master)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
|
||||
let endpoint = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref())
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
|
||||
let channel = endpoint
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.timeout(Duration::from_secs(10))
|
||||
.connect()
|
||||
.await
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("master connect: {}", e)))?;
|
||||
|
||||
let mut client = SeaweedClient::with_interceptor(channel, outgoing_request_id_interceptor)
|
||||
.max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE);
|
||||
|
||||
let resp = client
|
||||
.lookup_ec_volume(Request::new(LookupEcVolumeRequest { volume_id: vid.0 }))
|
||||
.await
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("lookup_ec_volume: {}", e)))?;
|
||||
let resp = resp.into_inner();
|
||||
|
||||
let mut out = HashMap::new();
|
||||
for entry in resp.shard_id_locations {
|
||||
let addrs: Vec<String> = entry
|
||||
.locations
|
||||
.iter()
|
||||
.map(format_location_as_server_address)
|
||||
.collect();
|
||||
out.insert(entry.shard_id as ShardId, addrs);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn write_back_shard_locations(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
locations: HashMap<ShardId, Vec<String>>,
|
||||
) {
|
||||
let store = state.store.read().unwrap();
|
||||
if let Some(ecv) = store.find_ec_volume(vid) {
|
||||
// Atomic swap + freshness stamp so a concurrent reader sees
|
||||
// either the prior cache or the fresh one — never an
|
||||
// intermediate half-replaced map with the freshness flag
|
||||
// already flipped.
|
||||
ecv.replace_shard_locations(locations);
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a SeaweedFS-style `host:httpPort.grpcPort` address from a
|
||||
/// master `Location` so the result is what `parse_grpc_address` (and
|
||||
/// the heartbeat path) already understand.
|
||||
fn format_location_as_server_address(loc: &master_pb::Location) -> String {
|
||||
let raw = loc
|
||||
.url
|
||||
.trim_start_matches("http://")
|
||||
.trim_start_matches("https://");
|
||||
if loc.grpc_port > 0 {
|
||||
if let Some((host, http_port)) = raw.rsplit_once(':') {
|
||||
return format!("{}:{}.{}", host, http_port, loc.grpc_port);
|
||||
}
|
||||
}
|
||||
raw.to_string()
|
||||
}
|
||||
|
||||
/// Try direct peer read; on failure, reconstruct via Reed-Solomon
|
||||
/// from the other shards. Mirrors `readOneEcShardInterval`'s tail.
|
||||
async fn fetch_one_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
shard_locations: &HashMap<ShardId, Vec<String>>,
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
// Direct peer read against the cached locations for this shard.
|
||||
if let Some(sources) = shard_locations.get(&shard_id) {
|
||||
if !sources.is_empty() {
|
||||
match read_remote_ec_shard_interval(
|
||||
state,
|
||||
sources,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(buf) => return Ok(buf),
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
"direct read ec shard {}.{} from {:?} failed: {} — will reconstruct",
|
||||
vid.0,
|
||||
shard_id,
|
||||
sources,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reconstruct: fan-out reads to every other shard at the same
|
||||
// (shard_offset, size). Mirrors `recoverOneRemoteEcShardInterval`.
|
||||
recover_one_remote_ec_shard_interval(
|
||||
state,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
shard_locations,
|
||||
data_shards,
|
||||
parity_shards,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_remote_ec_shard_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
sources: &[String],
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let mut last_err: Option<io::Error> = None;
|
||||
for src in sources {
|
||||
match do_read_remote_ec_shard_interval(
|
||||
state,
|
||||
src,
|
||||
vid,
|
||||
needle_id,
|
||||
shard_id,
|
||||
shard_offset,
|
||||
size,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(buf) => return Ok(buf),
|
||||
Err(e) => last_err = Some(e),
|
||||
}
|
||||
}
|
||||
Err(last_err.unwrap_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
format!("no source for ec shard {}.{}", vid.0, shard_id),
|
||||
)
|
||||
}))
|
||||
}
|
||||
|
||||
async fn do_read_remote_ec_shard_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
source: &str,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let grpc_addr =
|
||||
parse_grpc_address(source).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
|
||||
let endpoint = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref())
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
|
||||
let channel = endpoint
|
||||
.connect_timeout(Duration::from_secs(5))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.connect()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("connect to {}: {}", source, e),
|
||||
)
|
||||
})?;
|
||||
|
||||
// TODO(grpc-jwt): clusters with `jwt.signing.key` configured will
|
||||
// reject peer-to-peer VolumeEcShardRead calls until the Rust
|
||||
// crate grows an outgoing-JWT interceptor. The gap is shared
|
||||
// with every other peer gRPC call from this binary
|
||||
// (`copy_file_from_source`, `batch_delete`, …) — handling it
|
||||
// here in isolation would split the credential plumbing across
|
||||
// call sites. Re-visit when outgoing JWT signing lands as a
|
||||
// server-wide helper.
|
||||
let mut client = VolumeServerClient::with_interceptor(channel, outgoing_request_id_interceptor)
|
||||
.max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE);
|
||||
|
||||
let req = VolumeEcShardReadRequest {
|
||||
volume_id: vid.0,
|
||||
shard_id: shard_id as u32,
|
||||
offset: shard_offset,
|
||||
size: size as i64,
|
||||
file_key: needle_id.0,
|
||||
};
|
||||
let resp = client
|
||||
.volume_ec_shard_read(Request::new(req))
|
||||
.await
|
||||
.map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("volume_ec_shard_read {}.{} from {}: {}", vid.0, shard_id, source, e),
|
||||
)
|
||||
})?;
|
||||
let mut stream = resp.into_inner();
|
||||
|
||||
let mut out = Vec::with_capacity(size);
|
||||
while let Some(msg) = stream
|
||||
.message()
|
||||
.await
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("recv: {}", e)))?
|
||||
{
|
||||
if !msg.data.is_empty() {
|
||||
out.extend_from_slice(&msg.data);
|
||||
}
|
||||
}
|
||||
if out.len() < size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
format!(
|
||||
"short read from {} for ec shard {}.{}: got {} want {}",
|
||||
source,
|
||||
vid.0,
|
||||
shard_id,
|
||||
out.len(),
|
||||
size
|
||||
),
|
||||
));
|
||||
}
|
||||
out.truncate(size);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn recover_one_remote_ec_shard_interval(
|
||||
state: &Arc<VolumeServerState>,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
shard_id_to_recover: ShardId,
|
||||
shard_offset: i64,
|
||||
size: usize,
|
||||
shard_locations: &HashMap<ShardId, Vec<String>>,
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let total_shards = data_shards + parity_shards;
|
||||
let rs = ReedSolomon::new(data_shards, parity_shards).map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("reed-solomon init: {:?}", e),
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut bufs: Vec<Option<Vec<u8>>> = vec![None; total_shards];
|
||||
|
||||
// Phase 0: seed bufs from LOCALLY mounted shards. If this node
|
||||
// already holds enough sibling shards, reconstruction completes
|
||||
// without any peer fan-out — and even with a cold/incomplete
|
||||
// shard_locations cache or a failed master lookup, local
|
||||
// survivors still contribute. Mirrors Go's
|
||||
// recoverOneRemoteEcShardInterval behaviour, which is implicitly
|
||||
// local-aware because the Store fan-out targets ALL known
|
||||
// locations (including the caller's own server address); the
|
||||
// Rust port had been remote-only, so reconstructing with a cold
|
||||
// cache failed even when enough siblings were on disk.
|
||||
{
|
||||
let store = state.store.read().unwrap();
|
||||
if let Some(ecv) = store.find_ec_volume(vid) {
|
||||
for sid in 0..total_shards {
|
||||
if sid as ShardId == shard_id_to_recover {
|
||||
continue;
|
||||
}
|
||||
if let Some(Some(shard)) = ecv.shards.get(sid) {
|
||||
let mut buf = vec![0u8; size];
|
||||
if shard.read_at(&mut buf, shard_offset as u64).map(|n| n == size).unwrap_or(false) {
|
||||
bufs[sid] = Some(buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 1: remote fan-out — one task per known shard location
|
||||
// we DON'T already have locally and DON'T need to recover.
|
||||
let mut tasks = Vec::new();
|
||||
for (sid, locs) in shard_locations {
|
||||
if *sid == shard_id_to_recover || locs.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if bufs[*sid as usize].is_some() {
|
||||
continue;
|
||||
}
|
||||
let sid = *sid;
|
||||
let locs = locs.clone();
|
||||
let state = state.clone();
|
||||
tasks.push(async move {
|
||||
let res = read_remote_ec_shard_interval(
|
||||
&state,
|
||||
&locs,
|
||||
vid,
|
||||
needle_id,
|
||||
sid,
|
||||
shard_offset,
|
||||
size,
|
||||
)
|
||||
.await;
|
||||
(sid, res)
|
||||
});
|
||||
}
|
||||
let results = join_all(tasks).await;
|
||||
|
||||
for (sid, res) in results {
|
||||
match res {
|
||||
Ok(buf) => {
|
||||
if (sid as usize) < total_shards {
|
||||
bufs[sid as usize] = Some(buf);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!(
|
||||
"recover: read {}.{} for needle {} failed: {}",
|
||||
vid.0,
|
||||
sid,
|
||||
needle_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let available = bufs.iter().filter(|b| b.is_some()).count();
|
||||
if available < data_shards {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"cannot recover ec shard {}.{}: only {} shards available, need at least {}",
|
||||
vid.0, shard_id_to_recover, available, data_shards
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
rs.reconstruct(&mut bufs).map_err(|e| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"reed-solomon reconstruct ec shard {}.{}: {:?}",
|
||||
vid.0, shard_id_to_recover, e
|
||||
),
|
||||
)
|
||||
})?;
|
||||
|
||||
match bufs.into_iter().nth(shard_id_to_recover as usize).flatten() {
|
||||
Some(buf) => Ok(buf),
|
||||
None => Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"reconstructed buffer for shard {}.{} missing after RS reconstruct",
|
||||
vid.0, shard_id_to_recover
|
||||
),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// parse_grpc_address lives in `grpc_client.rs` and is re-exported
|
||||
// here via the use above so this module shares a single
|
||||
// HTTP↔gRPC port-translation routine with grpc_server.rs.
|
||||
@@ -81,16 +81,6 @@ pub struct VolumeServerState {
|
||||
pub master_url: String,
|
||||
/// Seed master addresses for UI rendering.
|
||||
pub master_urls: Vec<String>,
|
||||
/// Canonical http `host:port` form of every configured seed master.
|
||||
/// Built once at construction so Ping admission stays O(1). Mirrors
|
||||
/// Go's `seedMasterSet` on `VolumeServer`.
|
||||
pub seed_master_set: std::collections::HashSet<String>,
|
||||
/// Current master this server is heartbeating with, in canonical http
|
||||
/// `host:port` form. Empty when no heartbeat connection is active. The
|
||||
/// heartbeat goroutine writes; admission reads — the lock keeps them
|
||||
/// from racing on a leader change. Mirrors Go's `currentMaster` plus
|
||||
/// `currentMasterLock`.
|
||||
pub current_master_url: tokio::sync::RwLock<String>,
|
||||
/// This server's own address (ip:port) for filtering self from lookup results.
|
||||
pub self_url: String,
|
||||
/// HTTP client for proxy requests and master lookups.
|
||||
@@ -127,35 +117,6 @@ impl VolumeServerState {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build the seed master set from a list of raw `host:port[.grpcPort]`
|
||||
/// addresses, normalised the same way Go's `pb.ServerAddress.ToHttpAddress`
|
||||
/// does (drop the `.grpcPort` suffix, preserve everything else).
|
||||
pub fn build_seed_master_set(master_urls: &[String]) -> std::collections::HashSet<String> {
|
||||
master_urls
|
||||
.iter()
|
||||
.map(|m| to_http_address(m).into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns true iff `target` (normalised to canonical http `host:port`)
|
||||
/// is a master this server already knows about. Volume servers do not
|
||||
/// keep a peer-volume or peer-filer list, so Ping is scoped to masters.
|
||||
/// Mirrors Go's `VolumeServer.isKnownPingTarget`.
|
||||
pub async fn is_known_ping_target(&self, target: &str, target_type: &str) -> bool {
|
||||
if target_type != "master" {
|
||||
return false;
|
||||
}
|
||||
let key = to_http_address(target).into_owned();
|
||||
if key.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let current = self.current_master_url.read().await.clone();
|
||||
if !current.is_empty() && current == key {
|
||||
return true;
|
||||
}
|
||||
self.seed_master_set.contains(&key)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_metrics_router() -> Router {
|
||||
@@ -173,38 +134,6 @@ pub fn normalize_outgoing_http_url(scheme: &str, raw_target: &str) -> Result<Str
|
||||
Ok(format!("{}://{}", scheme, raw_target))
|
||||
}
|
||||
|
||||
/// Convert a SeaweedFS server address to its HTTP `host:port` form.
|
||||
///
|
||||
/// Mirrors Go's `pb.ServerAddress.ToHttpAddress()`: SeaweedFS encodes a
|
||||
/// server's gRPC port by appending `.grpcPort` to the HTTP port (e.g.
|
||||
/// `host:9333.19333`). For HTTP requests we want only the HTTP `host:port`.
|
||||
/// - `host:port.grpcPort` -> `host:port`
|
||||
/// - `host:port` -> `host:port` (unchanged)
|
||||
/// - Anything that does not look like `host:port[.grpcPort]` is returned unchanged.
|
||||
///
|
||||
/// Returns a `Cow<str>` so the common (no-suffix) case borrows from `addr`
|
||||
/// without allocating; only the rewrite branch produces a new `String`.
|
||||
pub fn to_http_address(addr: &str) -> std::borrow::Cow<'_, str> {
|
||||
let Some(ports_sep_index) = addr.rfind(':') else {
|
||||
return std::borrow::Cow::Borrowed(addr);
|
||||
};
|
||||
let ports = &addr[ports_sep_index + 1..];
|
||||
if let Some(dot_idx) = ports.rfind('.') {
|
||||
let http_port = &ports[..dot_idx];
|
||||
let grpc_port = &ports[dot_idx + 1..];
|
||||
// Only strip the suffix when both parts parse as real ports — leave
|
||||
// anything else (e.g. "host:abc.def") untouched so bad config surfaces
|
||||
// rather than being silently rewritten. Mirrors the validation already
|
||||
// done in `to_grpc_address` for the inverse direction.
|
||||
if let (Ok(_), Ok(_)) = (http_port.parse::<u16>(), grpc_port.parse::<u16>()) {
|
||||
return std::borrow::Cow::Owned(
|
||||
addr[..ports_sep_index + 1 + dot_idx].to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
std::borrow::Cow::Borrowed(addr)
|
||||
}
|
||||
|
||||
fn request_remote_addr(request: &Request) -> Option<SocketAddr> {
|
||||
request
|
||||
.extensions()
|
||||
@@ -463,78 +392,3 @@ pub fn build_public_router(state: Arc<VolumeServerState>) -> Router {
|
||||
.layer(middleware::from_fn(common_headers_middleware))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_strips_grpc_port_suffix() {
|
||||
assert_eq!(to_http_address("10.0.0.1:9333.19333"), "10.0.0.1:9333");
|
||||
assert_eq!(
|
||||
to_http_address("master.local:9333.19333"),
|
||||
"master.local:9333"
|
||||
);
|
||||
assert_eq!(to_http_address("10.85.183.6:5300.6300"), "10.85.183.6:5300");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_passthrough_without_grpc_suffix() {
|
||||
assert_eq!(to_http_address("10.0.0.1:9333"), "10.0.0.1:9333");
|
||||
assert_eq!(to_http_address("master.local:9333"), "master.local:9333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_returns_input_when_unparseable() {
|
||||
assert_eq!(to_http_address(""), "");
|
||||
assert_eq!(to_http_address("no-port"), "no-port");
|
||||
// Trailing colon: nothing after the separator, treat as unparseable.
|
||||
assert_eq!(to_http_address("host:"), "host:");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_borrows_when_unchanged_and_owns_when_stripped() {
|
||||
// The common case (no suffix) must not allocate.
|
||||
let result = to_http_address("10.0.0.1:9333");
|
||||
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
|
||||
// Stripping requires a new string.
|
||||
let result = to_http_address("10.0.0.1:9333.19333");
|
||||
assert!(matches!(result, std::borrow::Cow::Owned(_)));
|
||||
assert_eq!(result, "10.0.0.1:9333");
|
||||
// Unparseable / passthrough also borrows.
|
||||
let result = to_http_address("host:abc.def");
|
||||
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_keeps_non_numeric_dotted_suffix() {
|
||||
// The dotted form is only valid when both sides are real port numbers.
|
||||
// Otherwise the address is malformed config (e.g. a hostname like
|
||||
// "host:abc.def"), and silently rewriting it would just hide the bug.
|
||||
assert_eq!(to_http_address("host:abc.def"), "host:abc.def");
|
||||
assert_eq!(to_http_address("host:9333.notaport"), "host:9333.notaport");
|
||||
assert_eq!(to_http_address("host:notaport.19333"), "host:notaport.19333");
|
||||
// Out-of-range ports must not be silently truncated either.
|
||||
assert_eq!(to_http_address("host:99999.19333"), "host:99999.19333");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_to_http_address_handles_bracketed_ipv6_literals() {
|
||||
// The function uses `rfind(':')`, so for bracketed IPv6 the port
|
||||
// separator is correctly identified as the colon AFTER the closing
|
||||
// bracket — making IPv4 and IPv6 behave the same.
|
||||
assert_eq!(to_http_address("[::1]:9333.19333"), "[::1]:9333");
|
||||
assert_eq!(
|
||||
to_http_address("[2001:db8::10]:5300.6300"),
|
||||
"[2001:db8::10]:5300"
|
||||
);
|
||||
// Plain bracketed IPv6 without a dotted suffix is borrowed unchanged.
|
||||
let result = to_http_address("[2001:db8::1]:9333");
|
||||
assert!(matches!(result, std::borrow::Cow::Borrowed(_)));
|
||||
assert_eq!(result, "[2001:db8::1]:9333");
|
||||
// Non-numeric / out-of-range / missing suffix all preserve the input.
|
||||
assert_eq!(to_http_address("[::1]:"), "[::1]:");
|
||||
assert_eq!(to_http_address("[::1]:abc.def"), "[::1]:abc.def");
|
||||
assert_eq!(to_http_address("[::1]:99999.19333"), "[::1]:99999.19333");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,8 +207,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
|
||||
@@ -202,15 +202,6 @@ impl DiskLocation {
|
||||
}
|
||||
}
|
||||
|
||||
// After regular volumes, auto-discover EC shards on disk so a
|
||||
// fresh restart picks up shards without an explicit
|
||||
// VolumeEcShardsMount RPC. Mirrors Go's loadExistingVolumes
|
||||
// calling loadAllEcShards. Failures here only log — they
|
||||
// shouldn't fail the whole disk's startup.
|
||||
if let Err(e) = self.load_all_ec_shards() {
|
||||
warn!(directory = %self.directory, error = %e, "load_all_ec_shards failed");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -304,11 +295,7 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
/// Remove all EC-related files for a volume.
|
||||
/// `pub(crate)` so the Store-level cross-disk passes in
|
||||
/// `store_ec_reconcile.rs` can call it to scrub partial EC artefacts
|
||||
/// when a healthy `.dat` for the same vid lives on a sibling disk
|
||||
/// (seaweedfs/seaweedfs#9478).
|
||||
pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
|
||||
fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
const MAX_SHARD_COUNT: usize = 32;
|
||||
@@ -390,20 +377,13 @@ impl DiskLocation {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove, close, and delete all files for a volume. When keep_remote_data
|
||||
/// is true the cloud-tier object backing the volume is left intact — used
|
||||
/// by moves where another server is taking over the same .vif.
|
||||
pub fn delete_volume(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
only_empty: bool,
|
||||
keep_remote_data: bool,
|
||||
) -> Result<(), VolumeError> {
|
||||
/// Remove, close, and delete all files for a volume.
|
||||
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
|
||||
if let Some(mut v) = self.volumes.remove(&vid) {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
v.destroy(only_empty, keep_remote_data)?;
|
||||
v.destroy(only_empty)?;
|
||||
Ok(())
|
||||
} else {
|
||||
Err(VolumeError::NotFound)
|
||||
@@ -424,7 +404,7 @@ impl DiskLocation {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
if let Err(e) = v.destroy(false, false) {
|
||||
if let Err(e) = v.destroy(false) {
|
||||
warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
|
||||
}
|
||||
}
|
||||
@@ -564,114 +544,28 @@ impl DiskLocation {
|
||||
self.ec_volumes.contains_key(&vid)
|
||||
}
|
||||
|
||||
/// Reports whether this disk has a sealed `.ecx` index file for the
|
||||
/// given (collection, vid). Unlike [`Self::has_ec_volume`] this does
|
||||
/// not require the EC volume to be mounted in memory, which makes it
|
||||
/// the right primitive for placement decisions during `ec.balance` /
|
||||
/// `ec.rebuild` flows where shards may arrive before any
|
||||
/// `VolumeEcShardsMount` has happened on the receiving disk. Without
|
||||
/// checking the on-disk state, auto-select can split shards from the
|
||||
/// `.ecx` that travels with the first shard, which is the source of
|
||||
/// the orphan-shard layout reported in seaweedfs/seaweedfs#9212.
|
||||
///
|
||||
/// Mirrors `DiskLocation.HasEcxFileOnDisk` in
|
||||
/// `weed/storage/disk_location_ec.go`. Skips entries that are
|
||||
/// directories so a stray dir named `<collection>_<vid>.ecx` doesn't
|
||||
/// register as a present index file.
|
||||
pub fn has_ecx_file_on_disk(&self, collection: &str, vid: VolumeId) -> bool {
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
let idx_path = format!("{}.ecx", idx_base);
|
||||
if let Ok(meta) = fs::metadata(&idx_path) {
|
||||
if !meta.is_dir() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if self.idx_directory != self.directory {
|
||||
let data_base = volume_file_name(&self.directory, collection, vid);
|
||||
let data_path = format!("{}.ecx", data_base);
|
||||
if let Ok(meta) = fs::metadata(&data_path) {
|
||||
if !meta.is_dir() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Remove an EC volume, returning it.
|
||||
pub fn remove_ec_volume(&mut self, vid: VolumeId) -> Option<EcVolume> {
|
||||
self.ec_volumes.remove(&vid)
|
||||
}
|
||||
|
||||
/// Mount EC shards for a volume on this location.
|
||||
///
|
||||
/// `source_disk_type` is the source volume's disk type carried on the
|
||||
/// `VolumeEcShardsMount` RPC. When non-empty it overrides the in-memory
|
||||
/// EC volume's reported disk type so heartbeats keep reporting under
|
||||
/// the source's disk type after EC encoding (#9423). Empty means "use
|
||||
/// this location's disk type" — used by disk-scan reload paths that
|
||||
/// have no orchestrator context.
|
||||
pub fn mount_ec_shards(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_ids: &[u32],
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
let idx_dir = self.idx_directory.clone();
|
||||
self.mount_ec_shards_with_idx_dir(vid, collection, shard_ids, &idx_dir, source_disk_type)
|
||||
}
|
||||
|
||||
/// Mount EC shards but explicitly specify the idx directory the
|
||||
/// EcVolume should pull `.ecx` / `.ecj` / `.vif` from.
|
||||
///
|
||||
/// Cross-disk reconcile (mirrors `loadEcShardsWithIdxDir` in
|
||||
/// `weed/storage/disk_location_ec.go`): when a volume's shards live
|
||||
/// on this disk but the index files were left on a sibling disk
|
||||
/// (the seaweedfs/seaweedfs#9212 orphan-shard layout), the
|
||||
/// reconciler creates the EcVolume here while pointing it at the
|
||||
/// sibling's idx dir. Each shard still ends up registered in this
|
||||
/// disk's `ec_volumes` map so heartbeat reporting carries the right
|
||||
/// disk_id per shard.
|
||||
pub fn mount_ec_shards_with_idx_dir(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_ids: &[u32],
|
||||
idx_dir: &str,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
let dir = self.directory.clone();
|
||||
// Avoid the entry().or_insert_with() pattern here: that closure
|
||||
// can't return a Result, so any EcVolume::new failure (e.g.
|
||||
// .ecx open error, .ecj create error, malformed .vif) would
|
||||
// have to panic via unwrap(). Build the EcVolume up front and
|
||||
// propagate the error to the caller.
|
||||
if !self.ec_volumes.contains_key(&vid) {
|
||||
let ec_vol = EcVolume::new(&dir, idx_dir, collection, vid)
|
||||
.map_err(VolumeError::Io)?;
|
||||
self.ec_volumes.insert(vid, ec_vol);
|
||||
}
|
||||
let idx_dir = self.idx_directory.clone();
|
||||
let ec_vol = self
|
||||
.ec_volumes
|
||||
.get_mut(&vid)
|
||||
.expect("just inserted above");
|
||||
// When the orchestrator supplied a source disk type on the Mount
|
||||
// RPC, override the EC volume's disk type so heartbeats report
|
||||
// under the source volume's disk type (#9423). When the caller
|
||||
// passed "" (disk-scan reload, orphan-shard reconcile, restart)
|
||||
// we only default to this location's disk type for a fresh EC
|
||||
// volume; otherwise we leave whatever a prior RPC mount set in
|
||||
// place, so empty-source reloads don't reintroduce the drift.
|
||||
if !source_disk_type.is_empty() {
|
||||
ec_vol.set_disk_type(DiskType::from_string(source_disk_type));
|
||||
} else if ec_vol.shard_count() == 0 {
|
||||
ec_vol.set_disk_type(self.disk_type.clone());
|
||||
}
|
||||
.entry(vid)
|
||||
.or_insert_with(|| EcVolume::new(&dir, &idx_dir, collection, vid).unwrap());
|
||||
ec_vol.disk_type = self.disk_type.clone();
|
||||
|
||||
for &shard_id in shard_ids {
|
||||
let mut shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
|
||||
shard.disk_type = ec_vol.disk_type.clone();
|
||||
let shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
|
||||
ec_vol.add_shard(shard).map_err(VolumeError::Io)?;
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[collection, "ec_shards"])
|
||||
@@ -681,19 +575,10 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
/// Unmount EC shards for a volume on this location.
|
||||
///
|
||||
/// Only shards that are actually mounted decrement the per-shard
|
||||
/// metric — without this guard the gauge would underflow when the
|
||||
/// caller passes a shard that lives on a sibling disk
|
||||
/// (cross-disk reconcile makes that the common case for the same
|
||||
/// `vid` after reconciliation).
|
||||
pub fn unmount_ec_shards(&mut self, vid: VolumeId, shard_ids: &[u32]) {
|
||||
if let Some(ec_vol) = self.ec_volumes.get_mut(&vid) {
|
||||
let collection = ec_vol.collection.clone();
|
||||
for &shard_id in shard_ids {
|
||||
if !ec_vol.has_shard(shard_id as u8) {
|
||||
continue;
|
||||
}
|
||||
ec_vol.remove_shard(shard_id as u8);
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&collection, "ec_shards"])
|
||||
@@ -706,183 +591,6 @@ impl DiskLocation {
|
||||
}
|
||||
}
|
||||
|
||||
/// Auto-discover .ec?? shard files on disk and mount them.
|
||||
///
|
||||
/// Mirrors `loadAllEcShards` in `weed/storage/disk_location_ec.go`.
|
||||
/// Reads the data dir (and idx dir, if separate), groups shards by
|
||||
/// (collection, vid) using sorted iteration so a volume's shards
|
||||
/// land contiguously next to their `.ecx`. When the matching `.ecx`
|
||||
/// is encountered, validates and mounts the group.
|
||||
///
|
||||
/// Without this, an EC shard on disk is invisible after a Rust
|
||||
/// volume-server restart until something explicitly issues
|
||||
/// VolumeEcShardsMount — strict superset of seaweedfs/seaweedfs#9212.
|
||||
pub fn load_all_ec_shards(&mut self) -> io::Result<()> {
|
||||
// Use a HashSet during accumulation so a name appearing in
|
||||
// both data dir and idx dir (idempotent legacy layouts can
|
||||
// place .ecx in either) is processed exactly once. Without
|
||||
// dedup, mount_ec_shards' per-shard metric increment would
|
||||
// double-count for those filenames.
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
let mut entries: Vec<String> = Vec::new();
|
||||
for ent in fs::read_dir(&self.directory)? {
|
||||
let ent = ent?;
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
if seen.insert(name.clone()) {
|
||||
entries.push(name);
|
||||
}
|
||||
}
|
||||
if self.idx_directory != self.directory {
|
||||
for ent in fs::read_dir(&self.idx_directory)? {
|
||||
let ent = ent?;
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
if seen.insert(name.clone()) {
|
||||
entries.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
entries.sort();
|
||||
|
||||
let mut same_volume_shards: Vec<(String, u32)> = Vec::new(); // (filename, shard_id)
|
||||
let mut prev_vid: Option<VolumeId> = None;
|
||||
let mut prev_collection: String = String::new();
|
||||
|
||||
for name in entries {
|
||||
let Some(dot) = name.rfind('.') else {
|
||||
continue;
|
||||
};
|
||||
let (base, ext) = name.split_at(dot); // ext includes leading '.'
|
||||
let Some((collection, vid)) = parse_collection_volume_id(base) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if let Some(shard_id) = parse_ec_shard_extension(ext) {
|
||||
// Ignore zero-byte shards — same as Go.
|
||||
let path = format!("{}/{}", self.directory, name);
|
||||
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
|
||||
if size == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let same_group = match prev_vid {
|
||||
None => true,
|
||||
Some(p) => p == vid && prev_collection == collection,
|
||||
};
|
||||
if same_group {
|
||||
same_volume_shards.push((name, shard_id));
|
||||
} else {
|
||||
self.check_orphaned_shards(
|
||||
&same_volume_shards,
|
||||
&prev_collection,
|
||||
prev_vid.unwrap_or(VolumeId(0)),
|
||||
);
|
||||
same_volume_shards = vec![(name, shard_id)];
|
||||
}
|
||||
prev_vid = Some(vid);
|
||||
prev_collection = collection;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ext == ".ecx"
|
||||
&& prev_vid == Some(vid)
|
||||
&& prev_collection == collection
|
||||
&& !same_volume_shards.is_empty()
|
||||
{
|
||||
let shards = std::mem::take(&mut same_volume_shards);
|
||||
self.handle_found_ecx_file(&shards, &collection, vid);
|
||||
prev_vid = None;
|
||||
prev_collection.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Handle any trailing group that never saw a matching .ecx.
|
||||
self.check_orphaned_shards(
|
||||
&same_volume_shards,
|
||||
&prev_collection,
|
||||
prev_vid.unwrap_or(VolumeId(0)),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate + mount a (collection, vid) group when its `.ecx` is
|
||||
/// found. Mirrors `handleFoundEcxFile` in
|
||||
/// `weed/storage/disk_location_ec.go`.
|
||||
fn handle_found_ecx_file(&mut self, shards: &[(String, u32)], collection: &str, vid: VolumeId) {
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let dat_path = format!("{}.dat", base);
|
||||
let dat_exists = check_dat_file_exists(&dat_path);
|
||||
|
||||
if dat_exists && !self.validate_ec_volume(collection, vid) {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Incomplete or invalid EC volume: .dat exists but validation failed, cleaning up EC files",
|
||||
);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
return;
|
||||
}
|
||||
|
||||
let shard_ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
|
||||
if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids, "") {
|
||||
// mount_ec_shards adds shards one at a time and increments
|
||||
// the per-shard metric for each. If it fails halfway, plain
|
||||
// ec_volumes.remove(vid) would leak metric increments for
|
||||
// the shards that did mount. Drive cleanup through
|
||||
// unmount_ec_shards which mirror-decrements the metric, then
|
||||
// the empty EcVolume drops itself.
|
||||
if dat_exists {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Failed to load EC shards and .dat exists ({}), cleaning up EC files to use .dat",
|
||||
e,
|
||||
);
|
||||
self.unmount_ec_shards(vid, &shard_ids);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
} else {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Failed to load EC shards: {} (this may be normal for distributed EC volumes)",
|
||||
e,
|
||||
);
|
||||
self.unmount_ec_shards(vid, &shard_ids);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors `checkOrphanedShards`: if shards exist on disk without a
|
||||
/// matching `.ecx` AND the legacy `.dat` is still present, treat
|
||||
/// the encoding as interrupted before `.ecx` was written and clean
|
||||
/// up. If `.dat` is gone, leave files alone — they may be
|
||||
/// distributed-EC shards waiting for cross-disk reconciliation.
|
||||
fn check_orphaned_shards(
|
||||
&self,
|
||||
shards: &[(String, u32)],
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> bool {
|
||||
if shards.is_empty() || vid.0 == 0 {
|
||||
return false;
|
||||
}
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let dat_path = format!("{}.dat", base);
|
||||
if check_dat_file_exists(&dat_path) {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
"Found {} EC shards without .ecx file (incomplete encoding interrupted before .ecx), cleaning up",
|
||||
shards.len(),
|
||||
);
|
||||
self.remove_ec_volume_files(collection, vid);
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Total number of EC shards on this location.
|
||||
pub fn ec_shard_count(&self) -> usize {
|
||||
self.ec_volumes
|
||||
@@ -955,69 +663,6 @@ fn calculate_expected_shard_size(dat_file_size: i64) -> i64 {
|
||||
}
|
||||
|
||||
/// Parse a volume filename like "collection_42.dat" or "42.dat" into (collection, VolumeId).
|
||||
/// Parse a `<collection>_<vid>` or `<vid>` base name into its parts.
|
||||
/// Mirrors `parseCollectionVolumeId` in
|
||||
/// `weed/storage/disk_location.go`. Used when iterating raw filenames
|
||||
/// where the extension has already been stripped.
|
||||
///
|
||||
/// `pub(crate)` companion `parse_collection_volume_id_pub` is exposed
|
||||
/// so the cross-disk reconcile in `store_ec_reconcile.rs` can call it
|
||||
/// without re-implementing the parser.
|
||||
pub(crate) fn parse_collection_volume_id_pub(base: &str) -> Option<(String, VolumeId)> {
|
||||
parse_collection_volume_id(base)
|
||||
}
|
||||
|
||||
fn parse_collection_volume_id(base: &str) -> Option<(String, VolumeId)> {
|
||||
if let Some(pos) = base.rfind('_') {
|
||||
let collection = &base[..pos];
|
||||
let id_str = &base[pos + 1..];
|
||||
let id: u32 = id_str.parse().ok()?;
|
||||
Some((collection.to_string(), VolumeId(id)))
|
||||
} else {
|
||||
let id: u32 = base.parse().ok()?;
|
||||
Some((String::new(), VolumeId(id)))
|
||||
}
|
||||
}
|
||||
|
||||
/// `pub(crate)` re-export of [`parse_ec_shard_extension`] for the
|
||||
/// cross-disk reconcile in `store_ec_reconcile.rs`.
|
||||
pub(crate) fn is_ec_shard_extension(ext: &str) -> Option<u32> {
|
||||
parse_ec_shard_extension(ext)
|
||||
}
|
||||
|
||||
/// Recognise EC shard extensions `.ec00`–`.ec255` (the ShardId u8
|
||||
/// range — see `EcVolumeShard`'s typed shard id) and return the
|
||||
/// shard id. Returns `None` for any other extension.
|
||||
///
|
||||
/// Matches the regex `\.ec\d{2,3}` from the Go side, with the same
|
||||
/// per-shard `id <= 255` clamp Go's loader applies via
|
||||
/// `strconv.ParseInt(... 10, 64)` followed by the `if shardId < 0 ||
|
||||
/// shardId > 255` guard. The 3-digit form (`.ec100`–`.ec255`) is
|
||||
/// retained so the parser can still recognise shards from custom
|
||||
/// 32+ ratios that fit in a u8 even though OSS only ships 10+4.
|
||||
fn parse_ec_shard_extension(ext: &str) -> Option<u32> {
|
||||
let rest = ext.strip_prefix(".ec")?;
|
||||
if rest.len() < 2 || rest.len() > 3 {
|
||||
return None;
|
||||
}
|
||||
let id: u32 = rest.parse().ok()?;
|
||||
if id > 255 {
|
||||
return None;
|
||||
}
|
||||
Some(id)
|
||||
}
|
||||
|
||||
/// Robust `.dat` existence check: any unexpected stat error (permission,
|
||||
/// I/O) is treated as "exists" so we don't misclassify local EC as
|
||||
/// distributed EC. Mirrors `checkDatFileExists` in Go.
|
||||
fn check_dat_file_exists(path: &str) -> bool {
|
||||
match fs::metadata(path) {
|
||||
Ok(_) => true,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => false,
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_volume_filename(filename: &str) -> Option<(String, VolumeId)> {
|
||||
let stem = filename
|
||||
.strip_suffix(".dat")
|
||||
@@ -1189,7 +834,7 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(loc.volumes_len(), 2);
|
||||
|
||||
loc.delete_volume(VolumeId(1), false, false).unwrap();
|
||||
loc.delete_volume(VolumeId(1), false).unwrap();
|
||||
assert_eq!(loc.volumes_len(), 1);
|
||||
assert!(loc.find_volume(VolumeId(1)).is_none());
|
||||
}
|
||||
@@ -1262,7 +907,7 @@ mod tests {
|
||||
let shard_path = format!("{}/pics_7.ec00", dir);
|
||||
std::fs::write(&shard_path, b"ec-shard").unwrap();
|
||||
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0]).unwrap();
|
||||
assert!(loc.has_ec_volume(VolumeId(7)));
|
||||
assert!(std::path::Path::new(&shard_path).exists());
|
||||
assert!(std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
|
||||
@@ -1274,56 +919,6 @@ mod tests {
|
||||
assert!(!std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
|
||||
}
|
||||
|
||||
/// #9423: after an RPC `VolumeEcShardsMount` records the source
|
||||
/// volume's disk type on the EcVolume, a later empty-source mount
|
||||
/// call (disk-scan reload, orphan-shard reconcile, restart) must
|
||||
/// not clobber that override back to the physical location's disk
|
||||
/// type — otherwise heartbeat reporting drifts back to "hdd" on
|
||||
/// every reload.
|
||||
#[test]
|
||||
fn test_mount_ec_shards_empty_source_preserves_existing_disk_type() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Plant the first shard so the EC volume can mount, then call
|
||||
// mount_ec_shards with source_disk_type="ssd" — simulating the
|
||||
// VolumeEcShardsMount RPC path.
|
||||
std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd").unwrap();
|
||||
{
|
||||
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume mounted");
|
||||
assert_eq!(
|
||||
ec_vol.disk_type,
|
||||
DiskType::Ssd,
|
||||
"first RPC mount should have set disk type to source's value",
|
||||
);
|
||||
}
|
||||
|
||||
// Plant another shard and re-mount with an empty source —
|
||||
// matching what a disk-scan reload or orphan-shard reconcile
|
||||
// would pass. The previously-recorded "ssd" override must
|
||||
// survive.
|
||||
std::fs::write(format!("{}/pics_7.ec01", dir), b"ec-shard").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[1], "").unwrap();
|
||||
{
|
||||
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume still mounted");
|
||||
assert_eq!(
|
||||
ec_vol.disk_type,
|
||||
DiskType::Ssd,
|
||||
"empty-source remount must not reset disk type to the physical location's",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disk_location_persists_directory_uuid_and_tags() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -1353,152 +948,4 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(reloaded.directory_uuid, directory_uuid);
|
||||
}
|
||||
|
||||
/// Sanity: `parse_ec_shard_extension` accepts `.ec00` through
|
||||
/// `.ec999` and rejects anything else.
|
||||
#[test]
|
||||
fn test_parse_ec_shard_extension() {
|
||||
assert_eq!(parse_ec_shard_extension(".ec00"), Some(0));
|
||||
assert_eq!(parse_ec_shard_extension(".ec13"), Some(13));
|
||||
assert_eq!(parse_ec_shard_extension(".ec999"), None); // > 255
|
||||
assert_eq!(parse_ec_shard_extension(".ec255"), Some(255));
|
||||
assert_eq!(parse_ec_shard_extension(".ec0"), None); // need 2 digits
|
||||
assert_eq!(parse_ec_shard_extension(".ecx"), None);
|
||||
assert_eq!(parse_ec_shard_extension(".ecj"), None);
|
||||
assert_eq!(parse_ec_shard_extension(".dat"), None);
|
||||
assert_eq!(parse_ec_shard_extension("ec00"), None); // missing dot
|
||||
}
|
||||
|
||||
/// Auto-load: shards + .ecx on the same disk get mounted into
|
||||
/// `ec_volumes` after `load_existing_volumes` finishes. Without
|
||||
/// this, EC shards on disk are invisible until something issues
|
||||
/// `VolumeEcShardsMount`.
|
||||
#[test]
|
||||
fn test_load_all_ec_shards_mounts_pairs_with_ecx() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(1234);
|
||||
|
||||
// Two shards + .ecx + .ecj + .vif (the post-encode steady state).
|
||||
for &sid in &[0u8, 4u8] {
|
||||
let path = format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid);
|
||||
std::fs::write(&path, b"shard data nonempty").unwrap();
|
||||
}
|
||||
std::fs::write(format!("{}/{}_{}.ecx", dir, collection, vid.0), vec![0u8; 20])
|
||||
.unwrap();
|
||||
std::fs::write(format!("{}/{}_{}.ecj", dir, collection, vid.0), b"").unwrap();
|
||||
std::fs::write(
|
||||
format!("{}/{}_{}.vif", dir, collection, vid.0),
|
||||
br#"{"version":3}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
loc.load_existing_volumes(NeedleMapKind::InMemory).unwrap();
|
||||
|
||||
let ec_vol = loc.find_ec_volume(vid).expect("EcVolume should be mounted");
|
||||
assert_eq!(ec_vol.shard_count(), 2, "both shards should be loaded");
|
||||
}
|
||||
|
||||
/// Orphan shards with no `.ecx` and no `.dat` must stay on disk
|
||||
/// untouched (distributed-EC scenario, where the .ecx lives on
|
||||
/// another server). Mirrors Go's `checkOrphanedShards` no-op
|
||||
/// branch.
|
||||
#[test]
|
||||
fn test_load_all_ec_shards_keeps_orphan_shards_when_no_dat() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(2345);
|
||||
|
||||
// Shards on disk, but no .ecx and no .dat — distributed-EC
|
||||
// with the .ecx on a sibling/server.
|
||||
let shard_paths: Vec<String> = [0u8, 12u8]
|
||||
.iter()
|
||||
.map(|sid| format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid))
|
||||
.collect();
|
||||
for p in &shard_paths {
|
||||
std::fs::write(p, b"shard data nonempty").unwrap();
|
||||
}
|
||||
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
loc.load_existing_volumes(NeedleMapKind::InMemory).unwrap();
|
||||
|
||||
// No EcVolume should be mounted (no .ecx).
|
||||
assert!(loc.find_ec_volume(vid).is_none());
|
||||
// But the shard files must still be on disk for cross-disk
|
||||
// reconciliation / operator recovery.
|
||||
for p in &shard_paths {
|
||||
assert!(
|
||||
std::path::Path::new(p).exists(),
|
||||
"orphan shard {} was destroyed",
|
||||
p,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Orphan shards WITH a stale `.dat` are treated as an
|
||||
/// interrupted-encoding remnant and cleaned up. Mirrors Go's
|
||||
/// `checkOrphanedShards` cleanup branch. Guards a subtle case
|
||||
/// where a partial encoding wrote shards before .ecx and a crash
|
||||
/// left the .dat behind.
|
||||
#[test]
|
||||
fn test_load_all_ec_shards_cleans_orphan_shards_when_dat_exists() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(3456);
|
||||
|
||||
let shard_paths: Vec<String> = [0u8, 1u8]
|
||||
.iter()
|
||||
.map(|sid| format!("{}/{}_{}.ec{:02}", dir, collection, vid.0, sid))
|
||||
.collect();
|
||||
for p in &shard_paths {
|
||||
std::fs::write(p, b"shard data nonempty").unwrap();
|
||||
}
|
||||
// Stale .dat from interrupted encoding.
|
||||
let dat_path = format!("{}/{}_{}.dat", dir, collection, vid.0);
|
||||
std::fs::write(&dat_path, b"unfinished").unwrap();
|
||||
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// load_existing_volumes processes .dat first; ec auto-load
|
||||
// runs after and finds the orphans. The .dat itself may or may
|
||||
// not be loadable as a regular volume — that's not what this
|
||||
// test is about. Just check that the orphan shard files were
|
||||
// removed.
|
||||
let _ = loc.load_existing_volumes(NeedleMapKind::InMemory);
|
||||
|
||||
for p in &shard_paths {
|
||||
assert!(
|
||||
!std::path::Path::new(p).exists(),
|
||||
"orphan shard {} should have been cleaned up because .dat exists",
|
||||
p,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,28 +16,11 @@ use crate::storage::volume::volume_file_name;
|
||||
/// Calculate .dat file size from the max offset entry in .ecx.
|
||||
/// Reads the volume version from the first EC shard (.ec00) superblock,
|
||||
/// then scans .ecx entries to find the largest (offset + needle_actual_size).
|
||||
///
|
||||
/// `dir` is used both for reading `.ec00` and `.ecx`. For split-disk
|
||||
/// reconciled volumes call [`find_dat_file_size_with_dirs`] instead.
|
||||
pub fn find_dat_file_size(dir: &str, collection: &str, volume_id: VolumeId) -> io::Result<i64> {
|
||||
find_dat_file_size_with_dirs(dir, dir, collection, volume_id)
|
||||
}
|
||||
|
||||
/// Like [`find_dat_file_size`] but lets the caller pass separate dirs
|
||||
/// for `.ec00` (the data shard) and `.ecx` (the sealed index). This
|
||||
/// is the form needed when shards are split across data dirs and the
|
||||
/// `.ecx` lives on a sibling disk's idx dir (#9252).
|
||||
pub fn find_dat_file_size_with_dirs(
|
||||
ec00_dir: &str,
|
||||
ecx_dir: &str,
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<i64> {
|
||||
let ec00_base = volume_file_name(ec00_dir, collection, volume_id);
|
||||
let ecx_base = volume_file_name(ecx_dir, collection, volume_id);
|
||||
let base = volume_file_name(dir, collection, volume_id);
|
||||
|
||||
// Read volume version from .ec00 superblock
|
||||
let ec00_path = format!("{}.ec00", ec00_base);
|
||||
let ec00_path = format!("{}.ec00", base);
|
||||
let mut ec00 = File::open(&ec00_path)?;
|
||||
let mut sb_buf = [0u8; SUPER_BLOCK_SIZE];
|
||||
ec00.read_exact(&mut sb_buf)?;
|
||||
@@ -47,7 +30,7 @@ pub fn find_dat_file_size_with_dirs(
|
||||
let mut dat_size: i64 = SUPER_BLOCK_SIZE as i64;
|
||||
|
||||
// Scan .ecx entries
|
||||
let ecx_path = format!("{}.ecx", ecx_base);
|
||||
let ecx_path = format!("{}.ecx", base);
|
||||
let ecx_data = std::fs::read(&ecx_path)?;
|
||||
let entry_count = ecx_data.len() / NEEDLE_MAP_ENTRY_SIZE;
|
||||
|
||||
@@ -69,10 +52,7 @@ pub fn find_dat_file_size_with_dirs(
|
||||
|
||||
/// Reconstruct a .dat file from EC data shards.
|
||||
///
|
||||
/// Reads from .ec00-.ec09 and writes a new .dat file. All data shards
|
||||
/// must live in `dir`. For the cross-disk reconciled layout where
|
||||
/// shards are split across multiple data dirs of the same node, use
|
||||
/// [`write_dat_file_from_shards_with_dirs`] instead.
|
||||
/// Reads from .ec00-.ec09 and writes a new .dat file.
|
||||
pub fn write_dat_file_from_shards(
|
||||
dir: &str,
|
||||
collection: &str,
|
||||
@@ -80,53 +60,12 @@ pub fn write_dat_file_from_shards(
|
||||
dat_file_size: i64,
|
||||
data_shards: usize,
|
||||
) -> io::Result<()> {
|
||||
let dirs: Vec<String> = (0..data_shards).map(|_| dir.to_string()).collect();
|
||||
write_dat_file_from_shards_with_dirs(
|
||||
dir,
|
||||
collection,
|
||||
volume_id,
|
||||
dat_file_size,
|
||||
data_shards,
|
||||
&dirs,
|
||||
)
|
||||
}
|
||||
|
||||
/// Reconstruct a .dat file from EC data shards, taking the source
|
||||
/// directory for each shard separately.
|
||||
///
|
||||
/// `dat_dir` is where the produced `.dat` is written. `shard_dirs[i]`
|
||||
/// is the directory holding shard `i`. For the simple "all shards in
|
||||
/// one dir" case both can be the same value.
|
||||
///
|
||||
/// Mirrors Go's `WriteDatFile(baseFileName, datFileSize,
|
||||
/// shardFileNames)` shape — Go passes per-shard paths so a
|
||||
/// reconciled volume with shards split across disks of the same
|
||||
/// volume server can still be decoded back to a regular .dat
|
||||
/// (seaweedfs/seaweedfs#9252).
|
||||
pub fn write_dat_file_from_shards_with_dirs(
|
||||
dat_dir: &str,
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
dat_file_size: i64,
|
||||
data_shards: usize,
|
||||
shard_dirs: &[String],
|
||||
) -> io::Result<()> {
|
||||
if shard_dirs.len() < data_shards {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"shard_dirs len {} < data_shards {}",
|
||||
shard_dirs.len(),
|
||||
data_shards
|
||||
),
|
||||
));
|
||||
}
|
||||
let base = volume_file_name(dat_dir, collection, volume_id);
|
||||
let base = volume_file_name(dir, collection, volume_id);
|
||||
let dat_path = format!("{}.dat", base);
|
||||
|
||||
// Open data shards from their individual home dirs.
|
||||
// Open data shards
|
||||
let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
|
||||
.map(|i| EcVolumeShard::new(&shard_dirs[i as usize], collection, volume_id, i))
|
||||
.map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
|
||||
.collect();
|
||||
|
||||
for shard in &mut shards {
|
||||
|
||||
@@ -3,10 +3,9 @@
|
||||
//! Each EcVolume has a sorted index (.ecx) and a deletion journal (.ecj).
|
||||
//! Shards (.ec00-.ec13) may be distributed across multiple servers.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::sync::RwLock;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::pb::master_pb;
|
||||
@@ -29,72 +28,21 @@ pub struct EcVolume {
|
||||
ecx_file: Option<File>,
|
||||
ecx_file_size: i64,
|
||||
ecj_file: Option<File>,
|
||||
/// On-disk size of the .ecj deletion journal. Used only by IO helpers
|
||||
/// (seek / set_len on partial writes) — the authoritative runtime
|
||||
/// delete count comes from `deleted_needles.len()`.
|
||||
ecj_file_size: i64,
|
||||
/// In-memory set of needle ids that have been deleted since the volume
|
||||
/// was encoded. .ecx is immutable at runtime — it only stores the
|
||||
/// sorted (id, offset, size) index written at encode time — and runtime
|
||||
/// deletes are journaled to .ecj + tracked here. Reads consult this
|
||||
/// set to mask out deleted needles on top of the sealed .ecx lookup.
|
||||
/// Seeded from .ecj in `new()` and updated by `journal_delete`.
|
||||
deleted_needles: RwLock<HashSet<NeedleId>>,
|
||||
pub disk_type: DiskType,
|
||||
/// Directory where .ecx/.ecj were actually found (may differ from dir_idx after fallback).
|
||||
ecx_actual_dir: String,
|
||||
/// Maps shard ID -> list of server addresses where that shard exists.
|
||||
/// Used for distributed EC reads across the cluster. Wrapped in
|
||||
/// `RwLock` so the read path can refresh the map (under master
|
||||
/// lookup) without holding the Store write lock — mirrors Go's
|
||||
/// `ShardLocationsLock sync.RWMutex` in `weed/storage/erasure_coding/ec_volume.go`.
|
||||
pub shard_locations: std::sync::RwLock<HashMap<ShardId, Vec<String>>>,
|
||||
/// Wall-clock timestamp of the most recent successful
|
||||
/// `LookupEcVolume` refresh of `shard_locations`. `None` until the
|
||||
/// first refresh. Drives the staleness heuristic in
|
||||
/// `cached_lookup_ec_shard_locations` (mirrors Go's
|
||||
/// `ShardLocationsRefreshTime`).
|
||||
pub shard_locations_refresh_time: std::sync::Mutex<Option<std::time::Instant>>,
|
||||
/// Used for distributed EC reads across the cluster.
|
||||
pub shard_locations: HashMap<ShardId, Vec<String>>,
|
||||
/// EC volume expiration time (unix epoch seconds), set during EC encode from TTL.
|
||||
pub expire_at_sec: u64,
|
||||
}
|
||||
|
||||
/// Locate the `.vif` for a (collection, vid) by preferring the data dir
|
||||
/// and falling back to the idx dir when it lives there instead. The
|
||||
/// fallback covers the cross-disk reconcile path: when a volume's
|
||||
/// shards live on one disk but its `.ecx` / `.ecj` / `.vif` live on a
|
||||
/// sibling disk (seaweedfs/seaweedfs#9212 / #9244), we want to read the
|
||||
/// real `.vif` from the sibling rather than write a stub on the shard
|
||||
/// disk and lose the EC config + dat file size.
|
||||
fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: VolumeId) -> String {
|
||||
let data_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir, collection, volume_id),
|
||||
);
|
||||
if dir_idx != dir && !std::path::Path::new(&data_vif).exists() {
|
||||
let idx_vif = format!(
|
||||
"{}.vif",
|
||||
crate::storage::volume::volume_file_name(dir_idx, collection, volume_id),
|
||||
);
|
||||
if std::path::Path::new(&idx_vif).exists() {
|
||||
return idx_vif;
|
||||
}
|
||||
}
|
||||
data_vif
|
||||
}
|
||||
|
||||
/// Read EC data/parity shard counts from `.vif`, defaulting to the
|
||||
/// build's standard ratio when no `.vif` is present or is malformed.
|
||||
/// Looks at the data dir first, then the idx dir — see [`locate_vif_path`].
|
||||
pub fn read_ec_shard_config(
|
||||
dir: &str,
|
||||
dir_idx: &str,
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> (u32, u32) {
|
||||
pub fn read_ec_shard_config(dir: &str, collection: &str, volume_id: VolumeId) -> (u32, u32) {
|
||||
let mut data_shards = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
|
||||
let mut parity_shards = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
|
||||
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
|
||||
let vif_path = format!("{}.vif", base);
|
||||
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
|
||||
if let Ok(vif_info) =
|
||||
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
|
||||
@@ -121,7 +69,7 @@ impl EcVolume {
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<Self> {
|
||||
let (data_shards, parity_shards) = read_ec_shard_config(dir, dir_idx, collection, volume_id);
|
||||
let (data_shards, parity_shards) = read_ec_shard_config(dir, collection, volume_id);
|
||||
|
||||
let total_shards = (data_shards + parity_shards) as usize;
|
||||
let mut shards = Vec::with_capacity(total_shards);
|
||||
@@ -129,16 +77,10 @@ impl EcVolume {
|
||||
shards.push(None);
|
||||
}
|
||||
|
||||
// Read expire_at_sec, version, and dat_file_size from .vif if
|
||||
// present (matches Go's MaybeLoadVolumeInfo). Prefer the data
|
||||
// dir; fall back to the idx dir for the cross-disk reconcile
|
||||
// case (#9212 / #9244). `dat_file_size` is the source .dat
|
||||
// size at encode time, used both by `locate_ec_shard_needle`
|
||||
// for shard-size math and by the Store-level prune in
|
||||
// `store_ec_reconcile.rs` to verify a sibling-disk .dat is
|
||||
// plausibly the encoding source (#9478).
|
||||
let (expire_at_sec, vif_version, vif_dat_file_size) = {
|
||||
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
// Read expire_at_sec and version from .vif if present (matches Go's MaybeLoadVolumeInfo)
|
||||
let (expire_at_sec, vif_version) = {
|
||||
let base = crate::storage::volume::volume_file_name(dir, collection, volume_id);
|
||||
let vif_path = format!("{}.vif", base);
|
||||
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
|
||||
if let Ok(vif_info) =
|
||||
serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
|
||||
@@ -148,12 +90,12 @@ impl EcVolume {
|
||||
} else {
|
||||
Version::current()
|
||||
};
|
||||
(vif_info.expire_at_sec, ver, vif_info.dat_file_size)
|
||||
(vif_info.expire_at_sec, ver)
|
||||
} else {
|
||||
(0, Version::current(), 0)
|
||||
(0, Version::current())
|
||||
}
|
||||
} else {
|
||||
(0, Version::current(), 0)
|
||||
(0, Version::current())
|
||||
}
|
||||
};
|
||||
|
||||
@@ -164,18 +106,15 @@ impl EcVolume {
|
||||
dir_idx: dir_idx.to_string(),
|
||||
version: vif_version,
|
||||
shards,
|
||||
dat_file_size: vif_dat_file_size,
|
||||
dat_file_size: 0,
|
||||
data_shards,
|
||||
parity_shards,
|
||||
ecx_file: None,
|
||||
ecx_file_size: 0,
|
||||
ecj_file: None,
|
||||
ecj_file_size: 0,
|
||||
deleted_needles: RwLock::new(HashSet::new()),
|
||||
disk_type: DiskType::default(),
|
||||
ecx_actual_dir: dir_idx.to_string(),
|
||||
shard_locations: std::sync::RwLock::new(HashMap::new()),
|
||||
shard_locations_refresh_time: std::sync::Mutex::new(None),
|
||||
shard_locations: HashMap::new(),
|
||||
expire_at_sec,
|
||||
};
|
||||
|
||||
@@ -202,12 +141,10 @@ impl EcVolume {
|
||||
}
|
||||
}
|
||||
|
||||
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency.
|
||||
// Note: Go does NOT replay .ecj into .ecx at volume load (RebuildEcxFile
|
||||
// is only invoked from specific decode/rebuild gRPC handlers), so we
|
||||
// don't either. Tombstones from prior sessions were already written
|
||||
// in-place in .ecx, and the journal grows monotonically until a
|
||||
// decode/rebuild operation folds it in.
|
||||
// Replay .ecj journal into .ecx on startup (matches Go's RebuildEcxFile).
|
||||
vol.rebuild_ecx_from_journal()?;
|
||||
|
||||
// Open .ecj file (deletion journal) — use ecx_actual_dir for consistency
|
||||
let ecj_base =
|
||||
crate::storage::volume::volume_file_name(&vol.ecx_actual_dir, collection, volume_id);
|
||||
let ecj_path = format!("{}.ecj", ecj_base);
|
||||
@@ -217,75 +154,11 @@ impl EcVolume {
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&ecj_path)?;
|
||||
vol.ecj_file_size = ecj_file.metadata()?.len() as i64;
|
||||
vol.ecj_file = Some(ecj_file);
|
||||
|
||||
// Seed the in-memory deleted set from the journal.
|
||||
vol.load_deleted_needles_from_ecj()?;
|
||||
|
||||
Ok(vol)
|
||||
}
|
||||
|
||||
/// Walk the .ecj journal and populate `deleted_needles`. Called once
|
||||
/// from `new()` under exclusive ownership of the just-constructed
|
||||
/// EcVolume, so locking is not strictly required — but we take the
|
||||
/// write lock anyway for symmetry with later mutations.
|
||||
fn load_deleted_needles_from_ecj(&mut self) -> io::Result<()> {
|
||||
let ecj_file = match self.ecj_file.as_ref() {
|
||||
Some(f) => f,
|
||||
None => return Ok(()),
|
||||
};
|
||||
if self.ecj_file_size < NEEDLE_ID_SIZE as i64 {
|
||||
return Ok(());
|
||||
}
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
let mut set = self
|
||||
.deleted_needles
|
||||
.write()
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::Other, "deleted_needles lock poisoned"))?;
|
||||
let mut off: i64 = 0;
|
||||
while off + NEEDLE_ID_SIZE as i64 <= self.ecj_file_size {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecj_file.read_exact_at(&mut buf, off as u64)?;
|
||||
}
|
||||
set.insert(NeedleId::from_bytes(&buf));
|
||||
off += NEEDLE_ID_SIZE as i64;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns (file_count, delete_count) for this EC volume. Mirrors Go's
|
||||
/// `EcVolume.FileAndDeleteCount`:
|
||||
///
|
||||
/// file_count = ecx_file_size / NEEDLE_MAP_ENTRY_SIZE — total
|
||||
/// entries in the sealed sorted .ecx index.
|
||||
/// delete_count = deleted_needles.len() — unique
|
||||
/// runtime deletes tracked in memory (seeded from
|
||||
/// .ecj on load and updated by `journal_delete`).
|
||||
///
|
||||
/// Because each needle delete is applied on exactly one shard holder,
|
||||
/// the admin aggregation sums delete_count across nodes while taking
|
||||
/// file_count from a single holder (they are identical per volume).
|
||||
pub fn file_and_delete_count(&self) -> (u64, u64) {
|
||||
let file_count = (self.ecx_file_size as u64) / (NEEDLE_MAP_ENTRY_SIZE as u64);
|
||||
let delete_count = self
|
||||
.deleted_needles
|
||||
.read()
|
||||
.map(|s| s.len() as u64)
|
||||
.unwrap_or(0);
|
||||
(file_count, delete_count)
|
||||
}
|
||||
|
||||
/// Reports whether the given needle id is in the in-memory deleted set.
|
||||
pub fn is_needle_deleted(&self, needle_id: NeedleId) -> bool {
|
||||
self.deleted_needles
|
||||
.read()
|
||||
.map(|s| s.contains(&needle_id))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
// ---- File names ----
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -293,16 +166,8 @@ impl EcVolume {
|
||||
crate::storage::volume::volume_file_name(&self.dir, &self.collection, self.volume_id)
|
||||
}
|
||||
|
||||
/// Base path for the .ecx / .ecj index pair. Resolved from
|
||||
/// `ecx_actual_dir` (initialized to `dir_idx` and only updated after a
|
||||
/// successful idx-dir → data-dir fallback in `new()`), so every call site
|
||||
/// agrees on the same file regardless of whether the fallback fired.
|
||||
fn idx_base_name(&self) -> String {
|
||||
crate::storage::volume::volume_file_name(
|
||||
&self.ecx_actual_dir,
|
||||
&self.collection,
|
||||
self.volume_id,
|
||||
)
|
||||
crate::storage::volume::volume_file_name(&self.dir_idx, &self.collection, self.volume_id)
|
||||
}
|
||||
|
||||
pub fn ecx_file_name(&self) -> String {
|
||||
@@ -342,20 +207,6 @@ impl EcVolume {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Override the disk type the EC volume (and its already-mounted
|
||||
/// shards) reports under. Used by the `VolumeEcShardsMount` handler
|
||||
/// so the source volume's disk type is preserved across encoding
|
||||
/// (#9423). Not persisted across restarts — disk-scan reload paths
|
||||
/// default to the physical location's disk type.
|
||||
pub fn set_disk_type(&mut self, d: DiskType) {
|
||||
self.disk_type = d.clone();
|
||||
for slot in self.shards.iter_mut() {
|
||||
if let Some(shard) = slot {
|
||||
shard.disk_type = d.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove and close a shard.
|
||||
pub fn remove_shard(&mut self, shard_id: ShardId) {
|
||||
if let Some(ref mut shard) = self.shards[shard_id as usize] {
|
||||
@@ -380,24 +231,6 @@ impl EcVolume {
|
||||
self.shards.iter().filter(|s| s.is_some()).count()
|
||||
}
|
||||
|
||||
/// Reports whether `shard_id` is currently registered to this
|
||||
/// EcVolume (used by the cross-disk reconcile to skip already-
|
||||
/// loaded shards).
|
||||
pub fn has_shard(&self, shard_id: u8) -> bool {
|
||||
self.shards
|
||||
.get(shard_id as usize)
|
||||
.map(|s| s.is_some())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Directory where this EcVolume's `.ecx` was actually opened
|
||||
/// (may differ from `dir_idx` when the legacy "written before
|
||||
/// -dir.idx was set" fallback or the cross-disk reconcile path
|
||||
/// pointed it elsewhere).
|
||||
pub fn ecx_actual_dir(&self) -> &str {
|
||||
&self.ecx_actual_dir
|
||||
}
|
||||
|
||||
pub fn is_time_to_destroy(&self) -> bool {
|
||||
self.expire_at_sec > 0
|
||||
&& SystemTime::now()
|
||||
@@ -422,8 +255,6 @@ impl EcVolume {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let (file_count, delete_count) = self.file_and_delete_count();
|
||||
|
||||
vec![master_pb::VolumeEcShardInformationMessage {
|
||||
id: self.volume_id.0,
|
||||
collection: self.collection.clone(),
|
||||
@@ -432,53 +263,23 @@ impl EcVolume {
|
||||
disk_type: self.disk_type.to_string(),
|
||||
expire_at_sec: self.expire_at_sec,
|
||||
disk_id,
|
||||
file_count,
|
||||
delete_count,
|
||||
..Default::default()
|
||||
}]
|
||||
}
|
||||
|
||||
// ---- Shard locations (distributed tracking) ----
|
||||
|
||||
/// Set the list of server addresses for a single shard ID. Does
|
||||
/// NOT touch `shard_locations_refresh_time` — a per-shard write
|
||||
/// from inside a multi-shard population (e.g. iterating the
|
||||
/// `LookupEcVolume` response shard-by-shard) would otherwise
|
||||
/// flip the staleness flag while the map is still incomplete,
|
||||
/// letting a concurrent reader observe `needs_refresh == false`
|
||||
/// against a half-populated cache and return NotFound for the
|
||||
/// not-yet-inserted shards.
|
||||
///
|
||||
/// Callers populating the whole cache atomically should use
|
||||
/// [`Self::replace_shard_locations`] instead — it swaps the
|
||||
/// entire map under the write lock and advances the refresh
|
||||
/// timestamp in one step.
|
||||
pub fn set_shard_locations(&self, shard_id: ShardId, locations: Vec<String>) {
|
||||
self.shard_locations
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(shard_id, locations);
|
||||
/// Set the list of server addresses for a given shard ID.
|
||||
pub fn set_shard_locations(&mut self, shard_id: ShardId, locations: Vec<String>) {
|
||||
self.shard_locations.insert(shard_id, locations);
|
||||
}
|
||||
|
||||
/// Atomically replace the entire shard-locations map and stamp
|
||||
/// the refresh time. Used by the distributed-read path's
|
||||
/// post-`LookupEcVolume` write-back so the cache transitions
|
||||
/// from old → fresh in a single observable step — concurrent
|
||||
/// readers either see the full prior map or the full new map,
|
||||
/// never an intermediate state with the freshness flag flipped.
|
||||
pub fn replace_shard_locations(&self, locations: HashMap<ShardId, Vec<String>>) {
|
||||
*self.shard_locations.write().unwrap() = locations;
|
||||
*self.shard_locations_refresh_time.lock().unwrap() = Some(std::time::Instant::now());
|
||||
}
|
||||
|
||||
/// Get a cloned list of server addresses for a given shard ID.
|
||||
pub fn get_shard_locations(&self, shard_id: ShardId) -> Vec<String> {
|
||||
/// Get the list of server addresses for a given shard ID.
|
||||
pub fn get_shard_locations(&self, shard_id: ShardId) -> &[String] {
|
||||
self.shard_locations
|
||||
.read()
|
||||
.unwrap()
|
||||
.get(&shard_id)
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.map(|v| v.as_slice())
|
||||
.unwrap_or(&[])
|
||||
}
|
||||
|
||||
// ---- Index operations ----
|
||||
@@ -512,13 +313,6 @@ impl EcVolume {
|
||||
|
||||
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
// Apply runtime deletion state on top of the sealed .ecx
|
||||
// lookup: a needle in the in-memory deleted set is
|
||||
// reported with TOMBSTONE_FILE_SIZE even though the .ecx
|
||||
// record itself is untouched.
|
||||
if self.is_needle_deleted(needle_id) {
|
||||
return Ok(Some((offset, TOMBSTONE_FILE_SIZE)));
|
||||
}
|
||||
return Ok(Some((offset, size)));
|
||||
} else if key < needle_id {
|
||||
lo = mid + 1;
|
||||
@@ -776,41 +570,38 @@ impl EcVolume {
|
||||
|
||||
// ---- Deletion ----
|
||||
|
||||
/// Write `TOMBSTONE_FILE_SIZE` over the Size field of an existing .ecx
|
||||
/// entry, matching Go's `MarkNeedleDeleted`. Only used by the offline
|
||||
/// `rebuild_ecx_from_journal` path — the runtime delete path does not
|
||||
/// touch .ecx because the index is treated as an immutable sorted
|
||||
/// (id, offset, size) table. Returns `false` if the needle is not in
|
||||
/// the index (ignored by callers) and an error on IO failure.
|
||||
fn tombstone_ecx_entry(&self, needle_id: NeedleId) -> io::Result<bool> {
|
||||
let ecx_file = self.ecx_file.as_ref().ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"ec volume {} has no open .ecx file (closed or corrupt)",
|
||||
self.volume_id.0
|
||||
),
|
||||
)
|
||||
})?;
|
||||
/// Mark a needle as deleted in the .ecx file in-place.
|
||||
/// Matches Go's MarkNeedleDeleted: binary search the .ecx, then overwrite
|
||||
/// the size field with TOMBSTONE_FILE_SIZE.
|
||||
fn mark_needle_deleted_in_ecx(&self, needle_id: NeedleId) -> io::Result<bool> {
|
||||
let ecx_file = match self.ecx_file.as_ref() {
|
||||
Some(f) => f,
|
||||
None => return Ok(false),
|
||||
};
|
||||
|
||||
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
|
||||
if entry_count == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
// Binary search for the needle
|
||||
let mut lo: usize = 0;
|
||||
let mut hi: usize = entry_count;
|
||||
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
|
||||
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
|
||||
}
|
||||
let (key, _offset, _old_size) = idx_entry_from_bytes(&entry_buf);
|
||||
|
||||
let (key, _offset, _size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
// Found — overwrite the size field with TOMBSTONE_FILE_SIZE
|
||||
let size_offset = file_offset + NEEDLE_ID_SIZE as u64 + OFFSET_SIZE as u64;
|
||||
let mut size_buf = [0u8; SIZE_SIZE];
|
||||
TOMBSTONE_FILE_SIZE.to_bytes(&mut size_buf);
|
||||
@@ -826,18 +617,13 @@ impl EcVolume {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
|
||||
Ok(false) // not found
|
||||
}
|
||||
|
||||
/// Replay .ecj journal entries into .ecx: for each needle id in .ecj,
|
||||
/// overwrite its .ecx size field with a tombstone, then remove the
|
||||
/// journal file. Mirrors Go's `RebuildEcxFile`, which is invoked from
|
||||
/// specific decode / rebuild gRPC handlers — it is intentionally
|
||||
/// **not** called on volume load (runtime reads consult
|
||||
/// `deleted_needles` instead). The rebuild is atomic with respect to
|
||||
/// the journal: if any individual write fails the .ecj file is left
|
||||
/// in place and the error is propagated so tombstones are not lost.
|
||||
#[allow(dead_code)]
|
||||
/// Replay .ecj journal entries into .ecx on startup.
|
||||
/// Matches Go's RebuildEcxFile: for each needle ID in .ecj, marks it
|
||||
/// deleted in .ecx, then removes the .ecj file.
|
||||
fn rebuild_ecx_from_journal(&mut self) -> io::Result<()> {
|
||||
let ecj_path = self.ecj_file_name();
|
||||
if !std::path::Path::new(&ecj_path).exists() {
|
||||
@@ -856,23 +642,14 @@ impl EcVolume {
|
||||
break;
|
||||
}
|
||||
let needle_id = NeedleId::from_bytes(&data[start..start + NEEDLE_ID_SIZE]);
|
||||
// A needle that never made it into .ecx is fine (e.g. the
|
||||
// delete raced against encode). Any other IO error aborts the
|
||||
// rebuild so the journal survives to be retried later.
|
||||
self.tombstone_ecx_entry(needle_id)?;
|
||||
// Errors for individual entries are non-fatal (needle may not exist in .ecx)
|
||||
let _ = self.mark_needle_deleted_in_ecx(needle_id);
|
||||
}
|
||||
|
||||
// Durably flush the newly-written .ecx tombstones before dropping
|
||||
// the journal: the writes went through write_all_at and may still
|
||||
// be in page cache.
|
||||
if let Some(ref ecx_file) = self.ecx_file {
|
||||
ecx_file.sync_all()?;
|
||||
}
|
||||
// Remove the .ecj file after replay (matches Go)
|
||||
let _ = fs::remove_file(&ecj_path);
|
||||
|
||||
// Fold successful — drop and recreate the journal, clear the
|
||||
// in-memory deleted set (all of its contents are now materialized
|
||||
// in .ecx), and reset the cached size.
|
||||
fs::remove_file(&ecj_path)?;
|
||||
// Re-create .ecj for future deletions
|
||||
let ecj_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
@@ -880,126 +657,27 @@ impl EcVolume {
|
||||
.append(true)
|
||||
.open(&ecj_path)?;
|
||||
self.ecj_file = Some(ecj_file);
|
||||
self.ecj_file_size = 0;
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.clear();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Deletion journal ----
|
||||
|
||||
/// Record a needle delete: append the id to the .ecj deletion journal
|
||||
/// and insert it into the in-memory deleted set. `.ecx` is not touched
|
||||
/// at runtime — it is a sealed sorted (id, offset, size) index and
|
||||
/// runtime deletion state lives exclusively in .ecj + `deleted_needles`.
|
||||
/// A lookup via `find_needle_from_ecx` masks the id out by returning
|
||||
/// `TOMBSTONE_FILE_SIZE` on a subsequent read.
|
||||
///
|
||||
/// The .ecj append is the durable commit point. On any failure the
|
||||
/// file is truncated back to the pre-append length so the on-disk
|
||||
/// journal and in-memory state cannot drift. Only after the sync
|
||||
/// succeeds is the id published into the set, so a failure leaves
|
||||
/// the delete invisible to readers.
|
||||
/// Append a deleted needle ID to the .ecj journal and mark in .ecx.
|
||||
/// Matches Go's DeleteNeedleFromEcx: marks in .ecx first, then journals.
|
||||
pub fn journal_delete(&mut self, needle_id: NeedleId) -> io::Result<()> {
|
||||
// Look the needle up read-only. Missing is a silent no-op; a
|
||||
// pre-existing .ecx tombstone (from a prior decode/rebuild) is
|
||||
// mirrored into the in-memory set so delete_count stays accurate
|
||||
// without needing to walk .ecx on every heartbeat.
|
||||
match self.find_needle_from_ecx_raw(needle_id)? {
|
||||
None => return Ok(()),
|
||||
Some((_, size)) if size.is_deleted() => {
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.insert(needle_id);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
// Mark deleted in .ecx in-place (matches Go's MarkNeedleDeleted)
|
||||
let _ = self.mark_needle_deleted_in_ecx(needle_id);
|
||||
let ecj_file = self
|
||||
.ecj_file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
|
||||
|
||||
// Idempotent fast path for repeat deletes — avoids the journal
|
||||
// append entirely so the derived delete_count stays stable.
|
||||
if self.is_needle_deleted(needle_id) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let prev_ecj_size = self.ecj_file_size;
|
||||
let append_result: io::Result<()> = {
|
||||
let ecj_file = self
|
||||
.ecj_file
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecj file not open"))?;
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
needle_id.to_bytes(&mut buf);
|
||||
ecj_file
|
||||
.write_all(&buf)
|
||||
.and_then(|_| ecj_file.sync_all())
|
||||
};
|
||||
|
||||
match append_result {
|
||||
Ok(()) => {
|
||||
self.ecj_file_size += NEEDLE_ID_SIZE as i64;
|
||||
if let Ok(mut set) = self.deleted_needles.write() {
|
||||
set.insert(needle_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
// write_all may have extended the file on disk before
|
||||
// sync_all failed; truncate back to the known-good size so
|
||||
// the on-disk journal never drifts past `deleted_needles`.
|
||||
if let Some(ecj) = self.ecj_file.as_mut() {
|
||||
if let Err(trunc_err) = ecj.set_len(prev_ecj_size as u64) {
|
||||
tracing::error!(
|
||||
volume_id = self.volume_id.0,
|
||||
needle_id = needle_id.0,
|
||||
truncate_error = %trunc_err,
|
||||
"failed to truncate ecj after append failure"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal: binary search .ecx without masking by `deleted_needles`.
|
||||
/// Used by `journal_delete` so a repeat delete can still see the raw
|
||||
/// pre-existing .ecx tombstone from a prior rebuild.
|
||||
fn find_needle_from_ecx_raw(
|
||||
&self,
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<Option<(Offset, Size)>> {
|
||||
let ecx_file = self
|
||||
.ecx_file
|
||||
.as_ref()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "ecx file not open"))?;
|
||||
let entry_count = self.ecx_file_size as usize / NEEDLE_MAP_ENTRY_SIZE;
|
||||
if entry_count == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut lo: usize = 0;
|
||||
let mut hi: usize = entry_count;
|
||||
let mut entry_buf = [0u8; NEEDLE_MAP_ENTRY_SIZE];
|
||||
while lo < hi {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
let file_offset = (mid * NEEDLE_MAP_ENTRY_SIZE) as u64;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileExt;
|
||||
ecx_file.read_exact_at(&mut entry_buf, file_offset)?;
|
||||
}
|
||||
let (key, offset, size) = idx_entry_from_bytes(&entry_buf);
|
||||
if key == needle_id {
|
||||
return Ok(Some((offset, size)));
|
||||
} else if key < needle_id {
|
||||
lo = mid + 1;
|
||||
} else {
|
||||
hi = mid;
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
needle_id.to_bytes(&mut buf);
|
||||
ecj_file.write_all(&buf)?;
|
||||
ecj_file.sync_all()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a deleted needle ID to the .ecj journal, validating the cookie first.
|
||||
@@ -1100,17 +778,15 @@ impl EcVolume {
|
||||
let _ = fs::remove_file(format!("{}.ecx", actual_base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", actual_base));
|
||||
let _ = fs::remove_file(format!("{}.vif", actual_base));
|
||||
// Also sweep the originally-configured idx dir in case stale files
|
||||
// exist there (ecx_file_name() / ecj_file_name() now resolve from
|
||||
// ecx_actual_dir, so we have to build the idx-dir paths explicitly).
|
||||
// Also try the configured idx dir and data dir in case files exist in either
|
||||
if self.ecx_actual_dir != self.dir_idx {
|
||||
let _ = fs::remove_file(self.ecx_file_name());
|
||||
let _ = fs::remove_file(self.ecj_file_name());
|
||||
let idx_base = crate::storage::volume::volume_file_name(
|
||||
&self.dir_idx,
|
||||
&self.collection,
|
||||
self.volume_id,
|
||||
);
|
||||
let _ = fs::remove_file(format!("{}.ecx", idx_base));
|
||||
let _ = fs::remove_file(format!("{}.ecj", idx_base));
|
||||
let _ = fs::remove_file(format!("{}.vif", idx_base));
|
||||
}
|
||||
if self.ecx_actual_dir != self.dir && self.dir_idx != self.dir {
|
||||
@@ -1183,33 +859,16 @@ mod tests {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
// .ecj append is gated on a live->tombstone transition in .ecx, so
|
||||
// the fixture must contain the needles we are about to delete.
|
||||
let entries = vec![
|
||||
(NeedleId(10), Offset::from_actual_offset(8), Size(100)),
|
||||
(NeedleId(20), Offset::from_actual_offset(200), Size(200)),
|
||||
];
|
||||
write_ecx_file(dir, "", VolumeId(1), &entries);
|
||||
// Need ecx file for EcVolume::new to succeed
|
||||
write_ecx_file(dir, "", VolumeId(1), &[]);
|
||||
|
||||
let mut vol = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
|
||||
let (fc0, dc0) = vol.file_and_delete_count();
|
||||
assert_eq!((fc0, dc0), (2, 0));
|
||||
|
||||
vol.journal_delete(NeedleId(10)).unwrap();
|
||||
vol.journal_delete(NeedleId(20)).unwrap();
|
||||
|
||||
let deleted = vol.read_deleted_needles().unwrap();
|
||||
assert_eq!(deleted, vec![NeedleId(10), NeedleId(20)]);
|
||||
|
||||
let (fc, dc) = vol.file_and_delete_count();
|
||||
assert_eq!((fc, dc), (2, 2));
|
||||
|
||||
// Idempotent re-delete must not bump delete_count.
|
||||
vol.journal_delete(NeedleId(10)).unwrap();
|
||||
// Deleting a missing needle must not bump delete_count either.
|
||||
vol.journal_delete(NeedleId(999)).unwrap();
|
||||
let (fc, dc) = vol.file_and_delete_count();
|
||||
assert_eq!((fc, dc), (2, 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4,8 +4,6 @@ pub mod idx;
|
||||
pub mod needle;
|
||||
pub mod needle_map;
|
||||
pub mod store;
|
||||
pub mod store_ec_mirror;
|
||||
pub mod store_ec_reconcile;
|
||||
pub mod super_block;
|
||||
pub mod types;
|
||||
pub mod volume;
|
||||
|
||||
@@ -19,7 +19,6 @@ use compact_map::CompactMap;
|
||||
use redb::{Database, Durability, ReadableDatabase, ReadableTable, TableDefinition};
|
||||
|
||||
use crate::storage::idx;
|
||||
use crate::storage::needle::needle::get_actual_size;
|
||||
use crate::storage::types::*;
|
||||
|
||||
// ============================================================================
|
||||
@@ -64,11 +63,6 @@ pub struct NeedleMapMetric {
|
||||
pub deletion_count: AtomicI64,
|
||||
pub deletion_byte_count: AtomicU64,
|
||||
pub max_file_key: AtomicU64,
|
||||
/// Largest (offset.to_actual_offset() + get_actual_size(size, version))
|
||||
/// observed during the load walk. Used at volume load to verify that no
|
||||
/// .idx entry references bytes past the end of .dat (issue #8928)
|
||||
/// without paying for a second linear scan.
|
||||
pub max_needle_end: AtomicI64,
|
||||
}
|
||||
|
||||
impl NeedleMapMetric {
|
||||
@@ -114,29 +108,6 @@ impl NeedleMapMetric {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Update `max_needle_end` if this entry's (offset + actual size) exceeds
|
||||
/// the running maximum. Skips deleted/zero-offset entries because they
|
||||
/// don't reserve space in .dat.
|
||||
fn maybe_set_max_needle_end(&self, offset: Offset, size: Size, version: Version) {
|
||||
if offset.is_zero() || !size.is_valid() {
|
||||
return;
|
||||
}
|
||||
let end = offset.to_actual_offset() + get_actual_size(size, version);
|
||||
loop {
|
||||
let current = self.max_needle_end.load(Ordering::Relaxed);
|
||||
if end <= current {
|
||||
break;
|
||||
}
|
||||
if self
|
||||
.max_needle_end
|
||||
.compare_exchange(current, end, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -146,9 +117,9 @@ impl NeedleMapMetric {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum NeedleMapKind {
|
||||
InMemory,
|
||||
Redb,
|
||||
RedbMedium,
|
||||
RedbLarge,
|
||||
LevelDb,
|
||||
LevelDbMedium,
|
||||
LevelDbLarge,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -192,10 +163,9 @@ impl CompactNeedleMap {
|
||||
}
|
||||
|
||||
/// Load from an .idx file, building the in-memory map.
|
||||
pub fn load_from_idx<R: Read + Seek>(reader: &mut R, version: Version) -> io::Result<Self> {
|
||||
pub fn load_from_idx<R: Read + Seek>(reader: &mut R) -> io::Result<Self> {
|
||||
let mut nm = CompactNeedleMap::new();
|
||||
idx::walk_index_file(reader, 0, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
nm.delete_from_map(key);
|
||||
} else {
|
||||
@@ -212,13 +182,6 @@ impl CompactNeedleMap {
|
||||
self.idx_file_offset = offset;
|
||||
}
|
||||
|
||||
/// True when an .idx file writer is attached. A read-only load leaves
|
||||
/// this `false` — set_writable() must reattach a writer or subsequent
|
||||
/// puts silently skip the disk append.
|
||||
pub fn has_idx_writer(&self) -> bool {
|
||||
self.idx_file.is_some()
|
||||
}
|
||||
|
||||
// ---- Map operations ----
|
||||
|
||||
/// Insert or update an entry. Appends to .idx file if present.
|
||||
@@ -321,12 +284,6 @@ impl CompactNeedleMap {
|
||||
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
self.metric.max_needle_end.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
self.idx_file_offset
|
||||
}
|
||||
@@ -474,7 +431,7 @@ impl RedbNeedleMap {
|
||||
|
||||
/// Rebuild metrics by scanning all entries in the redb table.
|
||||
/// Called when reusing an existing .rdb without a full rebuild.
|
||||
fn rebuild_metrics_from_db(&self, version: Version) -> io::Result<()> {
|
||||
fn rebuild_metrics_from_db(&self) -> io::Result<()> {
|
||||
let txn = self
|
||||
.db
|
||||
.begin_read()
|
||||
@@ -496,8 +453,6 @@ impl RedbNeedleMap {
|
||||
arr.copy_from_slice(bytes);
|
||||
let nv = unpack_needle_value(&arr);
|
||||
self.metric.maybe_set_max_file_key(key);
|
||||
self.metric
|
||||
.maybe_set_max_needle_end(nv.offset, nv.size, version);
|
||||
if nv.size.is_valid() {
|
||||
self.metric.file_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.metric
|
||||
@@ -522,24 +477,20 @@ impl RedbNeedleMap {
|
||||
/// 2. If .idx size matches → reuse .rdb, rebuild metrics from scan
|
||||
/// 3. If .idx is larger → replay new entries incrementally
|
||||
/// 4. Otherwise (missing, corrupted, .idx smaller) → full rebuild
|
||||
pub fn load_from_idx<R: Read + Seek>(
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
pub fn load_from_idx<R: Read + Seek>(db_path: &str, reader: &mut R) -> io::Result<Self> {
|
||||
let idx_size = reader.seek(io::SeekFrom::End(0))?;
|
||||
reader.seek(io::SeekFrom::Start(0))?;
|
||||
|
||||
// Try to reuse existing .rdb
|
||||
if Path::new(db_path).exists() {
|
||||
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size, version) {
|
||||
if let Ok(nm) = Self::try_reuse_rdb(db_path, reader, idx_size) {
|
||||
return Ok(nm);
|
||||
}
|
||||
// Reuse failed — fall through to full rebuild
|
||||
reader.seek(io::SeekFrom::Start(0))?;
|
||||
}
|
||||
|
||||
Self::full_rebuild(db_path, reader, idx_size, version)
|
||||
Self::full_rebuild(db_path, reader, idx_size)
|
||||
}
|
||||
|
||||
/// Try to reuse an existing .rdb file. Returns Ok if successful,
|
||||
@@ -548,7 +499,6 @@ impl RedbNeedleMap {
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
idx_size: u64,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
let db = Database::open(db_path)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("redb open: {}", e)))?;
|
||||
@@ -573,7 +523,7 @@ impl RedbNeedleMap {
|
||||
}
|
||||
|
||||
// Rebuild metrics from existing data
|
||||
nm.rebuild_metrics_from_db(version)?;
|
||||
nm.rebuild_metrics_from_db()?;
|
||||
|
||||
if stored_idx_size < idx_size {
|
||||
// .idx grew — replay new entries incrementally
|
||||
@@ -584,7 +534,6 @@ impl RedbNeedleMap {
|
||||
io::Error::new(io::ErrorKind::Other, format!("redb open_table: {}", e))
|
||||
})?;
|
||||
idx::walk_index_file(reader, start_entry, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
let key_u64: u64 = key.into();
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
// Delete: look up old value for metric update, then
|
||||
@@ -658,7 +607,6 @@ impl RedbNeedleMap {
|
||||
db_path: &str,
|
||||
reader: &mut R,
|
||||
idx_size: u64,
|
||||
version: Version,
|
||||
) -> io::Result<Self> {
|
||||
let _ = std::fs::remove_file(db_path);
|
||||
let nm = RedbNeedleMap::new(db_path)?;
|
||||
@@ -666,7 +614,6 @@ impl RedbNeedleMap {
|
||||
// Collect entries from idx file, resolving duplicates/deletions
|
||||
let mut entries: HashMap<NeedleId, Option<NeedleValue>> = HashMap::new();
|
||||
idx::walk_index_file(reader, 0, |key, offset, size| {
|
||||
nm.metric.maybe_set_max_needle_end(offset, size, version);
|
||||
if offset.is_zero() || size.is_deleted() {
|
||||
entries.insert(key, None);
|
||||
} else {
|
||||
@@ -712,11 +659,6 @@ impl RedbNeedleMap {
|
||||
self.idx_file_offset = offset;
|
||||
}
|
||||
|
||||
/// True when an .idx file writer is attached. See CompactNeedleMap.
|
||||
pub fn has_idx_writer(&self) -> bool {
|
||||
self.idx_file.is_some()
|
||||
}
|
||||
|
||||
// ---- Map operations ----
|
||||
|
||||
/// Insert or update an entry. Writes to idx file first, then redb.
|
||||
@@ -846,12 +788,6 @@ impl RedbNeedleMap {
|
||||
NeedleId(self.metric.max_file_key.load(Ordering::Relaxed))
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. See `NeedleMapMetric::maybe_set_max_needle_end`.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
self.metric.max_needle_end.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
self.idx_file_offset
|
||||
}
|
||||
@@ -1012,14 +948,6 @@ impl NeedleMap {
|
||||
}
|
||||
}
|
||||
|
||||
/// True when an .idx file writer is attached.
|
||||
pub fn has_idx_writer(&self) -> bool {
|
||||
match self {
|
||||
NeedleMap::InMemory(nm) => nm.has_idx_writer(),
|
||||
NeedleMap::Redb(nm) => nm.has_idx_writer(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Content byte count.
|
||||
pub fn content_size(&self) -> u64 {
|
||||
match self {
|
||||
@@ -1060,16 +988,6 @@ impl NeedleMap {
|
||||
}
|
||||
}
|
||||
|
||||
/// Largest (offset + actual size) seen during the load walk; 0 if the
|
||||
/// map is empty. Used at volume load to detect .idx entries that
|
||||
/// reference past the end of .dat (issue #8928) without a second scan.
|
||||
pub fn max_needle_end(&self) -> i64 {
|
||||
match self {
|
||||
NeedleMap::InMemory(nm) => nm.max_needle_end(),
|
||||
NeedleMap::Redb(nm) => nm.max_needle_end(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Index file size in bytes.
|
||||
pub fn index_file_size(&self) -> u64 {
|
||||
match self {
|
||||
@@ -1239,7 +1157,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let mut cursor = Cursor::new(idx_data);
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut cursor, Version::current()).unwrap();
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut cursor).unwrap();
|
||||
|
||||
assert!(nm.get(NeedleId(1)).is_some());
|
||||
assert!(nm.get(NeedleId(2)).is_none()); // deleted
|
||||
@@ -1382,7 +1300,7 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let mut cursor = Cursor::new(idx_data);
|
||||
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor, Version::current()).unwrap();
|
||||
let nm = RedbNeedleMap::load_from_idx(db_path.to_str().unwrap(), &mut cursor).unwrap();
|
||||
|
||||
assert!(nm.get(NeedleId(1)).is_some());
|
||||
assert!(nm.get(NeedleId(2)).is_none()); // deleted and removed
|
||||
@@ -1459,7 +1377,7 @@ mod tests {
|
||||
|
||||
// Load back with CompactNeedleMap to verify
|
||||
let mut idx_file = std::fs::File::open(&idx_path).unwrap();
|
||||
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file, Version::current()).unwrap();
|
||||
let loaded = CompactNeedleMap::load_from_idx(&mut idx_file).unwrap();
|
||||
assert_eq!(loaded.file_count(), 2); // only live entries
|
||||
assert!(loaded.get(NeedleId(1)).is_some());
|
||||
assert!(loaded.get(NeedleId(2)).is_none()); // deleted, not saved
|
||||
|
||||
@@ -84,30 +84,6 @@ impl Store {
|
||||
}
|
||||
|
||||
self.locations.push(loc);
|
||||
|
||||
// First scrub partial EC artefacts left on one disk by an
|
||||
// interrupted encode while the source .dat still lives on a
|
||||
// sibling disk of the same store. The per-disk loader cannot
|
||||
// see the sibling .dat and so loads the partial shards as if
|
||||
// they were a distributed-EC layout, which makes the volume
|
||||
// server heartbeat both a regular replica and an EC shard set
|
||||
// for the same vid (seaweedfs/seaweedfs#9478). Running before
|
||||
// the cross-disk reconcile keeps that pass from later
|
||||
// re-loading shards we just cleaned up.
|
||||
self.prune_incomplete_ec_with_sibling_dat();
|
||||
|
||||
// Physically mirror EC sidecars onto every shard-bearing disk
|
||||
// so each disk mounts self-contained. Must run before the
|
||||
// cross-disk reconciler so the orphan pass can prefer the
|
||||
// local idx_directory.
|
||||
self.mirror_ec_metadata_to_shard_disks();
|
||||
|
||||
// Cross-disk fallback for orphan shards — ec.balance can land
|
||||
// shards on one disk while leaving the index on another. Still
|
||||
// needed after the mirror pass for volumes whose mirror failed
|
||||
// (read-only target, partial copy).
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -119,9 +95,6 @@ impl Store {
|
||||
tracing::error!("load_new_volumes error in {}: {}", loc.directory, e);
|
||||
}
|
||||
}
|
||||
self.prune_incomplete_ec_with_sibling_dat();
|
||||
self.mirror_ec_metadata_to_shard_disks();
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
}
|
||||
|
||||
// ---- Volume lookup ----
|
||||
@@ -224,72 +197,6 @@ impl Store {
|
||||
best.map(|(i, _)| i)
|
||||
}
|
||||
|
||||
/// Returns the index of the disk that should receive a new EC
|
||||
/// shard / index file for `(collection, vid)`. Selection order:
|
||||
///
|
||||
/// 1. a disk that already has the EC volume mounted (in-memory state),
|
||||
/// 2. a disk that owns the `.ecx` file on disk (volume not yet mounted),
|
||||
/// 3. any HDD with free space,
|
||||
/// 4. any disk with free space.
|
||||
///
|
||||
/// Step 2 is the missing primitive that pinned subsequent shards to
|
||||
/// the first-shard disk during `ec.rebuild`: rebuild only sets
|
||||
/// `CopyEcxFile=true` on the first shard, then relies on auto-select
|
||||
/// to land later shards on the same disk. Without an on-disk check,
|
||||
/// `has_ec_volume` returns false (no mount yet) and the fallback
|
||||
/// picks "any HDD with free space" — which can split shards from
|
||||
/// their index files across disks of the same node and lose them at
|
||||
/// startup. See seaweedfs/seaweedfs#9212.
|
||||
///
|
||||
/// `data_shard_count` is taken as a parameter rather than read from
|
||||
/// `DATA_SHARDS_COUNT` so custom-ratio builds can swap the default
|
||||
/// without touching this helper.
|
||||
///
|
||||
/// Single pass over `self.locations` with tier scoring; the
|
||||
/// highest-tier disk wins, ties broken by free shard-slot count.
|
||||
/// Mirrors `Store.FindEcShardTargetLocation` in
|
||||
/// `weed/storage/store_ec.go`.
|
||||
pub fn find_ec_shard_target_location(
|
||||
&self,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
data_shard_count: u32,
|
||||
) -> Option<usize> {
|
||||
const TIER_ANY_DISK: u8 = 1;
|
||||
const TIER_HDD: u8 = 2;
|
||||
const TIER_ECX_ON_DISK: u8 = 3;
|
||||
const TIER_MOUNTED: u8 = 4;
|
||||
|
||||
let mut best: Option<(usize, u8, i64)> = None;
|
||||
for (i, loc) in self.locations.iter().enumerate() {
|
||||
if loc.is_disk_space_low.load(Ordering::Relaxed) {
|
||||
continue;
|
||||
}
|
||||
let free = ec_free_shard_count(loc, data_shard_count);
|
||||
if free <= 0 {
|
||||
continue;
|
||||
}
|
||||
let mut tier = TIER_ANY_DISK;
|
||||
if loc.disk_type == DiskType::HardDrive {
|
||||
tier = TIER_HDD;
|
||||
}
|
||||
if loc.has_ecx_file_on_disk(collection, vid) {
|
||||
tier = TIER_ECX_ON_DISK;
|
||||
}
|
||||
if loc.has_ec_volume(vid) {
|
||||
tier = TIER_MOUNTED;
|
||||
}
|
||||
let better = match best {
|
||||
None => true,
|
||||
Some((_, b_tier, b_free)) => tier > b_tier || (tier == b_tier && free > b_free),
|
||||
};
|
||||
if better {
|
||||
best = Some((i, tier, free));
|
||||
}
|
||||
}
|
||||
best.map(|(i, _, _)| i)
|
||||
}
|
||||
|
||||
/// Create a new volume, placing it on the location with the most free space.
|
||||
pub fn add_volume(
|
||||
&mut self,
|
||||
@@ -322,18 +229,11 @@ impl Store {
|
||||
)
|
||||
}
|
||||
|
||||
/// Delete a volume from any location. When keep_remote_data is true the
|
||||
/// cloud-tier object backing the volume is left intact — used by moves
|
||||
/// where another server is taking over the same .vif.
|
||||
pub fn delete_volume(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
only_empty: bool,
|
||||
keep_remote_data: bool,
|
||||
) -> Result<(), VolumeError> {
|
||||
/// Delete a volume from any location.
|
||||
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
|
||||
for loc in &mut self.locations {
|
||||
if loc.find_volume(vid).is_some() {
|
||||
return loc.delete_volume(vid, only_empty, keep_remote_data);
|
||||
return loc.delete_volume(vid, only_empty);
|
||||
}
|
||||
}
|
||||
Err(VolumeError::NotFound)
|
||||
@@ -592,8 +492,7 @@ impl Store {
|
||||
let vol_count = loc.volumes_len() as i32;
|
||||
let loc_ec_shards = loc.ec_shard_count();
|
||||
let ec_equivalent = ((loc_ec_shards
|
||||
+ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT
|
||||
- 1)
|
||||
+ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT)
|
||||
/ crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT)
|
||||
as i32;
|
||||
let mut max_count = vol_count + ec_equivalent;
|
||||
@@ -651,25 +550,22 @@ impl Store {
|
||||
))
|
||||
})?;
|
||||
|
||||
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids, "")
|
||||
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids)
|
||||
}
|
||||
|
||||
/// Mount a single EC shard, searching all locations for the shard file.
|
||||
/// Matches Go's Store.MountEcShards which mounts one shard at a time.
|
||||
/// `source_disk_type` is the source volume's disk type from the
|
||||
/// `VolumeEcShardsMount` RPC (#9423); pass `""` for non-RPC paths.
|
||||
pub fn mount_ec_shard(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_id: u32,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
for loc in &mut self.locations {
|
||||
// Check if the shard file exists on this location
|
||||
let shard = EcVolumeShard::new(&loc.directory, collection, vid, shard_id as u8);
|
||||
if std::path::Path::new(&shard.file_name()).exists() {
|
||||
loc.mount_ec_shards(vid, collection, &[shard_id], source_disk_type)?;
|
||||
loc.mount_ec_shards(vid, collection, &[shard_id])?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -680,18 +576,11 @@ impl Store {
|
||||
}
|
||||
|
||||
/// Unmount EC shards for a volume (batch).
|
||||
///
|
||||
/// Iterates every location that has an EcVolume for `vid` and
|
||||
/// asks it to unmount whatever subset of `shard_ids` it actually
|
||||
/// has — required for cross-disk reconciled volumes where the
|
||||
/// requested shards may legitimately live on different disks of
|
||||
/// the same store (#9252). DiskLocation::unmount_ec_shards
|
||||
/// already skips shards that aren't mounted, so this is safe to
|
||||
/// fan out blindly.
|
||||
pub fn unmount_ec_shards(&mut self, vid: VolumeId, shard_ids: &[u32]) {
|
||||
for loc in &mut self.locations {
|
||||
if loc.has_ec_volume(vid) {
|
||||
loc.unmount_ec_shards(vid, shard_ids);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -699,23 +588,11 @@ impl Store {
|
||||
/// Unmount a single EC shard, searching all locations.
|
||||
/// Matches Go's Store.UnmountEcShards which unmounts one shard at a time.
|
||||
pub fn unmount_ec_shard(&mut self, vid: VolumeId, shard_id: u32) -> Result<(), VolumeError> {
|
||||
// Walk all locations rather than stopping at the first with the
|
||||
// vid — split-disk reconciled volumes can have the same vid on
|
||||
// multiple disks, with the target shard on any of them.
|
||||
for disk_id in 0..self.locations.len() {
|
||||
let has_shard = self.locations[disk_id]
|
||||
.find_ec_volume(vid)
|
||||
.is_some_and(|ec_vol| ec_vol.has_shard(shard_id as u8));
|
||||
if !has_shard {
|
||||
continue;
|
||||
for loc in &mut self.locations {
|
||||
if loc.has_ec_volume(vid) {
|
||||
loc.unmount_ec_shards(vid, &[shard_id]);
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!(
|
||||
volume_id = vid.0,
|
||||
shard_id,
|
||||
disk_id,
|
||||
"UnmountEcShards"
|
||||
);
|
||||
self.locations[disk_id].unmount_ec_shards(vid, &[shard_id]);
|
||||
}
|
||||
// Go returns nil if shard not found (no error)
|
||||
Ok(())
|
||||
@@ -746,90 +623,6 @@ impl Store {
|
||||
self.locations.iter().any(|loc| loc.has_ec_volume(vid))
|
||||
}
|
||||
|
||||
/// Returns every disk_id on this store that has an EcVolume entry
|
||||
/// for `vid`. Useful for diagnostic logging when a single
|
||||
/// `has_ec_volume` hit hides which disk is actually holding the
|
||||
/// mount (e.g., the ReceiveFile mounted-volume guard).
|
||||
/// Mirrors Go's `Store.FindEcVolumeDiskIds`.
|
||||
pub fn find_ec_volume_disk_ids(&self, vid: VolumeId) -> Vec<u32> {
|
||||
let mut ids = Vec::new();
|
||||
for (idx, loc) in self.locations.iter().enumerate() {
|
||||
if loc.has_ec_volume(vid) {
|
||||
ids.push(idx as u32);
|
||||
}
|
||||
}
|
||||
ids
|
||||
}
|
||||
|
||||
/// Returns the index of the disk location that has `(vid, shard_id)`
|
||||
/// mounted, if any. Mirrors Go's `Store.findEcShard` and is the
|
||||
/// right primitive for read/unmount/delete operations on a single
|
||||
/// shard, since reconciliation can mount the same `vid` on multiple
|
||||
/// disks (each holding a disjoint subset of the shards). Without
|
||||
/// this, callers using `find_ec_volume(vid)` would only see the
|
||||
/// first disk and miss shards that live on a sibling.
|
||||
pub fn find_ec_shard_location(&self, vid: VolumeId, shard_id: u32) -> Option<usize> {
|
||||
for (i, loc) in self.locations.iter().enumerate() {
|
||||
if let Some(ecv) = loc.find_ec_volume(vid) {
|
||||
if ecv.has_shard(shard_id as u8) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Like [`Self::find_ec_shard_location`] but returns the EcVolume
|
||||
/// reference directly. Borrows the store immutably for the
|
||||
/// EcVolume's lifetime.
|
||||
pub fn find_ec_volume_with_shard(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
shard_id: u32,
|
||||
) -> Option<&EcVolume> {
|
||||
for loc in &self.locations {
|
||||
if let Some(ecv) = loc.find_ec_volume(vid) {
|
||||
if ecv.has_shard(shard_id as u8) {
|
||||
return Some(ecv);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Aggregate the data dir of each mounted shard for `vid` across
|
||||
/// all locations. Returns the EcVolume to use for metadata
|
||||
/// (`.ecx`, `dir_idx`, etc — any per-disk EcVolume for this vid
|
||||
/// will do, since they all open the same `.ecx`) and a vector of
|
||||
/// per-shard data dirs (`None` when the shard isn't mounted on
|
||||
/// any disk).
|
||||
///
|
||||
/// Mirrors Go's `Store.CollectEcShards` in
|
||||
/// `weed/storage/store_ec.go`. The decoder needs per-shard paths
|
||||
/// to support cross-disk reconciled volumes whose shards are
|
||||
/// split across data dirs.
|
||||
pub fn collect_ec_shard_dirs(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
max_shard_count: usize,
|
||||
) -> Option<(&EcVolume, Vec<Option<String>>)> {
|
||||
let mut found_vol: Option<&EcVolume> = None;
|
||||
let mut dirs: Vec<Option<String>> = vec![None; max_shard_count];
|
||||
for loc in &self.locations {
|
||||
if let Some(ecv) = loc.find_ec_volume(vid) {
|
||||
if found_vol.is_none() {
|
||||
found_vol = Some(ecv);
|
||||
}
|
||||
for shard_id in 0..max_shard_count {
|
||||
if dirs[shard_id].is_none() && ecv.has_shard(shard_id as u8) {
|
||||
dirs[shard_id] = Some(loc.directory.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
found_vol.map(|v| (v, dirs))
|
||||
}
|
||||
|
||||
pub fn delete_expired_ec_volumes(
|
||||
&mut self,
|
||||
) -> (
|
||||
@@ -1126,48 +919,6 @@ fn save_vif_volume_info(path: &str, info: &VifVolumeInfo) -> Result<(), VolumeEr
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Free EC shard capacity of `loc`, expressed in shard slots (not
|
||||
/// volume-equivalent slots). `find_free_location_predicate` does similar
|
||||
/// math but divides by `data_shard_count` at the end. That truncation can
|
||||
/// exclude a disk that still has room for several individual shards
|
||||
/// (e.g. `MaxVolumeCount=1`, `EcShardCount=1`, `data_shard_count=10` →
|
||||
/// reports 0 despite 9 free shard slots), which would re-route subsequent
|
||||
/// shards off the `.ecx`-owning disk and re-introduce the orphan-shard
|
||||
/// layout this helper is meant to prevent (seaweedfs/seaweedfs#9212).
|
||||
///
|
||||
/// `MaxVolumeCount == 0` is the "unlimited" sentinel honoured elsewhere
|
||||
/// in the store; report a synthetic large free count decremented by
|
||||
/// current usage so unlimited disks are eligible and tie-breaks still
|
||||
/// prefer the less-loaded one.
|
||||
///
|
||||
/// Mirrors `ecFreeShardCount` in `weed/storage/store_ec.go`.
|
||||
fn ec_free_shard_count(loc: &DiskLocation, data_shard_count: u32) -> i64 {
|
||||
if data_shard_count == 0 {
|
||||
return 0;
|
||||
}
|
||||
let dsc = data_shard_count as i64;
|
||||
let max = loc.max_volume_count.load(Ordering::Relaxed) as i64;
|
||||
if max <= 0 {
|
||||
// Synthetic "unlimited" capacity. Use a large but
|
||||
// well-below-overflow base (1 << 60 ≈ 1.15e18) and saturating
|
||||
// arithmetic so even pathological usage never wraps and
|
||||
// tie-breaks among unlimited disks still meaningfully prefer
|
||||
// the less-loaded one. Clamp to ≥ 1 so the disk stays eligible
|
||||
// for placement no matter how loaded it is.
|
||||
const UNLIMITED_FREE: i64 = 1 << 60;
|
||||
let used = (loc.volumes_len() as i64)
|
||||
.saturating_mul(dsc)
|
||||
.saturating_add(loc.ec_shard_count() as i64);
|
||||
return UNLIMITED_FREE.saturating_sub(used).max(1);
|
||||
}
|
||||
let mut free = (max - loc.volumes_len() as i64) * dsc;
|
||||
free -= loc.ec_shard_count() as i64;
|
||||
if free < 0 {
|
||||
return 0;
|
||||
}
|
||||
free
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
@@ -1176,7 +927,6 @@ fn ec_free_shard_count(loc: &DiskLocation, data_shard_count: u32) -> i64 {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::storage::needle::needle::Needle;
|
||||
use crate::storage::volume::volume_file_name;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_test_store(dirs: &[&str]) -> Store {
|
||||
@@ -1408,13 +1158,7 @@ mod tests {
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
// Use a large volume_size_limit so the unused-space difference between
|
||||
// preallocate=true (0) and preallocate=false (~2 × limit) is big enough
|
||||
// that integer-division rounding and disk-free fluctuations between the
|
||||
// two get_disk_stats calls cannot make the quotients equal.
|
||||
store
|
||||
.volume_size_limit
|
||||
.store(100 * 1024 * 1024, Ordering::Relaxed);
|
||||
store.volume_size_limit.store(1024, Ordering::Relaxed);
|
||||
store
|
||||
.add_volume(
|
||||
VolumeId(61),
|
||||
@@ -1524,7 +1268,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/expired_ec_case_9.ec00", dir), b"expired").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0])
|
||||
.unwrap();
|
||||
store.find_ec_volume_mut(VolumeId(9)).unwrap().expire_at_sec = 1;
|
||||
|
||||
@@ -1550,151 +1294,4 @@ mod tests {
|
||||
let err = store.read_volume_needle(VolumeId(99), &mut n);
|
||||
assert!(matches!(err, Err(VolumeError::NotFound)));
|
||||
}
|
||||
|
||||
/// Build a Store with N HDD disk locations under a single TempDir.
|
||||
/// Returns the store and the TempDir guard so callers keep the dirs
|
||||
/// alive for the test's lifetime.
|
||||
fn make_ec_target_test_store(numdirs: usize) -> (Store, TempDir) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
for i in 0..numdirs {
|
||||
let path = tmp.path().join(format!("data{}", i));
|
||||
std::fs::create_dir_all(&path).unwrap();
|
||||
store
|
||||
.add_location(
|
||||
path.to_str().unwrap(),
|
||||
path.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
(store, tmp)
|
||||
}
|
||||
|
||||
/// Reproduces the placement half of seaweedfs/seaweedfs#9212. After
|
||||
/// `ec.rebuild`'s first VolumeEcShardsCopy lands `.ecx` on disk N,
|
||||
/// subsequent shards arrive with `CopyEcxFile=false` and rely on
|
||||
/// auto-select. Without an on-disk check, `has_ec_volume` returns
|
||||
/// false (no mount yet) and the fallback picks "any HDD with free
|
||||
/// space" — splitting shards from their index files across disks.
|
||||
/// `find_ec_shard_target_location` must pin to the `.ecx`-owning
|
||||
/// disk via the on-disk check.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_pins_to_ecx_on_disk() {
|
||||
let (store, _tmp) = make_ec_target_test_store(3);
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(1093);
|
||||
|
||||
// Drop a sealed .ecx onto disk 2. Nothing is mounted yet — this
|
||||
// is the state right after the first VolumeEcShardsCopy with
|
||||
// CopyEcxFile=true and before any VolumeEcShardsMount has run.
|
||||
let base = volume_file_name(&store.locations[2].idx_directory, collection, vid);
|
||||
std::fs::write(format!("{}.ecx", base), vec![0u8; 20]).unwrap();
|
||||
|
||||
let got = store.find_ec_shard_target_location(collection, vid, 10);
|
||||
assert_eq!(
|
||||
got,
|
||||
Some(2),
|
||||
"placement leaked off the .ecx-owning disk; got {:?}",
|
||||
got,
|
||||
);
|
||||
}
|
||||
|
||||
/// An already-mounted EC volume on disk 1 must win over a stray
|
||||
/// `.ecx` on disk 2. Protects the post-startup steady state from
|
||||
/// being perturbed by leftover index files from a prior failed move.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_prefers_mounted_over_ecx() {
|
||||
let (mut store, _tmp) = make_ec_target_test_store(3);
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(2222);
|
||||
|
||||
// Mount an EC shard on disk 1 so has_ec_volume returns true.
|
||||
std::fs::write(
|
||||
format!("{}/{}_{}.ec00", store.locations[1].directory, collection, vid.0),
|
||||
b"shard data",
|
||||
)
|
||||
.unwrap();
|
||||
store.locations[1]
|
||||
.mount_ec_shards(vid, collection, &[0], "")
|
||||
.unwrap();
|
||||
|
||||
// Stray .ecx on disk 2 must not win.
|
||||
let base = volume_file_name(&store.locations[2].idx_directory, collection, vid);
|
||||
std::fs::write(format!("{}.ecx", base), vec![0u8; 20]).unwrap();
|
||||
|
||||
let got = store.find_ec_shard_target_location(collection, vid, 10);
|
||||
assert_eq!(got, Some(1), "expected the mounted disk to win; got {:?}", got);
|
||||
}
|
||||
|
||||
/// Cold-volume case: no mount, no `.ecx` anywhere on this server.
|
||||
/// Selection should still fall through to an HDD fallback.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_falls_through_to_hdd_when_nothing_matches() {
|
||||
let (store, _tmp) = make_ec_target_test_store(2);
|
||||
let got = store.find_ec_shard_target_location("grafana-loki", VolumeId(3333), 10);
|
||||
assert!(got.is_some(), "expected an HDD fallback");
|
||||
assert_eq!(store.locations[got.unwrap()].disk_type, DiskType::HardDrive);
|
||||
}
|
||||
|
||||
/// `MaxVolumeCount=0` is the "unlimited disk" sentinel. The previous
|
||||
/// formula returned a negative free count for unlimited disks,
|
||||
/// making placement skip them entirely. The unlimited branch in
|
||||
/// `ec_free_shard_count` must report a synthetic large free count.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_honours_unlimited_disk() {
|
||||
let (store, _tmp) = make_ec_target_test_store(1);
|
||||
store.locations[0]
|
||||
.max_volume_count
|
||||
.store(0, Ordering::Relaxed);
|
||||
|
||||
let got = store.find_ec_shard_target_location("grafana-loki", VolumeId(4444), 10);
|
||||
assert_eq!(
|
||||
got,
|
||||
Some(0),
|
||||
"expected the only (unlimited) disk to be picked",
|
||||
);
|
||||
}
|
||||
|
||||
/// Truncation hazard: with `MaxVolumeCount=1, EcShardCount=1,
|
||||
/// data_shard_count=10`, the old formula `(1*10 - 1) / 10 = 0`
|
||||
/// would have rounded the disk to "full" and routed subsequent
|
||||
/// shards elsewhere — the orphan-shard layout this PR exists to
|
||||
/// prevent. Accounting in shard slots fixes it.
|
||||
#[test]
|
||||
fn test_find_ec_shard_target_location_tight_provisioning_keeps_ecx_disk() {
|
||||
let (mut store, _tmp) = make_ec_target_test_store(2);
|
||||
store.locations[0]
|
||||
.max_volume_count
|
||||
.store(1, Ordering::Relaxed);
|
||||
store.locations[1]
|
||||
.max_volume_count
|
||||
.store(1, Ordering::Relaxed);
|
||||
|
||||
let collection = "grafana-loki";
|
||||
let vid = VolumeId(5555);
|
||||
|
||||
// Seed disk 1 with one EC shard so it owns .ecx and has 9
|
||||
// free shard slots remaining; the old formula would have
|
||||
// rounded that to 0.
|
||||
std::fs::write(
|
||||
format!("{}/{}_{}.ec00", store.locations[1].directory, collection, vid.0),
|
||||
b"shard data",
|
||||
)
|
||||
.unwrap();
|
||||
store.locations[1]
|
||||
.mount_ec_shards(vid, collection, &[0], "")
|
||||
.unwrap();
|
||||
|
||||
let got = store.find_ec_shard_target_location(collection, vid, 10);
|
||||
assert_eq!(
|
||||
got,
|
||||
Some(1),
|
||||
"expected the .ecx-owning disk (1 shard placed, 9 free shard slots) to be picked; got {:?}",
|
||||
got,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,421 +0,0 @@
|
||||
//! Physical EC sidecar mirroring across disks of the same volume
|
||||
//! server. Mirrors `weed/storage/store_ec_mirror.go`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage::disk_location::{parse_collection_volume_id_pub, DiskLocation};
|
||||
use crate::storage::store::Store;
|
||||
use crate::storage::types::VolumeId;
|
||||
|
||||
// Listed in `EcVolume::new`'s open order.
|
||||
const EC_MIRRORED_SIDECARS: &[&str] = &[".ecx", ".ecj", ".vif"];
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
struct EcKey {
|
||||
collection: String,
|
||||
vid: VolumeId,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct EcxOwner {
|
||||
location: usize,
|
||||
idx_dir: String,
|
||||
data_dir: String,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Mirror EC sidecars onto every shard-bearing disk that lacks
|
||||
/// them, so each disk mounts self-contained. Runs before
|
||||
/// `reconcile_ec_shards_across_disks` so the orphan pass can
|
||||
/// prefer the local idx_directory.
|
||||
pub fn mirror_ec_metadata_to_shard_disks(&mut self) {
|
||||
if self.locations.len() < 2 {
|
||||
return;
|
||||
}
|
||||
let owners = self.index_ecx_owners_for_mirror();
|
||||
if owners.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Two-pass: gather work under an immutable borrow, then apply
|
||||
// copies under independent mutable borrows.
|
||||
struct Mirror<'a> {
|
||||
target_idx: usize,
|
||||
owner: &'a EcxOwner,
|
||||
collection: String,
|
||||
vid: VolumeId,
|
||||
}
|
||||
let mut mirrors: Vec<Mirror> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let orphans = collect_shard_disk_volumes(loc);
|
||||
for (key, _shards) in orphans {
|
||||
let Some(owner) = owners.get(&key) else {
|
||||
continue;
|
||||
};
|
||||
if owner.location == loc_idx {
|
||||
continue;
|
||||
}
|
||||
if disk_has_all_sidecars(loc, &key.collection, key.vid) {
|
||||
continue;
|
||||
}
|
||||
mirrors.push(Mirror {
|
||||
target_idx: loc_idx,
|
||||
owner,
|
||||
collection: key.collection,
|
||||
vid: key.vid,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for m in mirrors {
|
||||
let loc_dir = self.locations[m.target_idx].directory.clone();
|
||||
let loc_idx_dir = self.locations[m.target_idx].idx_directory.clone();
|
||||
match mirror_sidecars_for_volume(
|
||||
&m.owner.idx_dir,
|
||||
&m.owner.data_dir,
|
||||
&loc_dir,
|
||||
&loc_idx_dir,
|
||||
&m.collection,
|
||||
m.vid,
|
||||
) {
|
||||
Ok(0) => {}
|
||||
Ok(copied) => {
|
||||
info!(
|
||||
volume_id = m.vid.0,
|
||||
collection = %m.collection,
|
||||
from = %m.owner.data_dir,
|
||||
to = %loc_dir,
|
||||
copied,
|
||||
"mirrored EC sidecar(s) for same-disk invariant",
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
volume_id = m.vid.0,
|
||||
collection = %m.collection,
|
||||
from = %m.owner.data_dir,
|
||||
to = %loc_dir,
|
||||
error = %e,
|
||||
"mirror EC sidecars failed; cross-disk fallback will handle this volume",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Records both idx_dir (where .ecx was found) and data_dir, so
|
||||
// the mirror can resolve .vif from data_dir even when .ecx lives
|
||||
// in idx_directory.
|
||||
fn index_ecx_owners_for_mirror(&self) -> HashMap<EcKey, EcxOwner> {
|
||||
let mut owners: HashMap<EcKey, EcxOwner> = HashMap::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let mut seen: Vec<&str> = Vec::with_capacity(2);
|
||||
for scan in [loc.idx_directory.as_str(), loc.directory.as_str()] {
|
||||
if scan.is_empty() || seen.contains(&scan) {
|
||||
continue;
|
||||
}
|
||||
seen.push(scan);
|
||||
let Ok(read) = fs::read_dir(scan) else {
|
||||
continue;
|
||||
};
|
||||
for ent in read.flatten() {
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
let Some(base) = name.strip_suffix(".ecx") else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
owners
|
||||
.entry(EcKey { collection, vid })
|
||||
.or_insert_with(|| EcxOwner {
|
||||
location: loc_idx,
|
||||
idx_dir: scan.to_string(),
|
||||
data_dir: loc.directory.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
owners
|
||||
}
|
||||
}
|
||||
|
||||
// Checks the modern routing and the opposite directory — without
|
||||
// that fallback, a destination with a legacy pre-`-dir.idx` .ecx in
|
||||
// its data dir would be re-mirrored into idx_directory.
|
||||
fn disk_has_all_sidecars(loc: &DiskLocation, collection: &str, vid: VolumeId) -> bool {
|
||||
for ext in EC_MIRRORED_SIDECARS {
|
||||
let primary = sidecar_dest_path(&loc.directory, &loc.idx_directory, collection, vid, ext);
|
||||
if path_is_regular_file(&primary) {
|
||||
continue;
|
||||
}
|
||||
if loc.idx_directory != loc.directory {
|
||||
let (fallback_data, fallback_idx) = if *ext == ".vif" {
|
||||
(loc.idx_directory.as_str(), loc.directory.as_str())
|
||||
} else {
|
||||
(loc.directory.as_str(), loc.directory.as_str())
|
||||
};
|
||||
let fallback = sidecar_dest_path(fallback_data, fallback_idx, collection, vid, ext);
|
||||
if path_is_regular_file(&fallback) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn path_is_regular_file(path: &str) -> bool {
|
||||
fs::metadata(path).map(|m| !m.is_dir()).unwrap_or(false)
|
||||
}
|
||||
|
||||
// `.ecx`/`.ecj` route to idx_directory, `.vif` to directory.
|
||||
fn sidecar_dest_path(
|
||||
data_dir: &str,
|
||||
idx_dir: &str,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
ext: &str,
|
||||
) -> String {
|
||||
let dir = if ext == ".vif" { data_dir } else { idx_dir };
|
||||
if collection.is_empty() {
|
||||
format!("{}/{}{}", dir, vid.0, ext)
|
||||
} else {
|
||||
format!("{}/{}_{}{}", dir, collection, vid.0, ext)
|
||||
}
|
||||
}
|
||||
|
||||
fn mirror_sidecars_for_volume(
|
||||
src_idx_dir: &str,
|
||||
src_data_dir: &str,
|
||||
dst_data_dir: &str,
|
||||
dst_idx_dir: &str,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
) -> io::Result<usize> {
|
||||
let mut copied = 0usize;
|
||||
for ext in EC_MIRRORED_SIDECARS {
|
||||
let dst = sidecar_dest_path(dst_data_dir, dst_idx_dir, collection, vid, ext);
|
||||
// An existing local copy is authoritative — it may be newer
|
||||
// than the owner's after a delete journal append.
|
||||
if fs::metadata(&dst).is_ok() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let candidates = [
|
||||
sidecar_dest_path(src_data_dir, src_idx_dir, collection, vid, ext),
|
||||
sidecar_dest_path(src_idx_dir, src_data_dir, collection, vid, ext),
|
||||
];
|
||||
let mut src_path: Option<String> = None;
|
||||
for c in candidates.iter() {
|
||||
if fs::metadata(c).map(|m| !m.is_dir()).unwrap_or(false) {
|
||||
src_path = Some(c.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
let Some(src) = src_path else {
|
||||
continue;
|
||||
};
|
||||
copy_sidecar_atomic(Path::new(&src), Path::new(&dst))?;
|
||||
copied += 1;
|
||||
}
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
fn copy_sidecar_atomic(src: &Path, dst: &Path) -> io::Result<()> {
|
||||
if let Some(parent) = dst.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut src_file = fs::File::open(src)?;
|
||||
let tmp = {
|
||||
let mut s = dst.as_os_str().to_owned();
|
||||
s.push(".mirror.tmp");
|
||||
std::path::PathBuf::from(s)
|
||||
};
|
||||
let _ = fs::remove_file(&tmp);
|
||||
let mut dst_file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&tmp)?;
|
||||
let mut buf = [0u8; 64 * 1024];
|
||||
loop {
|
||||
let n = src_file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
dst_file.write_all(&buf[..n])?;
|
||||
}
|
||||
dst_file.sync_all()?;
|
||||
drop(dst_file);
|
||||
if let Err(e) = fs::rename(&tmp, dst) {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(e);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_shard_disk_volumes(loc: &DiskLocation) -> HashMap<EcKey, Vec<String>> {
|
||||
let mut out: HashMap<EcKey, Vec<String>> = HashMap::new();
|
||||
let Ok(read) = fs::read_dir(&loc.directory) else {
|
||||
return out;
|
||||
};
|
||||
for ent in read.flatten() {
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
let Some(dot) = name.rfind('.') else {
|
||||
continue;
|
||||
};
|
||||
let (base, ext) = name.split_at(dot);
|
||||
if crate::storage::disk_location::is_ec_shard_extension(ext).is_none() {
|
||||
continue;
|
||||
}
|
||||
match ent.metadata() {
|
||||
Ok(meta) if meta.len() > 0 => {}
|
||||
_ => continue,
|
||||
}
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
out.entry(EcKey { collection, vid })
|
||||
.or_default()
|
||||
.push(name);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::types::DiskType;
|
||||
use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo};
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn plant_shard(dir: &Path, collection: &str, vid: u32, shard_id: u8) {
|
||||
let path = if collection.is_empty() {
|
||||
dir.join(format!("{}.ec{:02}", vid, shard_id))
|
||||
} else {
|
||||
dir.join(format!("{}_{}.ec{:02}", collection, vid, shard_id))
|
||||
};
|
||||
fs::write(&path, b"shard data nonempty").unwrap();
|
||||
}
|
||||
|
||||
fn plant_ecx(dir: &Path, collection: &str, vid: u32, bytes: &[u8]) {
|
||||
let path = dir.join(format!("{}_{}.ecx", collection, vid));
|
||||
fs::write(&path, bytes).unwrap();
|
||||
}
|
||||
|
||||
fn plant_ecj(dir: &Path, collection: &str, vid: u32, bytes: &[u8]) {
|
||||
let path = dir.join(format!("{}_{}.ecj", collection, vid));
|
||||
fs::write(&path, bytes).unwrap();
|
||||
}
|
||||
|
||||
fn plant_vif(dir: &Path, collection: &str, vid: u32, data_shards: u32, parity_shards: u32) {
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards,
|
||||
parity_shards,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let path = dir.join(format!("{}_{}.vif", collection, vid));
|
||||
fs::write(&path, serde_json::to_string(&vif).unwrap()).unwrap();
|
||||
}
|
||||
|
||||
fn add_loc(store: &mut Store, dir: &Path) {
|
||||
store
|
||||
.add_location(
|
||||
dir.to_str().unwrap(),
|
||||
dir.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_copies_sidecars_to_shard_only_disk() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
fs::create_dir_all(&dir0).unwrap();
|
||||
fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "video-recordings";
|
||||
let vid = 4121u32;
|
||||
|
||||
plant_shard(&dir0, collection, vid, 0);
|
||||
plant_shard(&dir0, collection, vid, 12);
|
||||
plant_shard(&dir1, collection, vid, 1);
|
||||
|
||||
let ecx = vec![0xA1u8; 20];
|
||||
let ecj = vec![0xB2u8; 16];
|
||||
plant_ecx(&dir1, collection, vid, &ecx);
|
||||
plant_ecj(&dir1, collection, vid, &ecj);
|
||||
plant_vif(&dir1, collection, vid, 10, 4);
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
for ext in [".ecx", ".ecj", ".vif"] {
|
||||
let dst = dir0.join(format!("{}_{}{}", collection, vid, ext));
|
||||
assert!(
|
||||
dst.exists(),
|
||||
"mirror did not install sidecar {} on dir0",
|
||||
ext
|
||||
);
|
||||
}
|
||||
let ecx_dst = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
|
||||
assert_eq!(ecx_dst, ecx, ".ecx mirrored bytes differ from source");
|
||||
let ecj_dst = fs::read(dir0.join(format!("{}_{}.ecj", collection, vid))).unwrap();
|
||||
assert_eq!(ecj_dst, ecj, ".ecj mirrored bytes differ from source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_preserves_existing_destination_sidecars() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
fs::create_dir_all(&dir0).unwrap();
|
||||
fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "video-recordings";
|
||||
let vid = 7777u32;
|
||||
|
||||
plant_shard(&dir0, collection, vid, 0);
|
||||
plant_shard(&dir0, collection, vid, 12);
|
||||
plant_shard(&dir1, collection, vid, 1);
|
||||
|
||||
let ecx_owner = vec![0xC3u8; 20];
|
||||
let ecx_local = vec![0x5Au8; 20];
|
||||
let ecj_bytes = vec![0xD4u8; 16];
|
||||
plant_ecx(&dir1, collection, vid, &ecx_owner);
|
||||
plant_ecj(&dir1, collection, vid, &ecj_bytes);
|
||||
plant_vif(&dir1, collection, vid, 10, 4);
|
||||
plant_ecx(&dir0, collection, vid, &ecx_local);
|
||||
plant_ecj(&dir0, collection, vid, &ecj_bytes);
|
||||
plant_vif(&dir0, collection, vid, 10, 4);
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
add_loc(&mut store, &dir0);
|
||||
add_loc(&mut store, &dir1);
|
||||
|
||||
let post = fs::read(dir0.join(format!("{}_{}.ecx", collection, vid))).unwrap();
|
||||
assert_eq!(post, ecx_local, "mirror overwrote dir0's existing .ecx");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,8 +19,6 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use tracing::warn;
|
||||
|
||||
#[cfg(test)]
|
||||
use crate::storage::idx;
|
||||
use crate::storage::needle::needle::{self, get_actual_size, Needle, NeedleError};
|
||||
use crate::storage::needle_map::{CompactNeedleMap, NeedleMap, NeedleMapKind, RedbNeedleMap};
|
||||
use crate::storage::super_block::{ReplicaPlacement, SuperBlock, SUPER_BLOCK_SIZE};
|
||||
@@ -75,24 +73,6 @@ pub enum VolumeError {
|
||||
StreamingUnsupported,
|
||||
}
|
||||
|
||||
/// Returns true when a needle read failed because the on-disk bytes are
|
||||
/// unreadable in a permanent way (offset past EOF, header corruption, CRC
|
||||
/// mismatch, malformed v2/v3/v4 fields). Vacuum can safely drop such entries
|
||||
/// during compaction. Anything else (real disk EIO, ERROR_CRC on Windows,
|
||||
/// network timeouts, EROFS, etc.) is transient or environmental and must
|
||||
/// abort the compaction so an operator notices.
|
||||
fn is_skippable_needle_read_error(e: &VolumeError) -> bool {
|
||||
match e {
|
||||
VolumeError::Io(io_err) => io_err.kind() == io::ErrorKind::UnexpectedEof,
|
||||
VolumeError::Needle(NeedleError::SizeMismatch { .. }) => true,
|
||||
VolumeError::Needle(NeedleError::CrcMismatch { .. }) => true,
|
||||
VolumeError::Needle(NeedleError::IndexOutOfRange(_)) => true,
|
||||
VolumeError::Needle(NeedleError::TailTooShort) => true,
|
||||
VolumeError::SizeMismatch => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// VolumeInfo (.vif persistence)
|
||||
// ============================================================================
|
||||
@@ -732,28 +712,6 @@ impl Volume {
|
||||
"volumeDataIntegrityChecking failed"
|
||||
);
|
||||
}
|
||||
|
||||
// Structural check: no .idx entry may reference bytes past the
|
||||
// end of .dat. The needle map's load walk above already
|
||||
// populated max_needle_end, so this is a numeric comparison
|
||||
// — no extra disk I/O. A violation marks the volume read-only
|
||||
// so vacuum doesn't silently drop reachable data based on a
|
||||
// corrupt .idx left over from a crashed batched write.
|
||||
// See issue #8928.
|
||||
if let Some(ref nm) = self.nm {
|
||||
if let Ok(dat_size) = self.current_dat_file_size() {
|
||||
let max_end = nm.max_needle_end();
|
||||
if dat_size > 0 && max_end > dat_size as i64 {
|
||||
self.no_write_or_delete = true;
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
max_needle_end = max_end,
|
||||
dat_size,
|
||||
"idx references bytes past end of .dat; marking volume read-only"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -773,17 +731,10 @@ impl Volume {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn nm_or_not_found(&self) -> Result<&NeedleMap, VolumeError> {
|
||||
self.nm.as_ref().ok_or_else(|| {
|
||||
tracing::warn!(volume_id = self.id.0, "needle map not loaded");
|
||||
VolumeError::NotFound
|
||||
})
|
||||
}
|
||||
|
||||
fn load_index(&mut self) -> Result<(), VolumeError> {
|
||||
let use_redb = matches!(
|
||||
self.needle_map_kind,
|
||||
NeedleMapKind::Redb | NeedleMapKind::RedbMedium | NeedleMapKind::RedbLarge
|
||||
NeedleMapKind::LevelDb | NeedleMapKind::LevelDbMedium | NeedleMapKind::LevelDbLarge
|
||||
);
|
||||
|
||||
let idx_path = self.file_name(".idx");
|
||||
@@ -808,7 +759,7 @@ impl Volume {
|
||||
// Open read-only
|
||||
if Path::new(&idx_path).exists() {
|
||||
let mut idx_file = File::open(&idx_path)?;
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut idx_file, self.version())?;
|
||||
let nm = CompactNeedleMap::load_from_idx(&mut idx_file)?;
|
||||
self.nm = Some(NeedleMap::InMemory(nm));
|
||||
} else {
|
||||
// Missing .idx with existing .dat could orphan needles
|
||||
@@ -834,7 +785,7 @@ impl Volume {
|
||||
|
||||
let idx_size = idx_file.metadata()?.len();
|
||||
let mut idx_reader = io::BufReader::new(&idx_file);
|
||||
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader, self.version())?;
|
||||
let mut nm = CompactNeedleMap::load_from_idx(&mut idx_reader)?;
|
||||
|
||||
// Re-open for append-only writes
|
||||
let write_file = OpenOptions::new()
|
||||
@@ -857,7 +808,7 @@ impl Volume {
|
||||
// Open read-only
|
||||
if Path::new(&idx_path).exists() {
|
||||
let mut idx_file = File::open(&idx_path)?;
|
||||
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file, self.version())?;
|
||||
let nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_file)?;
|
||||
self.nm = Some(NeedleMap::Redb(nm));
|
||||
} else {
|
||||
// Missing .idx with existing .dat could orphan needles
|
||||
@@ -883,7 +834,7 @@ impl Volume {
|
||||
|
||||
let idx_size = idx_file.metadata()?.len();
|
||||
let mut idx_reader = io::BufReader::new(&idx_file);
|
||||
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader, self.version())?;
|
||||
let mut nm = RedbNeedleMap::load_from_idx(&rdb_path, &mut idx_reader)?;
|
||||
|
||||
// Re-open for append-only writes
|
||||
let write_file = OpenOptions::new()
|
||||
@@ -1065,7 +1016,7 @@ impl Volume {
|
||||
read_option: &mut ReadOption,
|
||||
) -> Result<i32, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
|
||||
|
||||
if nv.offset.is_zero() {
|
||||
@@ -1307,7 +1258,7 @@ impl Volume {
|
||||
read_deleted: bool,
|
||||
) -> Result<NeedleStreamInfo, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
|
||||
|
||||
if nv.offset.is_zero() {
|
||||
@@ -1409,7 +1360,7 @@ impl Volume {
|
||||
&self,
|
||||
needle_id: NeedleId,
|
||||
) -> Result<(u64, u16), VolumeError> {
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(needle_id).ok_or(VolumeError::NotFound)?;
|
||||
if nv.offset.is_zero() {
|
||||
return Err(VolumeError::NotFound);
|
||||
@@ -1726,7 +1677,7 @@ impl Volume {
|
||||
/// Read all live needles from the volume (for ReadAllNeedles streaming RPC).
|
||||
pub fn read_all_needles(&self) -> Result<Vec<Needle>, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let version = self.version();
|
||||
let dat_size = self.current_dat_file_size()? as i64;
|
||||
let mut needles = Vec::new();
|
||||
@@ -1801,11 +1752,6 @@ impl Volume {
|
||||
|
||||
let version = self.version();
|
||||
|
||||
// The deeper-than-tail structural check (every (offset + actual size)
|
||||
// fits inside .dat — issue #8928) is now handled in load() via the
|
||||
// needle map's max_needle_end accumulator, so we don't pay for a
|
||||
// second linear scan of the .idx here.
|
||||
|
||||
// Check last 10 index entries (matching Go's CheckVolumeDataIntegrity).
|
||||
// Go starts healthyIndexSize = indexSize and reduces on EOF.
|
||||
// On success: break (err != ErrorSizeMismatch when err == nil).
|
||||
@@ -1915,7 +1861,7 @@ impl Volume {
|
||||
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let dat_size = self.dat_file_size().map_err(VolumeError::Io)?;
|
||||
|
||||
let mut files_checked: u64 = 0;
|
||||
@@ -1975,7 +1921,7 @@ impl Volume {
|
||||
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
|
||||
let dat_size = self.dat_file_size().map_err(|e| VolumeError::Io(e))?;
|
||||
let version = self.version();
|
||||
@@ -2122,34 +2068,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
/// Mark this volume as writable (allow writes and deletes).
|
||||
///
|
||||
/// If the volume booted with .vif ReadOnly=true, `load_index` built the
|
||||
/// needle map without an .idx writer attached, so subsequent puts would
|
||||
/// silently skip the on-disk append and only mutate in-memory state —
|
||||
/// surviving until the next restart, then vanishing. Re-attach a writer
|
||||
/// here so writes persist again.
|
||||
pub fn set_writable(&mut self) -> Result<(), VolumeError> {
|
||||
// Attach the writer (if missing) before flipping the flag — otherwise
|
||||
// a transient open/metadata failure would leave the volume marked
|
||||
// writable with no .idx writer, and subsequent puts would silently
|
||||
// skip the on-disk append and vanish on the next restart.
|
||||
let needs_idx_writer = self
|
||||
.nm
|
||||
.as_ref()
|
||||
.map(|nm| !nm.has_idx_writer())
|
||||
.unwrap_or(false);
|
||||
if needs_idx_writer {
|
||||
let idx_path = self.file_name(".idx");
|
||||
let write_file = OpenOptions::new()
|
||||
.write(true)
|
||||
.append(true)
|
||||
.create(true)
|
||||
.open(&idx_path)?;
|
||||
let idx_size = write_file.metadata()?.len();
|
||||
if let Some(ref mut nm) = self.nm {
|
||||
nm.set_idx_file(Box::new(write_file), idx_size);
|
||||
}
|
||||
}
|
||||
self.no_write_or_delete = false;
|
||||
self.save_vif()
|
||||
}
|
||||
@@ -2764,8 +2683,6 @@ impl Volume {
|
||||
}
|
||||
entries.sort_by_key(|(_, offset, _)| *offset);
|
||||
|
||||
let mut skipped_needles: u64 = 0;
|
||||
let mut skipped_data_bytes: u64 = 0;
|
||||
for (id, offset, size) in entries {
|
||||
// Progress callback
|
||||
if !progress_fn(offset.to_actual_offset()) {
|
||||
@@ -2782,41 +2699,7 @@ impl Volume {
|
||||
id,
|
||||
..Needle::default()
|
||||
};
|
||||
match self.read_needle_data_at(&mut n, offset.to_actual_offset(), size) {
|
||||
Ok(()) => {}
|
||||
Err(e) => {
|
||||
// Record EIO for health monitoring (parity with Go's checkReadWriteError).
|
||||
if let VolumeError::Io(ref io_err) = e {
|
||||
self.check_read_write_error(Some(io_err));
|
||||
}
|
||||
// Only drop the entry when the failure is one of the well-
|
||||
// known permanent-corruption shapes. A transient disk fault,
|
||||
// a tiered-read timeout, or a Windows hardware error (which
|
||||
// surfaces as a generic Io rather than UnexpectedEof) must
|
||||
// abort so an operator notices, rather than silently
|
||||
// compacting away data that might come back on retry.
|
||||
// See issue #8928.
|
||||
if !is_skippable_needle_read_error(&e) {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("cannot hydrate needle from file: {}", e),
|
||||
)));
|
||||
}
|
||||
skipped_needles += 1;
|
||||
if size.is_valid() {
|
||||
skipped_data_bytes += size.0 as u64;
|
||||
}
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
key = id.0,
|
||||
offset = offset.to_actual_offset(),
|
||||
size = size.0,
|
||||
error = %e,
|
||||
"vacuum: dropping unreadable needle"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
self.read_needle_data_at(&mut n, offset.to_actual_offset(), size)?;
|
||||
|
||||
// Skip TTL-expired needles using the volume's TTL (matches Go's volume_vacuum.go)
|
||||
if n.has_ttl() {
|
||||
@@ -2838,15 +2721,6 @@ impl Volume {
|
||||
new_offset += bytes.len() as i64;
|
||||
}
|
||||
|
||||
if skipped_needles > 0 {
|
||||
warn!(
|
||||
volume_id = self.id.0,
|
||||
skipped_needles,
|
||||
skipped_data_bytes,
|
||||
"vacuum: dropped unreadable index entries during compaction"
|
||||
);
|
||||
}
|
||||
|
||||
dst.sync_all()?;
|
||||
|
||||
// Save new index
|
||||
@@ -3115,10 +2989,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
/// Remove all volume files from disk.
|
||||
/// Destroy removes everything related to this volume. When keep_remote_data
|
||||
/// is true the cloud-tier object backing the volume is left intact — used
|
||||
/// by moves where another server is taking over the same .vif.
|
||||
pub fn destroy(&mut self, only_empty: bool, keep_remote_data: bool) -> Result<(), VolumeError> {
|
||||
pub fn destroy(&mut self, only_empty: bool) -> Result<(), VolumeError> {
|
||||
if only_empty && self.file_count() > 0 {
|
||||
return Err(VolumeError::NotEmpty);
|
||||
}
|
||||
@@ -3130,11 +3001,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
let (storage_name, storage_key) = self.remote_storage_name_key();
|
||||
if !keep_remote_data
|
||||
&& self.has_remote_file
|
||||
&& !storage_name.is_empty()
|
||||
&& !storage_key.is_empty()
|
||||
{
|
||||
if self.has_remote_file && !storage_name.is_empty() && !storage_key.is_empty() {
|
||||
let backend = crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.read()
|
||||
.unwrap()
|
||||
@@ -3230,17 +3097,11 @@ fn get_append_at_ns(last: u64) -> u64 {
|
||||
}
|
||||
|
||||
/// Remove all files associated with a volume.
|
||||
/// .dat/.idx removals log at info level so destructive calls are traceable.
|
||||
pub(crate) fn remove_volume_files(base: &str) {
|
||||
for ext in &[
|
||||
".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
|
||||
] {
|
||||
let path = format!("{}{}", base, ext);
|
||||
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
|
||||
let existed = fs::remove_file(&path).is_ok();
|
||||
if existed && (*ext == ".dat" || *ext == ".idx") {
|
||||
tracing::info!("removed volume file {} (size={})", path, size);
|
||||
}
|
||||
let _ = fs::remove_file(format!("{}{}", base, ext));
|
||||
}
|
||||
// leveldb uses a directory
|
||||
let _ = fs::remove_dir_all(format!("{}.ldb", base));
|
||||
@@ -3364,7 +3225,6 @@ mod tests {
|
||||
let addr = listener.local_addr().unwrap();
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let (ready_tx, ready_rx) = std::sync::mpsc::channel::<()>();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
@@ -3419,7 +3279,6 @@ mod tests {
|
||||
}));
|
||||
|
||||
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||
let _ = ready_tx.send(());
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
@@ -3429,8 +3288,6 @@ mod tests {
|
||||
});
|
||||
});
|
||||
|
||||
// Wait for the server thread to be ready before returning.
|
||||
ready_rx.recv().unwrap();
|
||||
(format!("http://{}", addr), shutdown_tx)
|
||||
}
|
||||
|
||||
@@ -3592,118 +3449,6 @@ mod tests {
|
||||
assert!(matches!(err, VolumeError::Deleted));
|
||||
}
|
||||
|
||||
// Guard the Rust integrity-check tombstone path against the Go regression
|
||||
// where verifyDeletedNeedleIntegrity forwarded TombstoneFileSize into the
|
||||
// needle-size check, mismatched against the on-disk Size=0 header, and
|
||||
// sent every volume with a trailing deletion read-only on load. The Rust
|
||||
// check guards its size comparison with !size.is_deleted(); this test
|
||||
// keeps that guarantee from silently regressing.
|
||||
#[test]
|
||||
fn test_check_volume_data_integrity_with_deletion_tombstone() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
{
|
||||
let mut v = make_test_volume(dir);
|
||||
for i in 1..=3 {
|
||||
let data = format!("data {}", i);
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: data.as_bytes().to_vec(),
|
||||
data_size: data.len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
..Needle::default()
|
||||
})
|
||||
.unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
}
|
||||
|
||||
let v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
"",
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
Version::current(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!v.is_no_write_or_delete(),
|
||||
"volume should not be read-only after reload with trailing deletion tombstone"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scrub_empty_volume() {
|
||||
// Mirror of Go's TestScrubVolumeData "zero-size volume without index"
|
||||
// case (weed/storage/volume_checking_test.go): a freshly created /
|
||||
// pre-allocated volume has a superblock-only .dat and a zero-size .idx,
|
||||
// and must scrub clean instead of being flagged as corrupt.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = make_test_volume(dir);
|
||||
|
||||
// .dat holds only the superblock; .idx is empty.
|
||||
assert_eq!(v.dat_file_size().unwrap(), SUPER_BLOCK_SIZE as u64);
|
||||
|
||||
let (files_checked, broken) = v.scrub().unwrap();
|
||||
assert_eq!(files_checked, 0);
|
||||
assert!(
|
||||
broken.is_empty(),
|
||||
"empty volume should scrub clean, got {:?}",
|
||||
broken
|
||||
);
|
||||
|
||||
// The index-only mode must agree.
|
||||
let (idx_checked, idx_broken) = v.scrub_index().unwrap();
|
||||
assert_eq!(idx_checked, 0);
|
||||
assert!(
|
||||
idx_broken.is_empty(),
|
||||
"empty volume should scrub_index clean, got {:?}",
|
||||
idx_broken
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scrub_healthy_volume() {
|
||||
// Mirror of Go's TestScrubVolumeData "healthy volume" case: a volume
|
||||
// with live needles scrubs clean and the .dat size accounting matches.
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
for i in 1..=5 {
|
||||
let data = format!("needle data {}", i);
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: data.as_bytes().to_vec(),
|
||||
data_size: data.len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
let (files_checked, broken) = v.scrub().unwrap();
|
||||
assert_eq!(files_checked, 5);
|
||||
assert!(
|
||||
broken.is_empty(),
|
||||
"healthy volume should scrub clean, got {:?}",
|
||||
broken
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_volume_multiple_needles() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -3818,7 +3563,7 @@ mod tests {
|
||||
dat_path = v.file_name(".dat");
|
||||
idx_path = v.file_name(".idx");
|
||||
assert!(Path::new(&dat_path).exists());
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
}
|
||||
|
||||
assert!(!Path::new(&dat_path).exists());
|
||||
@@ -3961,138 +3706,6 @@ mod tests {
|
||||
v.cleanup_compact().unwrap();
|
||||
}
|
||||
|
||||
/// Vacuum compaction must tolerate an .idx entry whose offset points past
|
||||
/// the end of the .dat file (the failure mode in issue #8928). The bad
|
||||
/// entry is silently dropped from the resulting .cpx; healthy needles
|
||||
/// survive untouched.
|
||||
#[test]
|
||||
fn test_compact_by_index_drops_dangling_needle() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
// Write a handful of healthy needles to establish a baseline .dat/.idx.
|
||||
for i in 1..=5u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: format!("payload-{}", i).into_bytes(),
|
||||
data_size: format!("payload-{}", i).len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
let dat_size = v.dat_file_size().unwrap();
|
||||
let bad_key = NeedleId(9999);
|
||||
let bad_offset = Offset::from_actual_offset((dat_size + 1024 * 1024) as i64);
|
||||
let bad_size = Size(2048);
|
||||
v.nm
|
||||
.as_mut()
|
||||
.expect("needle map present")
|
||||
.put(bad_key, bad_offset, bad_size)
|
||||
.unwrap();
|
||||
v.nm.as_ref().unwrap().sync().unwrap();
|
||||
|
||||
// Vacuum must succeed in spite of the dangling entry.
|
||||
v.compact_by_index(0, 0, |_| true)
|
||||
.expect("compact_by_index should tolerate dangling entries");
|
||||
|
||||
// Walk the resulting .cpx and confirm the bad key was dropped while
|
||||
// every healthy key made it through.
|
||||
let cpx_path = v.file_name(".cpx");
|
||||
let mut cpx = File::open(&cpx_path).unwrap();
|
||||
let mut kept: Vec<u64> = Vec::new();
|
||||
idx::walk_index_file(&mut cpx, 0, |key, _, size| {
|
||||
if !size.is_deleted() {
|
||||
kept.push(key.0);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
!kept.contains(&bad_key.0),
|
||||
"dangling key {} should have been dropped from .cpx, got {:?}",
|
||||
bad_key.0,
|
||||
kept
|
||||
);
|
||||
for i in 1..=5u64 {
|
||||
assert!(
|
||||
kept.contains(&i),
|
||||
"healthy key {} missing from compacted .cpx, got {:?}",
|
||||
i,
|
||||
kept
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The needle map's max_needle_end accumulator must let volume.load
|
||||
/// detect an .idx whose entries point past the end of the .dat — the
|
||||
/// deeper-than-tail corruption shape from issue #8928 that the existing
|
||||
/// last-10-entries scan cannot see. The check is populated by the load
|
||||
/// walk and read in volume.load() to flip the volume read-only.
|
||||
#[test]
|
||||
fn test_max_needle_end_detects_dangling_entry() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
for i in 1..=4u64 {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: format!("data-{}", i).into_bytes(),
|
||||
data_size: format!("data-{}", i).len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
let dat_size = v.dat_file_size().unwrap() as i64;
|
||||
let idx_path = v.file_name(".idx");
|
||||
let version = v.version();
|
||||
|
||||
// Sanity: a fresh load walk over the healthy .idx puts max_needle_end
|
||||
// somewhere inside the .dat.
|
||||
let mut idx_reader = File::open(&idx_path).unwrap();
|
||||
let healthy_nm =
|
||||
CompactNeedleMap::load_from_idx(&mut idx_reader, version).unwrap();
|
||||
let healthy_end = healthy_nm.max_needle_end();
|
||||
assert!(
|
||||
healthy_end > 0 && healthy_end <= dat_size,
|
||||
"healthy volume should have max_needle_end ({}) in [1, dat_size={}]",
|
||||
healthy_end,
|
||||
dat_size
|
||||
);
|
||||
|
||||
// Inject a dangling entry by appending a bogus 16/17-byte record
|
||||
// directly to the .idx, then reload. The load walk should observe
|
||||
// max_needle_end past dat_size — which is exactly the signal
|
||||
// volume.load uses to mark the volume read-only.
|
||||
let bad_offset = Offset::from_actual_offset(dat_size + 4 * 1024 * 1024);
|
||||
let mut idx_append = OpenOptions::new()
|
||||
.write(true)
|
||||
.append(true)
|
||||
.open(&idx_path)
|
||||
.unwrap();
|
||||
idx::write_index_entry(&mut idx_append, NeedleId(9999), bad_offset, Size(1024))
|
||||
.unwrap();
|
||||
idx_append.sync_all().unwrap();
|
||||
|
||||
let mut idx_reread = File::open(&idx_path).unwrap();
|
||||
let bad_nm = CompactNeedleMap::load_from_idx(&mut idx_reread, version).unwrap();
|
||||
let bad_end = bad_nm.max_needle_end();
|
||||
assert!(
|
||||
bad_end > dat_size,
|
||||
"after dangling-entry inject max_needle_end ({}) should exceed dat_size ({})",
|
||||
bad_end,
|
||||
dat_size
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compaction_revision_relookup() {
|
||||
// Verifies that re_lookup_needle_data_offset returns the correct data offset
|
||||
@@ -4220,7 +3833,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "vif_rw_test".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: v.dat_file_size().unwrap(),
|
||||
@@ -4236,27 +3849,6 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Register a dummy S3 backend so load_remote_dat_file can look it
|
||||
// up. This test never reads remote data, so a dead endpoint is fine.
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry.
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
bucket: "bucket-a".to_string(),
|
||||
endpoint: "http://127.0.0.1:1".to_string(),
|
||||
storage_class: "STANDARD".to_string(),
|
||||
force_path_style: true,
|
||||
};
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.register(
|
||||
"s3.vif_rw_test".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
|
||||
v.dat_file_size().unwrap()
|
||||
};
|
||||
|
||||
@@ -4300,11 +3892,6 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(deleted_size.0 > 0);
|
||||
assert_eq!(v.dat_file_size().unwrap(), dat_size_before_reload);
|
||||
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.vif_rw_test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4330,91 +3917,6 @@ mod tests {
|
||||
assert!(v.no_write_can_delete);
|
||||
}
|
||||
|
||||
// A volume booted with .vif ReadOnly=true used to come back stuck —
|
||||
// load_index_inmemory built the CompactNeedleMap without an .idx writer
|
||||
// attached, and set_writable only flipped the flag and rewrote .vif.
|
||||
// The next put silently skipped the .idx append, so the write landed in
|
||||
// memory only and was lost on the next restart.
|
||||
#[test]
|
||||
fn test_set_writable_reattaches_idx_writer_after_persisted_readonly() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
{
|
||||
let mut v = make_test_volume(dir);
|
||||
let mut n = Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(1),
|
||||
data: b"initial".to_vec(),
|
||||
data_size: 7,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
v.set_read_only_persist(true).unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
}
|
||||
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
"",
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
Version::current(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
v.no_write_or_delete,
|
||||
"reloaded volume should be read-only from .vif"
|
||||
);
|
||||
assert!(
|
||||
!v.nm.as_ref().unwrap().has_idx_writer(),
|
||||
"read-only load should not attach an .idx writer"
|
||||
);
|
||||
|
||||
v.set_writable().unwrap();
|
||||
assert!(!v.is_read_only());
|
||||
assert!(
|
||||
v.nm.as_ref().unwrap().has_idx_writer(),
|
||||
"set_writable must reattach the .idx writer or post-restart writes vanish"
|
||||
);
|
||||
|
||||
let mut n = Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: Cookie(2),
|
||||
data: b"after-mark-writable".to_vec(),
|
||||
data_size: 19,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
v.sync_to_disk().unwrap();
|
||||
|
||||
// Reload one more time — the .idx must contain the post-mark-writable
|
||||
// entry, not just have it in memory.
|
||||
drop(v);
|
||||
let v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
"",
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
Version::current(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut probe = Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle(&mut probe).unwrap();
|
||||
assert_eq!(std::str::from_utf8(&probe.data).unwrap(), "after-mark-writable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_vif_defaults_local_version_and_bytes_offset() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -4552,9 +4054,10 @@ mod tests {
|
||||
std::fs::remove_file(&dat_path).unwrap();
|
||||
|
||||
let (endpoint, shutdown_tx) = spawn_fake_s3_server(dat_bytes.clone());
|
||||
// Use a test-specific backend_id to avoid racing with other tests
|
||||
// that share the global registry. Never call clear() — only
|
||||
// register/remove our own entries.
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.clear();
|
||||
let tier_config = crate::remote_storage::s3_tier::S3TierConfig {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
@@ -4569,7 +4072,11 @@ mod tests {
|
||||
.write()
|
||||
.unwrap();
|
||||
registry.register(
|
||||
"s3.remote_only_rw".to_string(),
|
||||
"s3.default".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
registry.register(
|
||||
"s3".to_string(),
|
||||
crate::remote_storage::s3_tier::S3TierBackend::new(&tier_config),
|
||||
);
|
||||
}
|
||||
@@ -4577,7 +4084,7 @@ mod tests {
|
||||
let vif = VifVolumeInfo {
|
||||
files: vec![VifRemoteFile {
|
||||
backend_type: "s3".to_string(),
|
||||
backend_id: "remote_only_rw".to_string(),
|
||||
backend_id: "default".to_string(),
|
||||
key: "remote-key".to_string(),
|
||||
offset: 0,
|
||||
file_size: dat_bytes.len() as u64,
|
||||
@@ -4634,10 +4141,6 @@ mod tests {
|
||||
assert_eq!(meta.data_size, 11);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.write()
|
||||
.unwrap()
|
||||
.remove("s3.remote_only_rw");
|
||||
}
|
||||
|
||||
/// Volume destroy removes .vif alongside the primary data files.
|
||||
@@ -4668,7 +4171,7 @@ mod tests {
|
||||
assert!(std::path::Path::new(&idx_path).exists());
|
||||
|
||||
// Destroy the volume
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
|
||||
// .dat and .idx should be gone
|
||||
assert!(
|
||||
@@ -4725,7 +4228,7 @@ mod tests {
|
||||
assert!(std::path::Path::new(&dat_path).exists());
|
||||
assert!(std::path::Path::new(&idx_path).exists());
|
||||
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
|
||||
assert!(
|
||||
!std::path::Path::new(&dat_path).exists(),
|
||||
|
||||
@@ -101,8 +101,6 @@ fn test_state_with_guard(
|
||||
read_mode: seaweed_volume::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
|
||||
@@ -33,11 +33,11 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/image v0.36.0 // indirect
|
||||
golang.org/x/net v0.50.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
||||
@@ -27,8 +27,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -84,35 +84,35 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
|
||||
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
|
||||
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
|
||||
golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
|
||||
+2
-2
@@ -999,9 +999,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.4"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
# V3 Phase 14 S8 Assignment
|
||||
|
||||
Date: 2026-04-19
|
||||
Status: draft
|
||||
Owner: sw, with architect and tester review
|
||||
Purpose: execute the final bounded P14 close by producing evidence, scenario classification, and a clean P15 handoff
|
||||
|
||||
## 1. One-Line Goal
|
||||
|
||||
Close P14 as a bounded internal single-active-master topology/control-plane loop, backed by L0/L1/L2 evidence and honest L3/P15 carry-forward.
|
||||
|
||||
## 2. Scope
|
||||
|
||||
Included:
|
||||
|
||||
1. S4-S7 route evidence consolidation
|
||||
2. acceptance matrix for all P14 internal claims
|
||||
3. final supported topology statement
|
||||
4. unsupported/deferred list
|
||||
5. V2 testrunner/scenario port classification
|
||||
6. process smoke gaps review
|
||||
7. 14A final targeted review packet
|
||||
8. P15 handoff mapping
|
||||
|
||||
Excluded:
|
||||
|
||||
1. CSI implementation
|
||||
2. external API implementation
|
||||
3. user data-path production readiness
|
||||
4. migration implementation
|
||||
5. security implementation
|
||||
6. multi-master / HA
|
||||
7. performance or soak claims
|
||||
|
||||
## 3. Required Work
|
||||
|
||||
### A. Evidence Inventory
|
||||
|
||||
Create an evidence table that maps S4-S7 claims to concrete tests and commands.
|
||||
|
||||
Must include:
|
||||
|
||||
1. observation institution
|
||||
2. durable authority institution
|
||||
3. convergence institution
|
||||
4. real-route restart closure
|
||||
5. placement/rebalance/failover policy for accepted topology
|
||||
6. unsupported topology evidence
|
||||
7. old-slot/stale observation rejection
|
||||
8. per-volume isolation
|
||||
|
||||
### B. Gap Classification
|
||||
|
||||
Classify every missing proof as one of:
|
||||
|
||||
1. P14 internal blocker
|
||||
2. P14 S8 test/harness gap
|
||||
3. P15 product-surface gap
|
||||
4. 14A verification follow-up
|
||||
5. explicit non-goal
|
||||
|
||||
Any P14 internal blocker must be fixed before S8 close. A P15 product-surface gap must be carried forward, not disguised as P14 closure.
|
||||
|
||||
### C. V2 Scenario Port Classification
|
||||
|
||||
Inspect V2 scenario/test assets and produce a table:
|
||||
|
||||
| V2 asset | P14 use | P15 use | Port shape | Blocker |
|
||||
|---|---|---|---|---|
|
||||
|
||||
Start with:
|
||||
|
||||
1. `weed/storage/blockvol/testrunner/`
|
||||
2. `weed/storage/blockvol/testrunner/scenarios/public/ha-restart-recovery.yaml`
|
||||
3. `weed/storage/blockvol/testrunner/scenarios/public/ha-failover.yaml`
|
||||
4. `weed/storage/blockvol/testrunner/scenarios/public/fault-partition.yaml`
|
||||
5. `weed/storage/blockvol/test/component/`
|
||||
6. `weed/server/qa_block_*`
|
||||
7. `learn/test` evidence layout
|
||||
|
||||
### D. L2 Process Smoke Decision
|
||||
|
||||
Answer whether current `sparrow` hidden smoke surfaces are sufficient for S8 L2 internal close.
|
||||
|
||||
If not sufficient, add the minimum internal S8 smoke surface needed to prove the S4-S7 route without inventing P15 external product APIs.
|
||||
|
||||
The smoke must check:
|
||||
|
||||
1. real process starts
|
||||
2. durable authority reloads
|
||||
3. observation or fixture input drives controller
|
||||
4. assignment reaches adapter through `VolumeBridge`
|
||||
5. restart does not mint backward
|
||||
6. structured output can be consumed by tester
|
||||
|
||||
### E. 14A Final Targeted Pass
|
||||
|
||||
Prepare a 14A checklist for:
|
||||
|
||||
1. stale authority after restart
|
||||
2. stale observation after reassignment
|
||||
3. old-slot delivery after authority move
|
||||
4. unsupported topology no-op/evidence
|
||||
5. convergence stuck/supersede clearing
|
||||
6. publication honesty during transition
|
||||
7. per-volume isolation
|
||||
|
||||
14A should review findings first. It should not expand P14 scope.
|
||||
|
||||
### F. P15 Handoff
|
||||
|
||||
Produce a P15 handoff table:
|
||||
|
||||
| Gap | Why not P14 | P15 owner track | Required first proof |
|
||||
|---|---|---|---|
|
||||
|
||||
Must include:
|
||||
|
||||
1. CSI lifecycle
|
||||
2. frontend data path
|
||||
3. external volume API
|
||||
4. security/auth
|
||||
5. diagnostics/operator workflow
|
||||
6. migration/coexistence
|
||||
7. deployment/hardening
|
||||
8. final cluster validation agent
|
||||
|
||||
## 4. Required Commands
|
||||
|
||||
At minimum run or document blockers for:
|
||||
|
||||
```text
|
||||
go test ./core/engine ./core/adapter ./core/authority -count=1
|
||||
go test ./cmd/sparrow ./core/authority -count=1
|
||||
go test ./... -count=1
|
||||
```
|
||||
|
||||
If sandbox or Windows filesystem behavior blocks durable-store tests, rerun outside the sandbox and record the reason.
|
||||
|
||||
If L3 hardware scenarios are not runnable, do not fake them. Record the scenario names and blockers.
|
||||
|
||||
## 5. Expected Files
|
||||
|
||||
Expected docs:
|
||||
|
||||
1. `sw-block/design/v3-phase-14-s8-final-bounded-close.md` — this scope document
|
||||
2. `sw-block/design/v3-phase-14-s8-assignment.md` — this assignment
|
||||
3. `sw-block/design/v3-phase-14-s8-closure.md` — final closure statement, created after implementation/evidence
|
||||
4. updates to `v3-phase-14-checklist.md`
|
||||
5. updates to `v3-phase-14-log.md`
|
||||
6. updates to `v3-phase-14a-checklist.md` if 14A final pass finds coverage gaps
|
||||
|
||||
Expected code/tests only if needed:
|
||||
|
||||
1. internal process smoke helper in `cmd/sparrow`
|
||||
2. component evidence consolidation tests in `core/authority` or a shared component package
|
||||
3. scenario classification artifacts under the chosen test/runner directory
|
||||
|
||||
## 6. Acceptance
|
||||
|
||||
S8 can be accepted only when:
|
||||
|
||||
1. the evidence matrix exists and is honest
|
||||
2. all P14 internal blockers are either fixed or explicitly rejected from the P14 claim
|
||||
3. L0 and L1 tests pass
|
||||
4. L2 process smoke is present or its absence is classified as a P15 product-surface blocker
|
||||
5. L3 scenarios are classified with runnable/deferred/blocker status
|
||||
6. 14A final targeted pass has no open P14 safety blocker
|
||||
7. P15 handoff maps every product gap to a P15 track
|
||||
8. final closure statement does not claim production readiness
|
||||
|
||||
## 7. Review Focus
|
||||
|
||||
Architect should review:
|
||||
|
||||
1. whether S8 closure is product-honest
|
||||
2. whether any P15 work was pulled backward into P14
|
||||
3. whether any P14 internal truth gap was pushed forward into P15
|
||||
4. whether test evidence matches the claimed layer
|
||||
5. whether V2 porting stays mechanism-only
|
||||
|
||||
Tester should review:
|
||||
|
||||
1. whether evidence is reproducible
|
||||
2. whether commands and scenario blockers are specific
|
||||
3. whether L2/L3 gaps are visible
|
||||
4. whether final artifacts are enough to start P15 without re-litigating P14
|
||||
@@ -1,114 +0,0 @@
|
||||
# V3 Phase 14 S8 Closure
|
||||
|
||||
Date: 2026-04-20
|
||||
Status: draft — pending 14A final pass + architect + tester sign-off
|
||||
Purpose: the final P14 closure statement per `v3-phase-14-s8-final-bounded-close.md` §10
|
||||
|
||||
## 1. What P14 Now Owns
|
||||
|
||||
After S4-S7, P14 closes **one bounded single-active-master internal topology/control-plane truth loop** for the accepted topology:
|
||||
|
||||
1. **Observation institution (S4)**: heartbeats and inventory flow through `ObservationHost` into a synthesized `ClusterSnapshot` + `SupportabilityReport`. Observation never mints authority. Partial / conflicting / duplicate / unknown / expired inventory becomes explicit `VolumeUnsupportedEvidence`; pending state is distinct from unsupported; per-volume isolation holds.
|
||||
|
||||
2. **Durable authority institution (S5)**: current authority line is durable, single-owner, atomically written, and reloaded synchronously at publisher boot. One record per volume. Corrupt records per-volume-fail-closed and surface as structured `ReloadSkips`. Process lock is exclusive and idempotent. Store never mints authority (boundary-guard test).
|
||||
|
||||
3. **Convergence institution (S6)**: desired assignments have bounded fate — confirmed (cleared), publish-not-observed (stuck-evidence, no re-mint, no churn), or superseded (by newer publisher line or different decision). Passive retry only; no active re-drive. Normal observation lag does NOT supersede. Per-volume retry clock; no global state.
|
||||
|
||||
4. **Real-route restart closure (S7)**: the full `ObservationHost → TopologyController → Publisher(reloaded) → VolumeBridge → VolumeReplicaAdapter` composes correctly at restart. Restart recomputes desired from durable authority + fresh observation (no durable desired store). Old-slot per-replica state is NOT revived from the store (S5 one-record-per-volume rule); live-route old-slot delivery via bridge is rejected by the adapter's monotonic guard. Stale observation cannot move authority backward (4 simulation shapes). Unsupported topology after restart records evidence, not silent idle. L2 process smoke proves real-binary restart × 2 preserves durable truth with no backward mint, via pinned structured JSON.
|
||||
|
||||
5. **Accepted topology** (S8 §4): single active master; multiple volumes; three replica slots per volume on distinct servers; one primary + two candidates; publisher-owned `Epoch` / `EndpointVersion`; durable current line; passive convergence; real `VolumeBridge` into adapter.
|
||||
|
||||
## 2. What P14 Explicitly Does NOT Own
|
||||
|
||||
1. CSI lifecycle (Create/Delete/Publish/Node*).
|
||||
2. iSCSI / NVMe-oF frontend data path.
|
||||
3. External volume management API (REST / gRPC).
|
||||
4. Security / auth (authn, CHAP, encryption).
|
||||
5. Operator diagnostics surface (dashboards, alerts, `/metrics`).
|
||||
6. Operator workflow (drain, planned failover, manual reassign, supervised rebuild).
|
||||
7. V2 ↔ V3 migration and coexistence.
|
||||
8. Deployment / packaging / hardening (systemd, container, backup of durable store).
|
||||
9. Performance / soak / release-hardening claims.
|
||||
10. Multi-master / leader election / distributed authority (explicit non-goal, not just deferred).
|
||||
11. V2 `HandleAssignment` / `promote` / `demote` semantics (explicit non-goal).
|
||||
12. Heartbeat-as-authority (explicit non-goal).
|
||||
|
||||
Every item 1–9 is mapped to a P15 track in `v3-phase-14-s8-p15-handoff.md` §2. Items 10–12 are permanent non-goals.
|
||||
|
||||
## 3. Evidence At L0 / L1 / L2 / L3
|
||||
|
||||
Full matrix: `v3-phase-14-s8-evidence-matrix.md` §3. Summary of coverage:
|
||||
|
||||
| Claim | L0 | L1 | L2 | L3 |
|
||||
|---|---|---|---|---|
|
||||
| 1. Observation system-fed | ✓ | ✓ | — (via L1) | L3 shape only |
|
||||
| 2. Supportability explicit | ✓ | ✓ | — | L3 shape only |
|
||||
| 3. Durable authority / reload | ✓ (13 tests) | ✓ | ✓ (subprocess) | L3 shape only |
|
||||
| 4. No old-slot durable revival | ✓ | ✓ | ✓ | L3 shape only |
|
||||
| 5. Live-route old-slot rejected | ✓ | ✓ | N/A (live-route) | N/A |
|
||||
| 6. Controller-driven bind/reassign/refresh | ✓ (7 tests) | ✓ | ✓ (Bind only — Reassign / RefreshEndpoint are L1-only; subprocess does not ingest heartbeats) | L3 shape only |
|
||||
| 7. Confirmation clears desired | ✓ | ✓ | N/A | L3 shape only |
|
||||
| 8. Stuck evidence bounded / no churn | ✓ (5 tests) | ✓ | N/A | L3 shape only |
|
||||
| 9. Supersede (2 modes) | ✓ (3 tests) | ✓ | N/A | L3 shape only |
|
||||
| 10. Restart re-anchors via VolumeBridge | ✓ | ✓ | **subclaim only** — L2 proves durable `Publisher` reload (Claim 3); the bridge/adapter re-anchor is NOT at L2 because `sparrow` subprocess does not construct `ObservationHost` / controller / bridge / adapter | L3 shape only |
|
||||
| 11. Stale observation cannot go backward | ✓ | ✓ (4 sub-cases) | — | L3 shape only |
|
||||
| 12. Unsupported → evidence | ✓ | ✓ | **not at L2** — `ReloadSkips` is S5 durable-corruption evidence, not observation-layer unsupported-topology. Controller `LastUnsupported` is L1 | L3 shape only |
|
||||
| 13. Per-volume isolation | ✓ (4 tests) | ✓ | N/A | L3 shape only |
|
||||
| 14. Placement / rebalance | ✓ (5 tests) | ✓ | N/A | L3 shape only |
|
||||
| 15. Fence route bounded | ✓ (adapter) | PARTIAL | N/A | L3 shape only |
|
||||
| 16. Boundary guards | ✓ | N/A (structural) | N/A | N/A |
|
||||
|
||||
Sixteen claims; fifteen PROVEN at their target level; one (Claim 15) PARTIAL — adapter-package follow-up per §5 below. L3 is classification-only at P14 (`v3-phase-14-s8-v2-scenario-classification.md`) — L3 entries are scenario SHAPES, not executed evidence. Runnable L3 is P15 Final Gate (Cluster Validation Agent). Claims 6, 10, and 12 have NARROWED L2 cells where the subprocess binary cannot carry the full claim surface; each is handed off to the correct canonical P15 track.
|
||||
|
||||
## 4. Test Baseline (2026-04-20)
|
||||
|
||||
```
|
||||
$ go test ./core/engine ./core/adapter ./core/authority ./cmd/sparrow -count=1
|
||||
ok github.com/seaweedfs/seaweed-block/core/engine 0.016s
|
||||
ok github.com/seaweedfs/seaweed-block/core/adapter 2.344s
|
||||
ok github.com/seaweedfs/seaweed-block/core/authority 4.569s
|
||||
ok github.com/seaweedfs/seaweed-block/cmd/sparrow 2.264s
|
||||
|
||||
$ go test ./... -count=1
|
||||
(14 packages, all PASS)
|
||||
```
|
||||
|
||||
193 Go tests across the P14 scope. L2 subprocess smoke (`TestS7Process_RealSubprocessRestartSmoke`) spawns a real `sparrow` binary twice on the same store directory and asserts the pinned JSON pass-line schema.
|
||||
|
||||
## 5. Residual Risks
|
||||
|
||||
| # | Risk | Scope / carry-forward |
|
||||
|---|---|---|
|
||||
| 1 | Windows `os.RemoveAll` on `t.TempDir()` can race with lock-file release during teardown. No correctness impact. | S7 sketch §8.5 — logged as `t.Logf`, not a failure. |
|
||||
| 2 | Claim 15 PARTIAL: fence quantitative timeout-bound is adapter-package-owned, depends on the uncommitted fence-watchdog branch committing. | `v3-phase-14-s8-p15-handoff.md` item 11 — P14 internal follow-up, NOT P15. Single adapter test replaces PARTIAL status. |
|
||||
| 3 | L2 subprocess drives mint via `StaticDirective`; it does NOT construct `ObservationHost` / controller / bridge / adapter. Full heartbeat-ingress and real-binary adapter route are L1-only in S8. | **P15 T1** Frontend + Data Path (handoff items #1 and #2). |
|
||||
|
||||
**No S8-blocking correctness residuals.** One adapter-owned quantitative fence proof (Claim 15) remains as P14-internal follow-up per row 11 of the handoff table — not a P15 track, not an S8 gate. Zero backward-authority-mint residuals. Zero silent-idle residuals.
|
||||
|
||||
## 6. P15 Track Owners
|
||||
|
||||
Handoff complete per `v3-phase-14-s8-p15-handoff.md`, aligned to canonical `v3-phase-15-product-plan.md` §4. Every P14 residual product gap has a named P15 track and a concrete first-proof gate:
|
||||
|
||||
- T1 Frontend + Data Path Contract (includes subprocess heartbeat ingress)
|
||||
- T2 CSI / External Lifecycle Surface
|
||||
- T3 External Control API
|
||||
- T4 Security And Auth Posture
|
||||
- T5 Diagnostics And Explainability
|
||||
- T6 Operator Workflow
|
||||
- T7 V2/V3 Coexistence And Migration
|
||||
- T8 Deployment / Upgrade / Release Hardening
|
||||
- Final Gate — Cluster Validation Agent
|
||||
|
||||
## 7. Final P14 Claim
|
||||
|
||||
> **P14 has closed one bounded single-active-master internal topology/control-plane truth loop for the accepted topology set.** Observation is system-fed; authority is durable, single-owner, and restart-recovered; convergence has bounded fate; the real `VolumeBridge → adapter` route re-anchors across restart without backward mint or old-slot revival.
|
||||
>
|
||||
> **P14 has NOT closed CSI, external API, frontend data path, migration, security, deployment, performance, or production readiness.** Those nine tracks are handed off to P15 with concrete first-proof gates. Multi-master, V2 `HandleAssignment/promote/demote` semantics, and heartbeat-as-authority are explicit non-goals — not deferrals.
|
||||
|
||||
## 8. Sign-off State
|
||||
|
||||
- **Architect** — pending review of this matrix + handoff.
|
||||
- **Tester** — pending reproducibility check (commands in §4).
|
||||
- **14A final targeted pass** — pending (scope in `v3-phase-14a-checklist.md`).
|
||||
|
||||
S8 cannot be accepted until all three sign off. After that, P14 is closed and the next active work is P15 T1 Frontend.
|
||||
@@ -1,241 +0,0 @@
|
||||
# V3 Phase 14 S8 Evidence Matrix
|
||||
|
||||
Date: 2026-04-20
|
||||
Status: draft (S8 evidence consolidation)
|
||||
Purpose: map every internal P14 claim from `v3-phase-14-s8-final-bounded-close.md` §5 to concrete tests and commands, classify L0/L1/L2/L3 coverage, and surface residual gaps for 14A / P15 handoff
|
||||
|
||||
## 1. How To Read This Matrix
|
||||
|
||||
Each row is one internal P14 claim. Columns:
|
||||
|
||||
- **L0 Unit** — package-local invariant / policy test that proves the claim inside one package.
|
||||
- **L1 Component** — in-process multi-package route test (real `ObservationHost` + `TopologyController` + `Publisher` + `VolumeBridge` + `VolumeReplicaAdapter`, no shell, no kernel).
|
||||
- **L2 Process** — real `sparrow` binary or in-process `Bootstrap()` against a real filesystem store.
|
||||
- **L3 Scenario** — hardware / YAML-driven scenario. For P14 S8, L3 is **classification-only**: the rows below say "L3 shape only, not executed" where a V2 YAML scenario describes the same claim shape. L3 entries are NOT executed evidence; they are the scenario shapes that P15 Cluster Validation (Final Gate) will run. A claim with an L3 shape listed is NOT proven at L3 by S8.
|
||||
- **Status** — `PROVEN` (explicit test); `PARTIAL` (route covered but some sub-claim deferred); `DEFERRED` (intentionally pushed to later slice / P15); `N/A` (not applicable at that level).
|
||||
- **Residual** — what is NOT proven and where it is carried.
|
||||
|
||||
All cited L0/L1/L2 tests live in the committed tree; re-run with:
|
||||
|
||||
```
|
||||
go test ./core/engine ./core/adapter ./core/authority ./cmd/sparrow -count=1
|
||||
```
|
||||
|
||||
Baseline on 2026-04-20: engine 0.02s / adapter 2.3s / authority 4.6s / sparrow 2.3s — all green, 193 Go tests total across the S4-S7 scope.
|
||||
|
||||
## 2. Accepted Topology Claim Recap
|
||||
|
||||
Pinned by the S8 close doc §4. Reproduced here so every row in the matrix below is read against the correct bounded shape:
|
||||
|
||||
1. single active master
|
||||
2. multiple volumes
|
||||
3. three bounded replica slots per volume, distinct servers
|
||||
4. one current authoritative primary, two failover/rebalance candidates
|
||||
5. publisher-owned `Epoch` / `EndpointVersion`
|
||||
6. durable current authority line, one record per volume
|
||||
7. passive convergence (publish → observe → confirm / stuck / supersede)
|
||||
8. real `VolumeBridge` delivery into adapter/engine
|
||||
|
||||
Anything outside this set is P15 or explicit non-goal.
|
||||
|
||||
## 3. Evidence Matrix
|
||||
|
||||
### Claim 1 — Observation is system-fed, not test-authored
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestObservation_BuilderStampsAuthorityFromReader` / `TestObservation_SupportabilityDoesNotReadClusterSnapshotOutput` / `TestObservation_NeverMintsAuthority_BoundaryGuard` | `TestObservation_EndToEnd_SystemFedSnapshotReachesController` / `TestConvergenceRoute_ObservationFedFailover_ConfirmsAndClearsDesired` / `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` | (covered at L1; `sparrow` run-path uses the same `ObservationHost` construction) | L3 shape only, not executed — `ha-restart-recovery.yaml` / `ha-failover.yaml` carry the L3 shape | PROVEN | none at L0/L1 |
|
||||
|
||||
### Claim 2 — Topology supportability is explicit and fail-closed (partial inventory / missing-server / conflict / duplicate / unknown-volume / expired)
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestObservation_PartialInventory_PendingThenUnsupported` / `TestObservation_MissingServerObservation_NeverOrdinaryIneligible` / `TestObservation_ConflictingPrimaryClaim_Unsupported` / `..._NoWinnerEverChosen` / `TestObservation_DuplicateServerTopology_UnsupportedAndEvidenceRecorded` / `TestObservation_UnknownObservedVolume_ProducesUnsupportedEvidence` / `TestObservation_ExpiredFact_SemanticallyIneligible` / `TestObservation_ExpiredDoesNotReachController` / `TestObservation_FreshnessWindowHonored_NotTickMultiples` | `TestObservation_SupportabilityCollapsePropagatesToController` / `TestObservation_SupportabilityCollapseClearsControllerDesired` / `TestS7_Restart_UnsupportedTopologyRecordsEvidence` | N/A (covered by L0/L1) | L3 shape only, not executed — `fault-partition.yaml` is the L3 shape | PROVEN | none |
|
||||
|
||||
### Claim 3 — Durable authority is single-owner, atomic, and restart-recovered
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestFileAuthorityStore_RoundTrip` / `TestFileAuthorityStore_LatestWinsPerVolumeID` / `TestFileAuthorityStore_EnvelopeShape` / `TestDurableAuthority_AtomicWrite_NoTornRecord` / `TestDurableAuthority_CorruptRecord_PerVolumeFailClosed` / `TestDurableAuthority_ProcessLock_ExclusiveOwner` / `TestDurableAuthority_ProcessLock_IdempotentRelease` / `TestDurableAuthority_ProcessLock_SurvivesStaleFileOnDisk` / `TestDurableAuthority_UnreadableIndex_BootFailsClosed` / `TestDurableAuthority_StoreNeverMintsAuthority_BoundaryGuard` / `TestDurableAuthority_StoreWriteCallerAllowlist_BoundaryGuard` / `TestEncodeVolumeIDForFilename_Injective` / `TestEncodeVolumeIDForFilename_NoFilesystemUnsafeOutput` | `TestDurableAuthority_ReloadReproducesCurrentLine` / `TestDurableAuthority_ControllerSeesReloadedLineAtBoot` / `TestDurableAuthority_StoreBackedPublisherRunsNormalRoute` / `TestDurableAuthority_PutFailureRollsBackInMemoryState` / `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` | `TestBootstrap_FreshDir_AcquiresLockAndReloads` / `TestBootstrap_WithExistingRecord_ReloadsPublisherState` / `TestBootstrap_CorruptRecord_LoggedAtStartup` / `TestBootstrap_LockContention_RefusesSecondBootstrap` / `TestS7Process_BootstrapReloadRouteSmoke` / **subprocess**: `TestS7Process_RealSubprocessRestartSmoke` | L3 shape only, not executed — `ha-restart-recovery.yaml` / `diag-restart-recovery.yaml` | PROVEN | Windows `os.RemoveAll` may race with lock release at teardown — logged as residual per S7 sketch §8.5; does NOT affect correctness |
|
||||
|
||||
### Claim 4 — No old-slot durable revival after restart
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestDurableAuthority_StalePreFailoverRecordRejectedOnReload` | `TestS7_Restart_OldSlotNotRevivedFromDurable` (explicitly asserts `LastAuthorityBasis("vr","r1")==false` and `("vr","r2")==Epoch=2` post-restart) | covered by S7 subprocess smoke plus in-process two-run `TestS7Process_RestartDoesNotRegressAuthorityLine` | L3 shape only, not executed | PROVEN | none — row 4a/4b split in S7 closes the original ambiguity |
|
||||
|
||||
### Claim 5 — Live-route monotonic guard rejects stale old-slot delivery (pre-restart)
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| adapter-internal: engine monotonic guards covered in `core/adapter/adapter_test.go` and `core/engine/*_test.go` | `TestS7_LiveRoute_OldSlotDeliveryRejectedByAdapter` (real `VolumeBridge`, in-memory publisher holds both per-replica keys, lower-epoch catch-up rejected) | N/A (live-route, pre-restart) | N/A | PROVEN | none |
|
||||
|
||||
### Claim 6 — Controller-driven bind / reassign / refresh over accepted topology (system-driven, not test-driven)
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestTopologyController_InitialPlacementBalancesAcrossVolumes` / `..._InitialPlacementUsesEvidenceTieBreakOnEqualLoad` / `..._FailoverUsesHighestEvidenceCandidate` / `..._RebalanceMovesToLighterServer` / `..._RebalanceSkipsWhenLoadAlreadyWithinBound` / `..._ObservedAuthorityClearsPending` / `..._StalePreConfirmSnapshotDoesNotDuplicateQueuedMove` | Placement / failover: `TestTopologyControllerToPublisher_E2E_MultiVolumePlacementAndFailover` / `TestObservationHost_IdenticalSupportedSnapshots_NoDuplicateAsks` / `TestObservationHost_VolumeRecovers_ClearsStaleDesired`. Rebalance: `TestTopologyControllerToPublisher_E2E_Rebalance`. **Refresh: `TestConvergenceRoute_RefreshEndpoint_ConfirmsAndClearsDesired`** (host heartbeat changes slot DataAddr/CtrlAddr while publisher holds old addrs → controller emits `IntentRefreshEndpoint` → publisher bumps `EndpointVersion` on same `Epoch` → bridge delivers refreshed `AssignmentInfo` → confirmation clears desired). | `TestS7Process_RealSubprocessRestartSmoke` covers the Bind mint via the real binary (Reassign/Refresh are L1-only in `sparrow` today; real binary does not ingest heartbeats — see §5 L2 scope note). | L3 shape only, not executed | PROVEN | none |
|
||||
|
||||
### Claim 7 — Convergence: confirmation clears desired
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestConvergence_Confirmation_ClearsDesired` / `TestConvergence_Confirmation_RequiresBothSources` / `TestConvergence_StaleObservation_NoConfirmation` / `TestConvergence_StuckThenConfirm_ClearsStuckEvidence` | `TestConvergenceRoute_ObservationFedFailover_ConfirmsAndClearsDesired` | N/A | L3 shape only, not executed | PROVEN | none |
|
||||
|
||||
### Claim 8 — Publish-not-observed stuck evidence is bounded and non-churning
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestConvergence_PublishNotObserved_StuckAfterWindow` / `TestConvergence_Stuck_DoesNotReMintReassign` / `TestConvergence_Reassign_CalledAtMostOncePerDesired` / `TestConvergence_RetryStateIsPerVolume` / `TestConvergence_SupportedRecovery_ClearsPriorUnsupportedEvidence` | `TestConvergenceRoute_PublishNotObserved_StuckWithoutChurn` | N/A | L3 shape only, not executed | PROVEN | none |
|
||||
|
||||
### Claim 9 — Supersede by newer authority or different decision
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestConvergence_Supersede_PublisherAdvancedToOtherReplica` / `TestConvergence_NormalLag_OldObservationDoesNotSupersede` / `TestConvergence_Supersede_DecisionTableProducesDifferentAsk` | `TestConvergenceRoute_Supersede_PublisherMovedElsewhere_DropsStaleDesired` | N/A | L3 shape only, not executed | PROVEN | Case 2 of the supersede rule was intentionally dropped (see S6 sketch §9) — that decision is tested indirectly by `TestConvergence_NormalLag_OldObservationDoesNotSupersede` which would otherwise fail |
|
||||
|
||||
### Claim 10 — Restart re-anchors authority via real `VolumeBridge` into the adapter/engine route
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestDurableAuthority_PublisherAdvancesFromReloaded` | `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` / `TestS7_Restart_RecomputesDesiredNotTransientState` (hard precondition prevents vacuous pass) | **Not at L2.** The `sparrow` subprocess smoke (`TestS7Process_RealSubprocessRestartSmoke`) reloads durable `Publisher` state only; it does NOT construct `ObservationHost`, `TopologyController`, `VolumeBridge`, or `VolumeReplicaAdapter`, so the bridge/adapter re-anchor cannot be counted at L2. L2 proves a strictly narrower subclaim (durable `Publisher` reload — Claim 3). | L3 shape only, not executed — `ha-restart-recovery.yaml` is the L3 shape | PROVEN at L1 (full route); L2 COVERS a subclaim only (durable reload, see Claim 3) | Subprocess heartbeat ingress + real-binary adapter route → **P15 T1** (handoff item #2) |
|
||||
|
||||
### Claim 11 — Stale observation cannot move authority backward
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestConvergence_StaleObservation_NoConfirmation` | `TestS7_Restart_StaleObservationCannotMoveBackward` (4 sub-cases: stale-epoch / stale-replica / stale-endpointVersion / unassigned via `snapshottingReader` host-reader freeze) / `TestConvergenceRoute_PublishNotObserved_StuckWithoutChurn` (proves host-reader lag scenario) | covered at L1 | L3 shape only, not executed — `fault-partition.yaml` | PROVEN | stale-basis simulation is via host-reader freeze; heartbeat wire does not carry authority (documented in S7 sketch §9.1) |
|
||||
|
||||
### Claim 12 — Unsupported topology after restart records evidence, not silent idle
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestConvergence_UnsupportedClearsDesiredWithEvidence` / `TestConvergence_PendingClearsDesired` | `TestS7_Restart_UnsupportedTopologyRecordsEvidence` | **Not at L2.** This claim is about the controller's `LastUnsupported(vid)` surface being populated from conflicting / incomplete observation; that requires `ObservationHost` + controller composition, which the subprocess does not construct. `Bootstrap.ReloadSkips` is a DIFFERENT surface (S5 durable-record corruption, see Claim 3 residual path) and is NOT observation-layer unsupported-topology evidence. | L3 shape only, not executed — `fault-partition.yaml` + `diag-restart-recovery.yaml` are the L3 shapes | PROVEN at L1 | Operator-visible surface over `LastUnsupported` / `LastConvergenceStuck` → **P15 T5** Diagnostics (handoff item #6) |
|
||||
|
||||
### Claim 13 — Per-volume isolation (one volume's failure does not block others)
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestObservation_BadVolumeDoesNotBlockHealthyVolume_Isolation` / `TestObservation_VolumeUnsupportedEvidence_IsolatedPerVolume` / `TestConvergence_OneStuckVolumeDoesNotBlockOthers` / `TestTopologyController_UnsupportedVolumeDoesNotBlockOtherVolumes` | `TestS7_Restart_PerVolumeIsolation` | N/A | L3 shape only, not executed | PROVEN | none |
|
||||
|
||||
### Claim 14 — Placement / rebalance / failover within the accepted topology
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestTopologyController_InitialPlacementBalancesAcrossVolumes` / `..._FailoverUsesHighestEvidenceCandidate` / `..._RebalanceMovesToLighterServer` / `..._RebalanceSkipsWhenLoadAlreadyWithinBound` / `..._OutOfTopologyAuthority_NoAskPlusEvidence` | `TestTopologyControllerToPublisher_E2E_MultiVolumePlacementAndFailover` / `TestTopologyControllerToPublisher_E2E_Rebalance` | N/A | L3 shape only, not executed — `ha-failover.yaml` carries the L3 shape | PROVEN | none |
|
||||
|
||||
### Claim 15 — Fence path routes through bridge + adapter without spurious Healthy
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| adapter fence-slot / monotonic-guard tests in `core/adapter/adapter_test.go` | `TestS7_Restart_FenceRouteBoundedWithoutCallback` (Fence command reaches executor via real bridge; withheld callback does not produce Healthy) | N/A | L3 shape only, not executed | PARTIAL | Full timeout-expiry quantitative bound (watchdog fires at exact deadline + structured event) is adapter-owned proof (sketch §10.1 Path B — S7 may not reconfigure the fence path). Carried to the adapter package's own tests; not an L1 S7 proof |
|
||||
|
||||
### Claim 16 — Boundary guards (no unauthorized `AssignmentInfo` minting, no observation-side asks, no store minting, no hidden adapter ingress)
|
||||
|
||||
| L0 | L1 | L2 | L3 | Status | Residual |
|
||||
|---|---|---|---|---|---|
|
||||
| `TestObservation_NeverMintsAuthority_BoundaryGuard` / `TestDurableAuthority_StoreNeverMintsAuthority_BoundaryGuard` / `TestDurableAuthority_StoreWriteCallerAllowlist_BoundaryGuard` / `TestNonForgeability_*` (AST-level guards in `authority_test.go`) | N/A (structural, not route) | N/A | N/A | PROVEN | none |
|
||||
|
||||
## 4. Gap Classification
|
||||
|
||||
Per S8 assignment §3.B, every missing proof must be classified. Scanning the matrix above:
|
||||
|
||||
### 4.1 P14 internal blockers
|
||||
**None.** Every P14 internal claim has at least one PROVEN row.
|
||||
|
||||
### 4.2 P14 S8 test/harness gap
|
||||
**Claim 15 (fence quantitative bound).** S7 deliberately capped at L1 route coverage. The full `fence timeout fires at exact deadline` proof lives in the adapter package and depends on the uncommitted fence-watchdog branch landing committed. After that commits, a single-file adapter test replaces the current PARTIAL status. Not an S8 gate.
|
||||
|
||||
### 4.3 P15 product-surface gaps (carried forward)
|
||||
See `v3-phase-14-s8-p15-handoff.md` for the full mapping (aligned to canonical `v3-phase-15-product-plan.md` §4 track numbering). Cross-reference summary:
|
||||
|
||||
- Frontend data path + subprocess heartbeat ingress → **P15 T1** Frontend + Data Path
|
||||
- CSI lifecycle → **P15 T2** CSI / External Lifecycle Surface
|
||||
- External control API → **P15 T3** External Control API
|
||||
- Security / auth → **P15 T4** Security And Auth Posture
|
||||
- Operator diagnostics for `LastUnsupported` / `LastConvergenceStuck` / `ReloadSkips` → **P15 T5** Diagnostics
|
||||
- Operator workflow (drain / supervised reassign / rebuild) → **P15 T6** Operator Workflow
|
||||
- V2/V3 migration → **P15 T7** Migration
|
||||
- Deployment / hardening → **P15 T8** Deployment / Release Hardening
|
||||
- End-to-end cluster validation → **P15 Final Gate**
|
||||
|
||||
### 4.4 14A verification follow-ups
|
||||
See `v3-phase-14a-checklist.md` for the sidecar list. The S8 matrix above is the input to 14A's final targeted review.
|
||||
|
||||
### 4.5 Explicit non-goals (do NOT carry)
|
||||
- multi-master / leader election
|
||||
- performance / soak claims at P14 close
|
||||
- long-running stability under production load
|
||||
|
||||
## 5. L2 Process Smoke Decision
|
||||
|
||||
Per S8 assignment §3.D, S8 must answer: are current `sparrow` smoke surfaces sufficient?
|
||||
|
||||
**Answer: scoped YES — L2 proves durable-restart only; the full S4-S7 route remains L1-proven.** (Round-2 architect correction.)
|
||||
|
||||
### 5.1 What L2 proves today
|
||||
|
||||
| Required check | Covered by |
|
||||
|---|---|
|
||||
| real process starts | `TestS7Process_RealSubprocessRestartSmoke` spawns the real `sparrow` binary. |
|
||||
| durable authority reloads | same test + component `TestS7Process_BootstrapReloadRouteSmoke`. |
|
||||
| restart does not mint backward | subprocess asserts `backwardMint: false` in the pinned JSON pass-line; component `TestS7Process_RestartDoesNotRegressAuthorityLine` asserts reloaded `Epoch` equals pre-restart `Epoch`. |
|
||||
| structured output can be consumed by tester | subprocess pass-line schema pinned in S7 sketch §8.4. |
|
||||
|
||||
### 5.2 What L2 deliberately does NOT prove
|
||||
|
||||
| Not covered at L2 | Where it IS proven | Carry-forward |
|
||||
|---|---|---|
|
||||
| "heartbeat wire → `ObservationHost` → `TopologyController`" in a real process. Subprocess drives mints via `StaticDirective`; there is no heartbeat ingress in `sparrow` today. | L1: `TestObservation_EndToEnd_SystemFedSnapshotReachesController`, `TestConvergenceRoute_ObservationFedFailover_ConfirmsAndClearsDesired`, `TestConvergenceRoute_RefreshEndpoint_ConfirmsAndClearsDesired`, `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` (all run real `ObservationHost` → controller → publisher → bridge → adapter, in-process). | **P15 T1 Frontend + Data Path** (subprocess heartbeat ingress is part of the frontend/transport contract). Handoff item #2 in `v3-phase-14-s8-p15-handoff.md`. |
|
||||
| "assignment reaches adapter via bridge IN THE REAL BINARY". The subprocess binary does not construct a VolumeReplicaAdapter — only the in-process component test does. | L1: `TestS7_ReloadedAuthority_ReanchorsAdapterViaVolumeBridge` composes the full stack in-process and waits for adapter `ModeHealthy`. | **P15 T1 Frontend + Data Path** — the first real-binary adapter ↔ frontend route is a T1 deliverable. |
|
||||
| Multi-process mixed-route scenarios (separate authority + observing processes). | N/A at P14 | **P15 Final Gate** (Cluster Validation). |
|
||||
|
||||
### 5.3 Decision
|
||||
|
||||
**No new S8 smoke is added.** The scope split above is honest:
|
||||
|
||||
- L2 subprocess smoke correctly proves durable restart truth.
|
||||
- The full S4-S7 composed route is proven at L1 — including the `RefreshEndpoint` path added in this pass.
|
||||
- Subprocess heartbeat ingress and full real-binary adapter route are genuinely P15 T1 surfaces; inventing a hidden `--s8-route-smoke` heartbeat fixture would be a P15 surface in P14 clothing.
|
||||
|
||||
This is a downgrade from the round-1 matrix claim ("L2 covers the S4-S7 internal route"), which was an overclaim.
|
||||
|
||||
## 6. Baseline Command Result (2026-04-20)
|
||||
|
||||
```
|
||||
$ go test ./core/engine ./core/adapter ./core/authority ./cmd/sparrow -count=1
|
||||
ok github.com/seaweedfs/seaweed-block/core/engine 0.016s
|
||||
ok github.com/seaweedfs/seaweed-block/core/adapter 2.344s
|
||||
ok github.com/seaweedfs/seaweed-block/core/authority 4.569s
|
||||
ok github.com/seaweedfs/seaweed-block/cmd/sparrow 2.264s
|
||||
|
||||
$ go test ./... -count=1
|
||||
(14 packages, all PASS; runtime/schema have no tests)
|
||||
```
|
||||
|
||||
Evidence is reproducible by any reviewer on Windows / Linux from the committed tree on `phase-14` at commit `a6421f9` or later.
|
||||
|
||||
## 7. Residual Risks
|
||||
|
||||
| # | Risk | Scope |
|
||||
|---|------|-------|
|
||||
| 1 | Windows `os.RemoveAll` on the lock file during `t.TempDir()` teardown races with lock release. Does NOT affect correctness — only teardown. | S7 sketch §8.5 residual; logged as `t.Logf` in `restart_route_test.go`. |
|
||||
| 2 | Claim 15 partial: L1 covers "fence command dispatched via real bridge + withheld callback does not produce Healthy"; full timeout-expiry bound is adapter-package-owned. | Closes when adapter fence-watchdog branch commits. Not an S8 blocker. |
|
||||
| 3 | L2 subprocess smoke drives the directive via `StaticDirective`, not real heartbeat ingress to `ObservationHost`. The full heartbeat-path is L1-proven; subprocess widening requires a P15 external surface. | P15 Frontend / Cluster Validation. |
|
||||
|
||||
**No S8-blocking correctness residuals.** One adapter-owned quantitative fence proof (Claim 15) remains as a P14-internal follow-up, landing when the fence-watchdog branch commits. No backward-authority-mint risk on restart. No unsupported-topology silent-idle path.
|
||||
|
||||
## 8. Closure Readiness
|
||||
|
||||
Against S8 acceptance gate (`v3-phase-14-s8-assignment.md` §6):
|
||||
|
||||
1. evidence matrix exists and is honest — **yes** (this document)
|
||||
2. P14 internal blockers fixed or explicitly non-P14 — **yes** (§4.1 empty)
|
||||
3. L0 + L1 tests pass — **yes** (baseline §6)
|
||||
4. L2 process smoke present or absence classified — **yes** (§5; present)
|
||||
5. L3 classified — **yes** (L3 shape only, not executed this slice; see `v3-phase-14-s8-v2-scenario-classification.md`)
|
||||
6. 14A final pass has no open P14 safety blocker — **pending** (14A review on this matrix)
|
||||
7. P15 handoff maps every product gap — **yes** (see `v3-phase-14-s8-p15-handoff.md`)
|
||||
8. final closure statement does not claim production readiness — **yes** (see `v3-phase-14-s8-closure.md`, draft)
|
||||
|
||||
**S8 is evidence-complete pending 14A sign-off.** Awaiting architect + tester review.
|
||||
@@ -1,212 +0,0 @@
|
||||
# V3 Phase 14 S8 Final Bounded Close
|
||||
|
||||
Date: 2026-04-19
|
||||
Status: draft
|
||||
Purpose: close the bounded P14 internal topology/control-plane claim and hand off product surfaces to P15 without pretending P14 is production-ready block storage
|
||||
|
||||
## 1. Target
|
||||
|
||||
`P14 S8` closes one bounded internal control-plane product shape:
|
||||
|
||||
`heartbeat/observation -> ClusterSnapshot -> TopologyController -> Publisher durable authority -> VolumeBridge -> adapter/engine -> observed convergence/restart evidence`.
|
||||
|
||||
S8 is not another protocol slice and not a broad product surface slice. It is the final acceptance and evidence package for the internal single-active-master topology/control-plane loop built by S4-S7.
|
||||
|
||||
## 2. What S8 Must Prove
|
||||
|
||||
S8 must prove, at the appropriate test layer, that:
|
||||
|
||||
1. observation is system-fed, not test-authored
|
||||
2. topology supportability is explicit and fail-closed
|
||||
3. authority is durable, single-owner, and restart-recovered
|
||||
4. controller decisions are system-driven for the accepted topology set
|
||||
5. convergence has bounded fate: confirmed, stuck, or superseded
|
||||
6. restart re-anchors current authority through the real bridge/adapter route
|
||||
7. stale observation, stale old-slot delivery, and unsupported topology do not silently move the system backward
|
||||
8. per-volume isolation holds under mixed supported/unsupported states
|
||||
9. the accepted topology set and unsupported list are explicit
|
||||
10. every product-facing gap is handed to P15, not hidden inside P14 closure
|
||||
|
||||
## 3. What S8 Does Not Prove
|
||||
|
||||
S8 does not prove:
|
||||
|
||||
1. CSI completion
|
||||
2. iSCSI/NVMe product frontend completion
|
||||
3. external volume lifecycle API completion
|
||||
4. real user data-path production readiness
|
||||
5. V2/V3 migration readiness
|
||||
6. multi-master HA / leader election / distributed authority
|
||||
7. final performance or soak readiness
|
||||
|
||||
Those are P15 tracks and the P15 final cluster validation gate.
|
||||
|
||||
S8 may prepare scenario shapes and port test runner muscle for those later gates, but it must not claim them as P14 closure.
|
||||
|
||||
## 4. Accepted Topology Claim
|
||||
|
||||
The S8 supported topology claim is bounded to:
|
||||
|
||||
1. single active master
|
||||
2. multiple volumes
|
||||
3. three bounded replica slots per volume
|
||||
4. distinct server per slot
|
||||
5. one current authoritative primary
|
||||
6. two bounded failover/rebalance candidates
|
||||
7. publisher-owned `Epoch` / `EndpointVersion`
|
||||
8. durable current authority line, one record per volume
|
||||
9. passive convergence: publish, observe, confirm / stuck / supersede
|
||||
10. real `VolumeBridge` delivery into the adapter/engine path
|
||||
|
||||
Anything outside this set is unsupported or deferred.
|
||||
|
||||
## 5. Required Evidence Matrix
|
||||
|
||||
S8 must produce a closure table with these columns:
|
||||
|
||||
| Claim | L0 Unit | L1 Component | L2 Process | L3 Scenario | Status | Residual |
|
||||
|---|---|---|---|---|---|---|
|
||||
|
||||
Minimum claims:
|
||||
|
||||
1. observation supportability
|
||||
2. duplicate/missing/conflicting inventory handling
|
||||
3. durable authority reload
|
||||
4. no old-slot durable revival
|
||||
5. controller-driven bind/reassign/refresh
|
||||
6. convergence confirm/clear
|
||||
7. publish-not-observed stuck evidence
|
||||
8. supersede by newer authority
|
||||
9. restart re-anchor via `VolumeBridge`
|
||||
10. stale observation cannot move backward
|
||||
11. unsupported topology records evidence
|
||||
12. per-volume isolation
|
||||
13. placement/rebalance within accepted topology
|
||||
14. P15 handoff gaps identified
|
||||
|
||||
## 6. Required Test Levels
|
||||
|
||||
### L0 Unit / Invariant
|
||||
|
||||
Required command baseline:
|
||||
|
||||
```text
|
||||
go test ./core/engine ./core/adapter ./core/authority -count=1
|
||||
```
|
||||
|
||||
Must include boundary guards for:
|
||||
|
||||
1. no non-authority `AssignmentInfo` minting
|
||||
2. no observation code constructing assignment asks
|
||||
3. no store minting authority
|
||||
4. no hidden adapter ingress
|
||||
|
||||
### L1 Component
|
||||
|
||||
Required routes:
|
||||
|
||||
1. `ObservationHost -> TopologyController -> Publisher -> VolumeBridge -> VolumeReplicaAdapter`
|
||||
2. observation-fed failover confirms and clears desired
|
||||
3. stuck evidence appears without ask churn
|
||||
4. supersede drops stale desired
|
||||
5. restart route reloads and re-anchors authority
|
||||
6. old-slot live delivery is rejected by adapter monotonicity
|
||||
7. unsupported topology records evidence
|
||||
|
||||
### L2 Local Process
|
||||
|
||||
Required shapes:
|
||||
|
||||
1. real `sparrow` durable authority restart smoke
|
||||
2. process-level S4-S7 route smoke if the current binary surface can expose it cleanly
|
||||
3. if process route smoke cannot be added without inventing P15 surfaces, S8 must record the blocker and carry it to P15 T1/T2/T3
|
||||
|
||||
L2 tests must check exit code, artifact/log output, lock release, reload count, and no backward mint.
|
||||
|
||||
### L3 Hardware / Scenario
|
||||
|
||||
S8 should port scenario shapes from V2 and classify each as:
|
||||
|
||||
1. runnable in P14 internal mode
|
||||
2. blocked by missing P15 frontend/API/data-path surface
|
||||
3. deferred to P15 final cluster validation
|
||||
|
||||
Initial scenario candidates:
|
||||
|
||||
1. restart recovery
|
||||
2. failover
|
||||
3. partition / delayed heartbeat
|
||||
4. unsupported topology
|
||||
5. IO continuity only if frontend/data path exists
|
||||
|
||||
S8 can close P14 with L3 blockers only if the blockers are genuinely P15 product-surface gaps, not internal truth gaps.
|
||||
|
||||
## 7. V2 Port Plan
|
||||
|
||||
### Classify now (S8 scope), port deferred to P15
|
||||
|
||||
S8 **does not port testrunner machinery into the V3 tree** (round-2 doc-consistency fix, aligned with `v3-phase-14-s8-v2-scenario-classification.md` §2). The items below are the V2 muscles S8 inventories and classifies; actual porting is a P15 Final Gate (Cluster Validation Agent) deliverable, not an S8 one.
|
||||
|
||||
1. `weed/storage/blockvol/testrunner/`
|
||||
- **classify now**: runner architecture, scenario YAML shape, action vocabulary, artifact collection, report generation — recorded as PORT-MECHANISM in the classification table. **Port in P15 Final Gate.**
|
||||
|
||||
2. `weed/storage/blockvol/testrunner/scenarios/public` and selected internal scenarios
|
||||
- **classify now**: map each scenario shape to RUNNABLE-P14 / BLOCKED-FRONTEND / BLOCKED-OPS / BLOCKED-HA / BLOCKED-PERF. **Port in P15 Final Gate.**
|
||||
|
||||
3. `weed/storage/blockvol/test/component/`
|
||||
- **classify now**: component harness structure and failure-injection style. **Port in P15 T1 Frontend + Data Path** as that track's real-route harness.
|
||||
|
||||
4. `weed/server/qa_block_*`
|
||||
- **classify now**: coverage taxonomy and route scenario ideas as reference for the evidence matrix (§5). **Port only as applicable per P15 track needs.**
|
||||
|
||||
5. `learn/test` evidence convention
|
||||
- **classify now**: manifest / result / report / run-notes convention referenced as the target shape for future P15 evidence bundles. **Adopt when P15 ships real L3/L4 runs.**
|
||||
|
||||
### Do not port
|
||||
|
||||
1. `promotion.go`
|
||||
2. `HandleAssignment` / `promote` / `demote`
|
||||
3. local runtime self-promotion
|
||||
4. heartbeat timing directly becoming authority
|
||||
5. V2 authority ownership or HA assumptions
|
||||
6. CSI/iSCSI/NVMe product integration as P14 close criteria
|
||||
|
||||
## 8. Deliverables
|
||||
|
||||
S8 must deliver:
|
||||
|
||||
1. final P14 supported-topology statement
|
||||
2. final P14 unsupported/deferred list
|
||||
3. acceptance evidence matrix
|
||||
4. scenario-port classification table
|
||||
5. tester sign-off packet template
|
||||
6. 14A final targeted review checklist
|
||||
7. P15 handoff statement mapping every remaining gap to a P15 track
|
||||
8. code/tests only where needed to make the evidence route real
|
||||
|
||||
## 9. Reject Conditions
|
||||
|
||||
Reject S8 if:
|
||||
|
||||
1. closure is based only on local unit tests
|
||||
2. a claimed route still uses manual assignment choreography
|
||||
3. S8 invents CSI/API/operator surfaces to make tests look complete
|
||||
4. unsupported topology becomes silent idle
|
||||
5. stale observation can emit a newer decision from an old basis
|
||||
6. durable restart can revive stale old-slot truth
|
||||
7. convergence can churn epochs without observation confirmation
|
||||
8. S8 claims production readiness without P15 frontend/data-path/security/cluster validation
|
||||
|
||||
## 10. Closure Statement Shape
|
||||
|
||||
The final S8 closure statement must say exactly:
|
||||
|
||||
1. what P14 now owns
|
||||
2. what P14 explicitly does not own
|
||||
3. what evidence exists at L0/L1/L2/L3
|
||||
4. what risks remain
|
||||
5. which P15 track owns each product gap
|
||||
|
||||
The expected final claim is:
|
||||
|
||||
`P14 has closed one bounded single-active-master internal topology/control-plane truth loop for the accepted topology set. P14 has not closed CSI, external API, frontend data path, migration, security, deployment, performance, or production readiness.`
|
||||
@@ -1,86 +0,0 @@
|
||||
# V3 Phase 14 S8 — P15 Handoff
|
||||
|
||||
Date: 2026-04-20
|
||||
Status: draft (S8 P15 handoff, round-2 aligned to canonical P15 plan)
|
||||
Purpose: map every P14-residual product gap to the canonical P15 track in `v3-phase-15-product-plan.md`, so P15 execution starts from the correct ownership map
|
||||
|
||||
## 1. Handoff Contract
|
||||
|
||||
Per `v3-phase-14-s8-assignment.md` §3.F: every gap that P14 does NOT close must appear here with four columns — the gap, why it is not P14, which canonical P15 track owns it, and the minimum first proof gate.
|
||||
|
||||
**Canonical P15 track numbering** (from `v3-phase-15-product-plan.md` §4):
|
||||
|
||||
1. `T1` Frontend + Data Path Contract
|
||||
2. `T2` CSI / External Lifecycle Surface
|
||||
3. `T3` External Control API
|
||||
4. `T4` Security And Auth Posture
|
||||
5. `T5` Diagnostics And Explainability
|
||||
6. `T6` Operator Workflow
|
||||
7. `T7` V2/V3 Coexistence And Migration
|
||||
8. `T8` Deployment / Upgrade / Release Hardening
|
||||
9. `Final Gate` Cluster Validation Agent
|
||||
|
||||
S8 does NOT invent new tracks, renumber existing ones, or prescribe P15 implementation — only the acceptance gate each P15 track must pass for the handed-off gap.
|
||||
|
||||
## 2. Handoff Table
|
||||
|
||||
| # | Gap | Why not P14 | P15 owner track | Required first proof |
|
||||
|---|---|---|---|---|
|
||||
| 1 | **Frontend data path** (attach → real read → real write → stale-primary fence at frontend boundary) | P14 stops at adapter/engine projection; there is no user-data I/O path in P14, and `adapter.PublishHealthy` is a control-plane signal, not a data-path handshake. | **P15 T1** Frontend + Data Path | L2: V3 frontend backend attaches to an internal volume, writes data, reads it back, triggers P14 reassignment/failover, and verifies the stale primary can no longer serve writes. Covered by T1 §5 Acceptance. |
|
||||
| 2 | **Subprocess heartbeat ingress** — the L2 subprocess smoke (`TestS7Process_RealSubprocessRestartSmoke`) drives mint via `StaticDirective`, not a real heartbeat wire. The full `heartbeat → observation → controller` path is L1-only in the subprocess. | Real heartbeat ingress is a frontend/transport surface. P14 has no intention of adding one. | **P15 T1** Frontend + Data Path (observation ingress is part of the frontend/transport contract) | L2: subprocess accepts heartbeats over a real transport; observation store mutates; controller sees the change; adapter advances — end-to-end in a real process. |
|
||||
| 3 | **CSI driver lifecycle** (CreateVolume / DeleteVolume / ControllerPublish / NodeStage / NodePublish / NodeUnpublish / ControllerUnpublish) | P14 has no orchestrator-facing lifecycle surface; CSI sits above the P14 authority/adapter route. | **P15 T2** CSI / External Lifecycle Surface | L2: CSI sidecar ↔ sparrow process, full Create → Publish → use → Unpublish → Delete cycle via real CSI gRPC against V3 authority (T2 §5 Acceptance). |
|
||||
| 4 | **External control API** (REST / gRPC verbs for list / inspect / create / delete / resize / pause / resume) | P14 exposes hidden test flags on `sparrow` only. An external API is a net-new surface. | **P15 T3** External Control API | L2: OpenAPI / gRPC verbs backed by V3 authority state, exercised by an external client that does NOT construct `AssignmentInfo` directly. |
|
||||
| 5 | **Security / auth posture** (API authn/authz, CHAP for iSCSI, TLS/mTLS where applicable, audit trail) | P14 has no external authn surface. | **P15 T4** Security And Auth | L1+L2: CHAP positive+negative for any iSCSI target shipped; API authn positive+negative; tenant-scoped listing; audit log for mutating verbs. |
|
||||
| 6 | **Operator diagnostics surface** (health dashboard, structured readout of `LastUnsupported` / `LastConvergenceStuck` / `ReloadSkips`, alerts) | P14 records internal evidence (per-volume maps + structured JSON pass-line in S7 subprocess smoke) but does NOT expose it to operators. | **P15 T5** Diagnostics And Explainability | L2: structured `/healthz` + `/metrics` surfacing P14 internal states; L3 `cp85-metrics-verify.yaml` scenario class reshaped for V3. |
|
||||
| 7 | **Operator workflow** (planned failover, graceful drain, supervised reassign, supervised rebuild) | P14 rejects V2 `promote/demote/HandleAssignment`. Operator-initiated actions MUST route through the publisher mint path as ordinary `AssignmentAsk`s. A new workflow surface is required. | **P15 T6** Operator Workflow | L2: operator-triggered reassign lands in `TopologyController` as an `AssignmentAsk`; publisher mints new epoch; adapter converges. Manual promote is explicitly reshaped from V2 semantics. |
|
||||
| 8 | **V2 ↔ V3 migration / coexistence** (online migrate a V2 volume to V3 authority; rollback; mixed V2+V3 cluster) | Migration is a product-level transition requiring both T1 and T6. P14 is V3-only. | **P15 T7** V2/V3 Coexistence And Migration | L3 + L4: `op-upgrade-rollback.yaml` scenario class on a mixed cluster; explicit no-split-brain check under migration. |
|
||||
| 9 | **Deployment / hardening** (systemd / container packaging, lockfile+storedir pathing, crash policy, log rotation, store backup) | `sparrow` is currently a test binary with hidden smoke flags; no deployment story. | **P15 T8** Deployment / Upgrade / Release Hardening | L2: deployable package starts `sparrow` with the durable store correctly mounted; crash-restart cycle preserves the P14 restart truth. L4: release-hardening soak (`cp84-soak-4h.yaml`, `cp85-soak-24h.yaml`). |
|
||||
| 10 | **Final cluster validation** (end-to-end: operator creates volume → data written → primary killed → failover → operator reads data back → no loss, on real hardware) | P14 covers the control plane only; full cluster validation needs T1 + T2 + T3 + T4 + T5 + T6 all landed. | **P15 Final Gate** Cluster Validation Agent | Composite L3/L4 pack: `smoke-block-api` + `ha-restart-recovery` + `ha-failover` + `ha-io-continuity` + `fault-partition` in one run on real hardware, with evidence artifacts in `learn/test/`. |
|
||||
| 11 | **Adapter fence-watchdog quantitative timeout bound** (watchdog fires at exact fence deadline + structured watchdog event for fence lineage) | S7 L1 covers "Fence command reaches executor via bridge + withheld callback does not produce Healthy"; the quantitative bound is adapter-package-owned and depends on the currently-uncommitted fence-watchdog branch. S7 sketch §10.1 Path B forbids S7 from reconfiguring the fence path. | **P14 internal follow-up** (NOT a P15 track) | Single adapter-package test replacing the PARTIAL row for evidence-matrix Claim 15. Lands when the fence-watchdog branch commits. Not an S8 gate. |
|
||||
|
||||
## 3. Out-Of-Scope (NOT Handed Off)
|
||||
|
||||
These are explicit non-goals for both P14 and current P15 bounds — not failures, not deferrals, permanent exclusions.
|
||||
|
||||
| Non-goal | Reason |
|
||||
|---|---|
|
||||
| Multi-master / leader election / distributed authority store | `v3-phase-14-s8-final-bounded-close.md` §3.6 and `v3-phase-15-product-plan.md` §2 both reject. Single-active-master is the accepted topology. Any multi-master claim would require a separate phase with a distributed-authority institution. |
|
||||
| V2 `HandleAssignment` / `promote` / `demote` authority-owning semantic | Rejected at S2, reaffirmed S3-S8. Not ported under any P15 track. Operator promote surface in T6 is reshaped, not ported. |
|
||||
| V2 heartbeat-as-authority | Rejected at S4. Heartbeats are observation inputs only under any P14 or P15 surface. |
|
||||
|
||||
## 4. Handoff Integrity Check
|
||||
|
||||
The table splits into two distinct row classes, each with its own integrity rule:
|
||||
|
||||
**Rows 1-10 — P15 product-surface gaps.** Each of these must satisfy all four checks as of 2026-04-20:
|
||||
|
||||
1. **Gap is not proven at any P14 level.** Cross-reference `v3-phase-14-s8-evidence-matrix.md` §3 — none of rows 1-10 appear as PROVEN or PARTIAL rows of the internal matrix. These are genuinely product-surface gaps P14 never attempted.
|
||||
2. **Gap is assigned to a canonical P15 track.** Numbering matches `v3-phase-15-product-plan.md` §4.
|
||||
3. **First-proof gate is concrete**, not a hand-wave. Every row names a specific test layer and a specific scenario or surface shape.
|
||||
4. **Gap does not silently depend on a P14 claim being wider than is actually proven.** Example: P15 T1 is allowed to depend on P14 Claim 10 (restart re-anchors via VolumeBridge, PROVEN at L1) but NOT on any data-path claim — P14 makes none.
|
||||
|
||||
**Row 11 — P14 internal follow-up (exception to rule #1).** Row 11 is NOT a P15 gap; it is a targeted P14 internal completion that lands when the fence-watchdog branch commits. Its integrity rules are narrower:
|
||||
|
||||
1. **Row 11 MUST correspond to an existing PARTIAL or subclaim-only row of the internal matrix** — the whole point of the row is to name the follow-up that closes that partial. In this case row 11 corresponds to evidence-matrix **Claim 15 PARTIAL** (fence quantitative timeout bound). This is the inverse of rule #1 above: row 11 is listed here precisely BECAUSE it appears as PARTIAL in the matrix.
|
||||
2. **Row 11 MUST NOT be assigned to any P15 track** — it is explicitly labeled "P14 internal follow-up (NOT a P15 track)" in the gap table.
|
||||
3. **Row 11's first-proof gate MUST be a single-package test that closes the matrix PARTIAL row** — not a multi-track composite.
|
||||
|
||||
Splitting the integrity check this way keeps rule #1 for P15 gaps strict (any cross-reference to PROVEN/PARTIAL in the matrix is a bug there) while making row 11's purpose explicit and auditable (every P14 internal follow-up row should have a matrix PARTIAL anchor).
|
||||
|
||||
## 5. Per-Track Dependency Summary
|
||||
|
||||
| P15 Track | Depends on P14 claims | Notes |
|
||||
|---|---|---|
|
||||
| T1 Frontend + Data Path | Claims 3, 6, 10, 14 (durable reload, controller-driven, restart re-anchor, placement/rebalance) | Core product dependency. |
|
||||
| T2 CSI / Lifecycle | T1 + Claims 6, 10 | CSI needs a working data path. |
|
||||
| T3 External API | Claims 6, 14 | API translates external verbs to `AssignmentAsk`s. |
|
||||
| T4 Security | T1 + T3 | Authn/authz sits above the surfaces T1 and T3 expose. |
|
||||
| T5 Diagnostics | Claims 2, 8, 11, 12 (supportability / stuck / unsupported evidence) | Surfaces what P14 already records internally. |
|
||||
| T6 Operator Workflow | Claims 6, 9, 14 (controller-driven, supersede, placement) | Operator actions are `AssignmentAsk`s through the controller. |
|
||||
| T7 Migration | T1 + T6 | Needs both frontend and operator surfaces. |
|
||||
| T8 Deployment | Claim 3 (durable reload) + T1 | Depends on durable store shape and frontend. |
|
||||
| Final Gate | All P14 claims + all P15 tracks | Composite. |
|
||||
|
||||
## 6. Closure
|
||||
|
||||
Every product gap that S8 identifies is listed here with a canonical P15 track or explicit P14 follow-up. Track numbering matches `v3-phase-15-product-plan.md`. P15 execution starts from this table without needing to re-derive the ownership map.
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user