mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-06 14:45:51 +00:00
Compare commits
16
Commits
@@ -1,66 +0,0 @@
|
||||
name: Fix fusermount3 setuid
|
||||
description: >
|
||||
Make sure the fusermount3 an unprivileged mount will find can actually mount.
|
||||
Some runner images carry a second, source-built fusermount3 in /usr/local/bin
|
||||
that shadows the distro one in PATH; it is neither setuid nor root-owned, so
|
||||
every mount fails with "mount failed: Operation not permitted". Both the Go
|
||||
mount and sw-fuse resolve the helper through PATH.
|
||||
Run it after the step that apt-installs fuse3.
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Point PATH at a fusermount3 that can mount
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# setuid only grants root when root owns the file, and exec follows
|
||||
# symlinks, so judge the target.
|
||||
can_mount() {
|
||||
[ -u "$1" ] && [ "$(stat -Lc %u "$1")" = 0 ]
|
||||
}
|
||||
|
||||
bin=$(command -v fusermount3 || true)
|
||||
if [ -z "$bin" ]; then
|
||||
echo "no fusermount3 in PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
if can_mount "$bin"; then
|
||||
ls -l "$bin"
|
||||
exit 0
|
||||
fi
|
||||
echo "$bin cannot mount unprivileged:"
|
||||
ls -l "$bin"
|
||||
|
||||
# The distro fusermount3 is setuid root and is the one meant to be used.
|
||||
# Reach it through a symlink earlier in PATH: exec resolves the link, so
|
||||
# the target keeps its setuid bit, and nothing on the image is modified.
|
||||
for distro in /usr/bin/fusermount3 /bin/fusermount3; do
|
||||
if [ "$distro" != "$bin" ] && can_mount "$distro"; then
|
||||
mkdir -p "$RUNNER_TEMP/fuse-bin"
|
||||
ln -sf "$distro" "$RUNNER_TEMP/fuse-bin/fusermount3"
|
||||
echo "$RUNNER_TEMP/fuse-bin" >> "$GITHUB_PATH"
|
||||
echo "using $distro instead"
|
||||
ls -l "$distro"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
# No usable distro binary, so setting the bit is the only way out - but
|
||||
# the target came from PATH: do that only for a root-owned system binary,
|
||||
# never for anything else that happens to sit there.
|
||||
case "$bin" in
|
||||
/bin/*|/sbin/*|/usr/bin/*|/usr/sbin/*|/usr/local/bin/*|/usr/local/sbin/*) ;;
|
||||
*) echo "refusing to setuid $bin: outside the system bin paths" >&2; exit 1 ;;
|
||||
esac
|
||||
if [ -L "$bin" ] || [ ! -f "$bin" ] || [ ! -x "$bin" ]; then
|
||||
echo "refusing to setuid $bin: not a regular executable file" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(stat -c %u "$bin")" != 0 ] || [ "$(stat -c %g "$bin")" != 0 ]; then
|
||||
echo "refusing to setuid $bin: not owned by root:root" >&2
|
||||
exit 1
|
||||
fi
|
||||
sudo chmod u+s "$bin"
|
||||
ls -l "$bin"
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.37.4
|
||||
uses: github/codeql-action/init@v4
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4.37.4
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -49,4 +49,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4.37.4
|
||||
uses: github/codeql-action/analyze@v4
|
||||
|
||||
@@ -135,14 +135,14 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -221,13 +221,13 @@ jobs:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -275,7 +275,7 @@ jobs:
|
||||
fi
|
||||
- name: Login to GHCR
|
||||
if: needs.setup.outputs.publish == 'true'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -367,7 +367,7 @@ jobs:
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4.37.4
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
@@ -430,12 +430,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -219,7 +219,7 @@ jobs:
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -293,7 +293,7 @@ jobs:
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -374,7 +374,7 @@ jobs:
|
||||
variant: large_disk
|
||||
steps:
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -400,7 +400,7 @@ jobs:
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4.37.4
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
@@ -423,13 +423,13 @@ jobs:
|
||||
latest_tag: latest_large_disk
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4.6.0
|
||||
uses: docker/login-action@v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -88,7 +88,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v1
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
@@ -8,7 +8,6 @@ on:
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/workflows/fuse-dlm-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
@@ -16,7 +15,6 @@ on:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/cluster/**'
|
||||
- 'test/fuse_dlm/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration
|
||||
@@ -47,9 +45,6 @@ jobs:
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
|
||||
@@ -7,14 +7,12 @@ on:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
pull_request:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/**'
|
||||
- 'test/fuse_integration/**'
|
||||
- '.github/workflows/fuse-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/fuse-integration
|
||||
@@ -51,9 +49,6 @@ jobs:
|
||||
fusermount3 --version || fusermount --version || true
|
||||
ls -la /dev/fuse
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
cd weed
|
||||
|
||||
@@ -11,7 +11,6 @@ on:
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/workflows/fuse-p2p-integration.yml'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
@@ -22,7 +21,6 @@ on:
|
||||
- 'weed/pb/mount_peer.proto'
|
||||
- 'weed/pb/filer.proto'
|
||||
- 'test/fuse_p2p/**'
|
||||
- '.github/actions/fix-fusermount-setuid/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration
|
||||
@@ -53,9 +51,6 @@ jobs:
|
||||
echo 'user_allow_other' | sudo tee -a /etc/fuse.conf
|
||||
sudo chmod 644 /etc/fuse.conf
|
||||
|
||||
- name: Repair the fusermount3 setuid bit
|
||||
uses: ./.github/actions/fix-fusermount-setuid
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: go build -o weed/weed -buildvcs=false ./weed
|
||||
|
||||
|
||||
@@ -75,39 +75,6 @@ jobs:
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
build-cross:
|
||||
name: Build cross-platform (${{ matrix.goos }}/${{ matrix.goarch }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# One target's breakage should not hide the other three.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- { goos: windows, goarch: amd64 }
|
||||
- { goos: windows, goarch: arm64 }
|
||||
- { goos: freebsd, goarch: amd64 }
|
||||
- { goos: darwin, goarch: arm64 }
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
# Nothing else on a PR compiles these, so per-OS syscall constants creep
|
||||
# back into shared files unnoticed and only a release build catches it.
|
||||
- name: Cross-compile
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
run: |
|
||||
go build ./weed/...
|
||||
# Tests too: they reach for per-OS syscall constants the build does
|
||||
# not, and only compiling them catches an untagged one.
|
||||
go vet ./weed/mount/... ./weed/command/... 2>&1 |
|
||||
grep -v "MessageState contains sync.Mutex" | tee /tmp/vet.txt
|
||||
if grep -q "vet:" /tmp/vet.txt; then exit 1; fi
|
||||
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -246,172 +246,6 @@ jobs:
|
||||
PYEOF
|
||||
echo "IAM gRPC decoupling tests passed"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing bucket hook credentials ==="
|
||||
# The bucket hook pipes s3.configure into `weed shell`, which needs
|
||||
# the filer JWT signing key once jwtSigning.filerWrite is on: the
|
||||
# filer rejects unsigned IAM gRPC calls, and `weed shell` still
|
||||
# exits 0, so the Job goes green while anonymousRead is dropped.
|
||||
# The key reaches the hook either through security.toml or through
|
||||
# an environment override in extraEnvironmentVars.
|
||||
python3 - "$CHART_DIR" <<'PYEOF'
|
||||
import re, subprocess, sys, yaml
|
||||
|
||||
chart = sys.argv[1]
|
||||
|
||||
def render(values):
|
||||
args = ["helm", "template", "test", chart]
|
||||
for k, v in values.items():
|
||||
args += ["--set", f"{k}={v}"]
|
||||
return subprocess.check_output(args, text=True)
|
||||
|
||||
def hook_pod(manifest):
|
||||
for d in yaml.safe_load_all(manifest):
|
||||
if d and d.get("kind") == "Job" and d["metadata"]["name"].endswith("-bucket-hook"):
|
||||
return d["spec"]["template"]["spec"]
|
||||
return None
|
||||
|
||||
def cluster_endpoints(manifest):
|
||||
"""Where the chart's own Services put master and filer.
|
||||
|
||||
The hook's WEED_CLUSTER_* values have to match these: a defined
|
||||
name pointing anywhere else fails the readiness wait exactly as
|
||||
an undefined one does.
|
||||
"""
|
||||
wanted = {"-master": "swfs-master", "-filer-client": "swfs-filer"}
|
||||
found = {}
|
||||
for d in yaml.safe_load_all(manifest):
|
||||
if not d or d.get("kind") != "Service":
|
||||
continue
|
||||
for suffix, portName in wanted.items():
|
||||
if not d["metadata"]["name"].endswith(suffix):
|
||||
continue
|
||||
port = next((p["port"] for p in d["spec"]["ports"] if p.get("name") == portName), None)
|
||||
if port:
|
||||
found[suffix] = f"{d['metadata']['name']}.{d['metadata']['namespace']}:{port}"
|
||||
return found.get("-master"), found.get("-filer-client")
|
||||
|
||||
buckets = {
|
||||
"s3.enabled": "true",
|
||||
"s3.createBuckets[0].name": "data",
|
||||
"s3.createBuckets[0].anonymousRead": "true",
|
||||
}
|
||||
jwt = dict(buckets, **{"global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true"})
|
||||
|
||||
failed = []
|
||||
|
||||
# The script in the hook addresses master and filer through
|
||||
# WEED_CLUSTER_*, now rendered next to extraEnvironmentVars.
|
||||
# Duplicate names would be a rendering bug.
|
||||
out = render(buckets)
|
||||
pod = hook_pod(out)
|
||||
if pod is None:
|
||||
failed.append("createBuckets: bucket hook Job missing")
|
||||
else:
|
||||
env = pod["containers"][0]["env"]
|
||||
names = [e["name"] for e in env]
|
||||
dupes = sorted({n for n in names if names.count(n) > 1})
|
||||
if dupes:
|
||||
failed.append(f"createBuckets: duplicate env entries {dupes}")
|
||||
values = {e["name"]: e.get("value") for e in env}
|
||||
if not values.get("WEED_CLUSTER_DEFAULT"):
|
||||
failed.append("createBuckets: hook env WEED_CLUSTER_DEFAULT is empty")
|
||||
rendered = (values.get("WEED_CLUSTER_SW_MASTER"), values.get("WEED_CLUSTER_SW_FILER"))
|
||||
expected = cluster_endpoints(out)
|
||||
if not all(expected):
|
||||
failed.append(f"createBuckets: no master/filer-client Service to compare against, found {expected}")
|
||||
elif rendered != expected:
|
||||
failed.append(f"createBuckets: hook env holds {rendered}, the services are at {expected}")
|
||||
elif not dupes:
|
||||
print("createBuckets: hook env resolves the cluster addresses once")
|
||||
if any(v["name"] == "security-config" for v in pod.get("volumes", [])):
|
||||
failed.append("createBuckets: hook mounts security-config without JWT signing")
|
||||
|
||||
# filerWrite=true: the hook needs the same security.toml the filer
|
||||
# gets, otherwise s3.configure fails with "missing authorization
|
||||
# metadata".
|
||||
pod = hook_pod(render(jwt))
|
||||
if pod is None:
|
||||
failed.append("filerWrite=true: bucket hook Job missing")
|
||||
else:
|
||||
vol = next((v for v in pod.get("volumes", []) if v["name"] == "security-config"), None)
|
||||
mount = next((m for m in pod["containers"][0].get("volumeMounts", []) if m["name"] == "security-config"), None)
|
||||
# The file has to land on one of weed's config search paths, so
|
||||
# the target matters as much as the volume itself - and a volume
|
||||
# of that name backed by anything but the chart's ConfigMap
|
||||
# leaves `weed shell` without the signing key just the same.
|
||||
target = ("/etc/seaweedfs/security.toml", "security.toml")
|
||||
configmap = "test-seaweedfs-security-config"
|
||||
if vol is None or mount is None:
|
||||
failed.append("filerWrite=true: bucket hook does not mount security-config (s3.configure would fail)")
|
||||
elif (mount.get("mountPath"), mount.get("subPath")) != target:
|
||||
failed.append(f"filerWrite=true: bucket hook mounts security-config as {mount}")
|
||||
elif vol.get("configMap", {}).get("name") != configmap:
|
||||
failed.append(f"filerWrite=true: security-config is not backed by the {configmap} ConfigMap: {vol}")
|
||||
else:
|
||||
print("filerWrite=true: bucket hook mounts security-config at /etc/seaweedfs/security.toml")
|
||||
|
||||
# The readiness waits dereference the cluster env names, so renaming
|
||||
# the cluster alias has to rename them too: an undefined name leaves
|
||||
# the hook polling "http://" for five minutes and then failing the
|
||||
# release. The endpoints stay chart-computed, so an address kept in
|
||||
# extraEnvironmentVars is replaced, not rendered twice.
|
||||
out = render(dict(buckets, **{
|
||||
"global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_DEFAULT": "prod",
|
||||
"global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_PROD_MASTER": "stale:9333",
|
||||
"global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_SW_MASTER": "null",
|
||||
"global.seaweedfs.extraEnvironmentVars.WEED_CLUSTER_SW_FILER": "null",
|
||||
}))
|
||||
pod = hook_pod(out)
|
||||
if pod is None:
|
||||
failed.append("cluster alias: bucket hook Job missing")
|
||||
else:
|
||||
container = pod["containers"][0]
|
||||
env = {e["name"]: e.get("value") for e in container["env"]}
|
||||
waited = re.findall(r'wait_for_service "http://\$(\w+)', container["command"][2])
|
||||
undefined = sorted(n for n in set(waited) if not env.get(n))
|
||||
renamed = (env.get("WEED_CLUSTER_PROD_MASTER"), env.get("WEED_CLUSTER_PROD_FILER"))
|
||||
expected = cluster_endpoints(out)
|
||||
if not waited:
|
||||
failed.append("cluster alias: hook script has no readiness waits")
|
||||
elif undefined:
|
||||
failed.append(f"cluster alias: hook waits on undefined env {undefined}")
|
||||
elif not all(expected):
|
||||
failed.append(f"cluster alias: no master/filer-client Service to compare against, found {expected}")
|
||||
elif renamed != expected:
|
||||
# Both addresses under the renamed alias, not just the one
|
||||
# the values tried to keep: the stale one would fail the
|
||||
# release, and a missing filer address just as much.
|
||||
failed.append(f"cluster alias: hook waits on {renamed}, the services are at {expected}")
|
||||
else:
|
||||
print("cluster alias: hook waits on the renamed names and the chart's addresses")
|
||||
|
||||
# Keys kept in a Secret are referenced, not inlined, so the hook
|
||||
# must render valueFrom for non-string extraEnvironmentVars.
|
||||
out = render(dict(jwt, **{
|
||||
"global.seaweedfs.extraEnvironmentVars.WEED_JWT_FILER_SIGNING_KEY.secretKeyRef.name": "signing-keys",
|
||||
"global.seaweedfs.extraEnvironmentVars.WEED_JWT_FILER_SIGNING_KEY.secretKeyRef.key": "filerWrite",
|
||||
}))
|
||||
pod = hook_pod(out)
|
||||
if pod is None:
|
||||
failed.append("secretKeyRef: bucket hook Job missing")
|
||||
else:
|
||||
entry = next((e for e in pod["containers"][0]["env"] if e["name"] == "WEED_JWT_FILER_SIGNING_KEY"), None)
|
||||
if entry is None:
|
||||
failed.append("secretKeyRef: hook env WEED_JWT_FILER_SIGNING_KEY missing")
|
||||
elif entry.get("valueFrom", {}).get("secretKeyRef") != {"name": "signing-keys", "key": "filerWrite"}:
|
||||
failed.append(f"secretKeyRef: hook env rendered as {entry}")
|
||||
else:
|
||||
print("secretKeyRef: hook env keeps the secret reference")
|
||||
|
||||
if failed:
|
||||
print("\nFAIL:", file=sys.stderr)
|
||||
for f in failed:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
echo "Bucket hook credential tests passed"
|
||||
|
||||
echo "=== Testing with monitoring enabled ==="
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.monitoring.enabled=true \
|
||||
@@ -1417,12 +1251,9 @@ jobs:
|
||||
- {}
|
||||
EOF
|
||||
|
||||
# Only the marker line, so a probe that never produced one cannot be
|
||||
# mistaken for a verdict: "pod/x created" on stdout would read as
|
||||
# "not exit=0", which is how the denied case passes. busybox wget
|
||||
# reports a timeout on stderr even under -q and kubectl logs returns
|
||||
# both streams, so the marker has to be picked out rather than taken
|
||||
# as the whole log. Empty output stays empty - the callers fail on it.
|
||||
# Only the pod's own output, so a probe that never produced one cannot
|
||||
# be mistaken for a verdict: "pod/x created" on stdout would read as
|
||||
# "not exit=0", which is how the denied case passes.
|
||||
probe() {
|
||||
local name=$1
|
||||
local labels=$2
|
||||
@@ -1431,7 +1262,7 @@ jobs:
|
||||
sh -c "wget -T 5 -q -O /dev/null http://$FILER_IP:8888/; echo exit=\$?" >/dev/null
|
||||
kubectl wait -n "$NS" --for=jsonpath='{.status.phase}'=Succeeded \
|
||||
"pod/$name" --timeout=120s >/dev/null
|
||||
kubectl logs -n "$NS" "$name" | grep '^exit=' || true
|
||||
kubectl logs -n "$NS" "$name"
|
||||
}
|
||||
|
||||
# A pod carrying the release labels is what the filer's policy allows.
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
name: "mount: windows conformance"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp-conformance/**'
|
||||
- '.github/workflows/mount-windows-conformance.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
conformance:
|
||||
name: WinFsp conformance
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and run winfsp-tests
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Write-Host "::group::winfsp-tests"
|
||||
& pwsh -File test/winfsp-conformance/run.ps1 -MountPoint S:\
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "winfsp-tests failed with exit $code" }
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -1,194 +0,0 @@
|
||||
name: "mount: windows"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
# No base branch filter: this is the only thing that runs the Windows mount,
|
||||
# so it should cover a pull request stacked on another one too.
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/mount/**'
|
||||
- 'weed/command/mount*.go'
|
||||
- 'weed/storage/volume_vacuum*.go'
|
||||
- 'weed/storage/volume_loading.go'
|
||||
- 'weed/storage/disk_location.go'
|
||||
- 'test/winfsp/**'
|
||||
- '.github/workflows/mount-windows.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
mount-windows:
|
||||
name: Mount on Windows
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 40
|
||||
env:
|
||||
# The runner ships MinGW, so cgo is on by default and cgofuse picks its
|
||||
# cgo variant, which wants WinFsp's headers. The nocgo variant loads
|
||||
# winfsp-x64.dll at run time instead, which is how weed.exe is released.
|
||||
CGO_ENABLED: 0
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
# cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but
|
||||
# not to build.
|
||||
- name: Install WinFsp
|
||||
run: choco install winfsp -y --no-progress
|
||||
|
||||
- name: Build weed.exe
|
||||
run: go build -o weed.exe ./weed
|
||||
|
||||
# The runner tears down a step's process tree when its shell exits, so a
|
||||
# cluster started in one step is gone by the next. Everything that needs
|
||||
# the cluster and the mount alive has to share a step.
|
||||
- name: Mount and exercise
|
||||
shell: pwsh
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function Test-Port($port) {
|
||||
# A plain connect, because Test-NetConnection has reported success
|
||||
# here for a port nothing was listening on.
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try { $client.Connect('127.0.0.1', $port); return $client.Connected }
|
||||
catch { return $false }
|
||||
finally { $client.Dispose() }
|
||||
}
|
||||
|
||||
function Start-Mount($log) {
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=S:' `
|
||||
-RedirectStandardOutput "C:\$log.log" -RedirectStandardError "C:\$log.err.log"
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if (Test-Path S:\) { Write-Host "S: is mounted"; return }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
Get-Content "C:\$log.log", "C:\$log.err.log" -ErrorAction SilentlyContinue
|
||||
throw "S: never appeared"
|
||||
}
|
||||
|
||||
function Stop-Mount {
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
$deadline = (Get-Date).AddMinutes(1)
|
||||
while ((Get-Date) -lt $deadline -and (Test-Path S:\)) { Start-Sleep -Seconds 2 }
|
||||
if (Test-Path S:\) { throw "S: still present after stopping the mount" }
|
||||
Write-Host "unmounted"
|
||||
}
|
||||
|
||||
function Invoke-Tests($label, [string[]]$goArgs) {
|
||||
Write-Host "::group::$label"
|
||||
& go @goArgs
|
||||
$code = $LASTEXITCODE
|
||||
Write-Host "::endgroup::"
|
||||
if ($code -ne 0) { throw "$label failed with exit $code" }
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path C:\seaweed-data | Out-Null
|
||||
# -ip pins the cluster to loopback; it otherwise advertises and binds
|
||||
# the runner's LAN address, which 127.0.0.1 cannot reach.
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mini','-dir=C:\seaweed-data','-ip=127.0.0.1' `
|
||||
-RedirectStandardOutput C:\seaweed-mini.log -RedirectStandardError C:\seaweed-mini.err.log
|
||||
|
||||
$deadline = (Get-Date).AddMinutes(3)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# The mount dials grpc, not http, so both ports have to answer.
|
||||
if ((Test-Port 8888) -and (Test-Port 18888)) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not ((Test-Port 8888) -and (Test-Port 18888))) {
|
||||
Get-Content C:\seaweed-mini.log, C:\seaweed-mini.err.log -ErrorAction SilentlyContinue
|
||||
throw "filer never came up"
|
||||
}
|
||||
Write-Host "filer is up on http 8888 and grpc 18888"
|
||||
|
||||
Start-Mount 'seaweed-mount'
|
||||
|
||||
Invoke-Tests 'exercise' @('test','-v','-timeout','20m','./test/winfsp','-mountpoint=S:\')
|
||||
Invoke-Tests 'persist-write' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=write','-filer=127.0.0.1:8888')
|
||||
|
||||
Stop-Mount
|
||||
Start-Mount 'seaweed-remount'
|
||||
|
||||
Invoke-Tests 'persist-verify' @('test','-v','-timeout','15m','./test/winfsp','-run','TestPersistence','-mountpoint=S:\','-phase=verify')
|
||||
|
||||
# WinFsp creates the mount directory itself, so the path must not
|
||||
# exist; only its parent has to.
|
||||
Write-Host "::group::mount over a directory"
|
||||
Stop-Mount
|
||||
Remove-Item C:\seaweed-mnt -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Start-Process -FilePath .\weed.exe `
|
||||
-ArgumentList '-logtostderr','mount','-filer=127.0.0.1:8888','-dir=C:\seaweed-mnt' `
|
||||
-RedirectStandardOutput C:\seaweed-dirmount.log -RedirectStandardError C:\seaweed-dirmount.err.log
|
||||
$deadline = (Get-Date).AddMinutes(2)
|
||||
$ok = $false
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
# Listing succeeds on the plain empty directory too, so wait for the
|
||||
# reparse point WinFsp turns it into. Otherwise this step passes
|
||||
# without a mount and writes to local disk.
|
||||
$item = Get-Item C:\seaweed-mnt -Force -ErrorAction SilentlyContinue
|
||||
if ($null -ne $item -and $item.Attributes.ToString() -like '*ReparsePoint*') { $ok = $true; break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
if (-not $ok) {
|
||||
Get-Content C:\seaweed-dirmount.log, C:\seaweed-dirmount.err.log -ErrorAction SilentlyContinue
|
||||
throw "mounting over a directory failed"
|
||||
}
|
||||
Set-Content -Path C:\seaweed-mnt\dirmount.txt -Value 'via directory mount'
|
||||
if ((Get-Content C:\seaweed-mnt\dirmount.txt) -ne 'via directory mount') { throw "readback through the directory mount differs" }
|
||||
Remove-Item C:\seaweed-mnt\dirmount.txt -Force
|
||||
Get-CimInstance Win32_Process -Filter "Name = 'weed.exe'" |
|
||||
Where-Object { $_.CommandLine -like '*mount*' } |
|
||||
ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
|
||||
Start-Sleep -Seconds 5
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
Start-Mount 'seaweed-remount2'
|
||||
|
||||
Write-Host "::group::explorer-style walk"
|
||||
New-Item -ItemType Directory -Force -Path S:\walk | Out-Null
|
||||
1..200 | ForEach-Object { Set-Content -Path "S:\walk\f$_.txt" -Value "line $_" }
|
||||
$names = @(Get-ChildItem S:\walk | ForEach-Object { $_.Name })
|
||||
if ($names.Count -ne 200) {
|
||||
# Name the strays: a dot entry surfacing here is a different problem
|
||||
# from a missing or duplicated file.
|
||||
$unexpected = $names | Where-Object { $_ -notmatch '^f\d+\.txt$' }
|
||||
throw "listed $($names.Count) entries, expected 200; unexpected: $($unexpected -join ', ')"
|
||||
}
|
||||
$body = Get-Content S:\walk\f42.txt
|
||||
if ($body -ne 'line 42') { throw "unexpected content: $body" }
|
||||
Copy-Item S:\walk\f42.txt S:\walk\copy.txt
|
||||
Remove-Item S:\walk -Recurse -Force
|
||||
if (Test-Path S:\walk) { throw "directory survived recursive delete" }
|
||||
Write-Host "::endgroup::"
|
||||
|
||||
- name: Logs
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
foreach ($f in 'C:\seaweed-mount.log','C:\seaweed-mount.err.log','C:\seaweed-remount.log','C:\seaweed-remount.err.log','C:\seaweed-remount2.log','C:\seaweed-remount2.err.log','C:\seaweed-dirmount.log','C:\seaweed-dirmount.err.log','C:\seaweed-mini.log','C:\seaweed-mini.err.log') {
|
||||
if (Test-Path $f) { Write-Host "===== $f"; Get-Content $f -Tail 200 }
|
||||
}
|
||||
@@ -42,12 +42,6 @@ jobs:
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
@@ -55,9 +49,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
rust-
|
||||
|
||||
- name: Build Rust volume server
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
@@ -85,12 +79,6 @@ jobs:
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
@@ -98,9 +86,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
@@ -146,9 +134,6 @@ jobs:
|
||||
name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -170,12 +155,6 @@ jobs:
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
# cargo tracks its own inputs but not the runner's C toolchain, so a cached
|
||||
# target/ can carry C objects built against a different glibc than we link against.
|
||||
- name: Fingerprint build toolchain
|
||||
id: toolchain
|
||||
run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache cargo registry and target
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
@@ -183,9 +162,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
seaweed-volume/target
|
||||
key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
key: rust-${{ hashFiles('seaweed-volume/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
rust-${{ steps.toolchain.outputs.fingerprint }}-
|
||||
rust-
|
||||
|
||||
- name: Build Go weed binary
|
||||
run: |
|
||||
@@ -197,28 +176,30 @@ jobs:
|
||||
- name: Build Rust volume binary
|
||||
run: cd seaweed-volume && cargo build --release
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test -tags 5BytesOffset ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests with Rust volume
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/seaweed-volume/target/release/weed-volume
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
run: |
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running Go volume server tests with Rust volume for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
go test -v -count=1 -tags 5BytesOffset -timeout=30m ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
@@ -239,6 +220,23 @@ jobs:
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Rust Volume - Go Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Volume server: Rust (VOLUME_SERVER_IMPL=rust)" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -38,22 +38,7 @@ jobs:
|
||||
working-directory: test/s3/versioning
|
||||
run: |
|
||||
set -x
|
||||
# Run every versioning test, so a regression test lands covered instead
|
||||
# of waiting for someone to remember this file. Name a test in EXCLUDE,
|
||||
# with the reason, to keep it out.
|
||||
#
|
||||
# TestVersioningPagination*: opt-in stress tests that build 1500+
|
||||
# versions. They self-skip without ENABLE_STRESS_TESTS and have their
|
||||
# own make target, so this gate should not carry them.
|
||||
EXCLUDE='TestVersioningPagination.*'
|
||||
tests=$(go test . -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and pass this job vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no versioning tests"; exit 1; }
|
||||
selected=$(echo "$tests" | grep -vE "^($EXCLUDE)$")
|
||||
[ -n "$selected" ] || { echo "EXCLUDE matched every test"; exit 1; }
|
||||
echo "running $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) versioning tests"
|
||||
# make swallows a lone trailing $, taking the anchor with it, so escape it.
|
||||
make test-with-server TEST_PATTERN="^($(echo "$selected" | paste -sd'|' -))"'$$'
|
||||
make test-with-server TEST_PATTERN="TestVersioningCompleteMultipartUploadIsIdempotent|TestVersioningSelfCopyMetadataReplaceCreatesNewVersion|TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion|TestSuspendedDeleteCreatesDeleteMarker"
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
|
||||
@@ -694,12 +694,10 @@ jobs:
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-copy-test-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
set -x
|
||||
# Each test bucket is its own collection and grows 7 volumes; the suite runs
|
||||
# faster than the heartbeat that returns slots from the deleted collections.
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=300 -volume.preStopSeconds=1 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
|
||||
-s3.allowDeleteBucketNotEmpty=true -s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" -master.peers=none &
|
||||
pid=$!
|
||||
|
||||
@@ -29,8 +29,6 @@ permissions:
|
||||
|
||||
env:
|
||||
TEST_TIMEOUT: '30m'
|
||||
# Keep in step with the length of matrix.shard below.
|
||||
SHARD_COUNT: 3
|
||||
|
||||
jobs:
|
||||
volume-server-integration-tests:
|
||||
@@ -59,26 +57,28 @@ jobs:
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
# Dealing the listed tests out one by one keeps the shards even. Bucketing
|
||||
# them by first letter did not: names cluster, so ^Test[I-S] drew 50 of
|
||||
# the 114 grpc tests and ran nearly twice as long as the other two shards.
|
||||
- name: Select this shard's tests
|
||||
env:
|
||||
TEST_TYPE: ${{ matrix.test-type }}
|
||||
SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
tests=$(go test ./test/volume_server/"$TEST_TYPE"/... -list '.*' | grep '^Test' | sort -u)
|
||||
# An empty list would make -run match nothing and the shard pass vacuously.
|
||||
[ -n "$tests" ] || { echo "listed no tests in test/volume_server/$TEST_TYPE"; exit 1; }
|
||||
selected=$(echo "$tests" | awk -v n="$SHARD_COUNT" -v i="$SHARD" 'NR % n == i - 1')
|
||||
echo "shard $SHARD of $SHARD_COUNT runs $(echo "$selected" | wc -l) of $(echo "$tests" | wc -l) tests"
|
||||
echo "TEST_PATTERN=^($(echo "$selected" | paste -sd'|' -))\$" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run volume server integration tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
run: |
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }} of ${SHARD_COUNT})..."
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "Running volume server integration tests for ${{ matrix.test-type }} (Shard ${{ matrix.shard }}, pattern: ${TEST_PATTERN})..."
|
||||
go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run "${TEST_PATTERN}"
|
||||
|
||||
- name: Collect logs on failure
|
||||
@@ -99,6 +99,23 @@ jobs:
|
||||
- name: Test summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ matrix.test-type }}" == "grpc" ]; then
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-H]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[I-S]"
|
||||
else
|
||||
TEST_PATTERN="^Test[T-Z]"
|
||||
fi
|
||||
else
|
||||
if [ "${{ matrix.shard }}" == "1" ]; then
|
||||
TEST_PATTERN="^Test[A-G]"
|
||||
elif [ "${{ matrix.shard }}" == "2" ]; then
|
||||
TEST_PATTERN="^Test[H-R]"
|
||||
else
|
||||
TEST_PATTERN="^Test[S-Z]"
|
||||
fi
|
||||
fi
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (Pattern: ${TEST_PATTERN})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -45,12 +45,12 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.19.1
|
||||
github.com/klauspost/compress v1.19.0
|
||||
github.com/klauspost/reedsolomon v1.14.1
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.8
|
||||
github.com/mailru/easyjson v0.9.2 // indirect
|
||||
github.com/mattn/go-isatty v0.0.23 // indirect
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.22 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/olivere/elastic/v7 v7.0.32
|
||||
@@ -59,14 +59,14 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/prometheus/client_golang v1.24.0
|
||||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/common v0.70.0 // indirect
|
||||
github.com/prometheus/procfs v0.21.1
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.2.0
|
||||
github.com/seaweedfs/raft v1.1.8
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
@@ -92,17 +92,17 @@ require (
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.46.0
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
|
||||
golang.org/x/image v0.44.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/image v0.43.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/tools v0.48.0 // indirect
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.289.0
|
||||
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect
|
||||
google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6
|
||||
google.golang.org/grpc v1.82.1
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
@@ -123,10 +123,10 @@ require (
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.2
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.33
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.32
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.25
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.30
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2
|
||||
github.com/cognusion/imaging v1.0.4
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
@@ -144,11 +144,11 @@ require (
|
||||
github.com/parquet-go/parquet-go v0.30.1
|
||||
github.com/pkg/sftp v1.13.11
|
||||
github.com/rabbitmq/amqp091-go v1.11.0
|
||||
github.com/rclone/rclone v1.75.0
|
||||
github.com/rclone/rclone v1.74.3
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.21.0
|
||||
github.com/schollz/progressbar/v3 v3.19.1
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.4
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.6
|
||||
github.com/tarantool/go-option v1.1.0
|
||||
github.com/tarantool/go-tarantool/v3 v3.0.0
|
||||
@@ -156,7 +156,7 @@ require (
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.146.3
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.144.5
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.12
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.22.0
|
||||
@@ -176,20 +176,19 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v3 v3.4.1 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.1 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.7.0 // indirect
|
||||
github.com/apache/thrift v0.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.2 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.6.0 // indirect
|
||||
github.com/apache/thrift v0.23.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
github.com/buger/jsonparser v1.2.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
|
||||
@@ -204,7 +203,7 @@ require (
|
||||
github.com/containerd/platforms v1.0.0-rc.4 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/dave/dst v0.27.2 // indirect
|
||||
github.com/diskfs/go-diskfs v1.9.4 // indirect
|
||||
github.com/diskfs/go-diskfs v1.7.0 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
@@ -223,7 +222,7 @@ require (
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -235,7 +234,6 @@ require (
|
||||
github.com/lib/pq v1.12.0 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/lpar/calendar v0.2.0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.2.0 // indirect
|
||||
@@ -246,7 +244,6 @@ require (
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/moby/term v0.5.2 // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
@@ -257,16 +254,15 @@ require (
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.83 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.4 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.3 // indirect
|
||||
github.com/rogpeppe/go-internal v1.15.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
github.com/sony/gobreaker/v2 v2.4.0 // indirect
|
||||
github.com/stretchr/objx v0.5.3 // indirect
|
||||
github.com/substrait-io/substrait v0.87.0 // indirect
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.1 // indirect
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.0 // indirect
|
||||
github.com/substrait-io/substrait-protobuf/go v0.85.0 // indirect
|
||||
github.com/twmb/avro v1.7.2 // indirect
|
||||
github.com/twpayne/go-geom v1.6.1 // indirect
|
||||
@@ -285,12 +281,12 @@ require (
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.38.0 // indirect
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.2 // indirect
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
@@ -301,50 +297,53 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.12.0 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.2 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.33 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.33 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.33 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.33 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.2
|
||||
github.com/aws/smithy-go v1.27.5
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.0
|
||||
github.com/aws/smithy-go v1.27.3
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.4 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.16.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
@@ -353,29 +352,29 @@ require (
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/elastic/gosigar v0.14.3 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.3.1 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.8 // indirect
|
||||
github.com/go-openapi/strfmt v0.27.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.3 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/goccy/go-json v0.10.6 // indirect
|
||||
@@ -406,16 +405,17 @@ require (
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/k0kubun/pp v3.0.1+incompatible
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect
|
||||
github.com/mattn/go-colorable v0.1.15 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.24 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.22 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
@@ -430,8 +430,8 @@ require (
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.121.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.12.1 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.5 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
@@ -451,27 +451,26 @@ require (
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/samber/lo v1.53.0 // indirect
|
||||
github.com/samber/lo v1.52.0 // indirect
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.6.4 // indirect
|
||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.3 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
github.com/unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260428144813-1c07baab7f7b // indirect
|
||||
@@ -481,12 +480,12 @@ require (
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.5.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.12 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.44.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.44.0 // indirect
|
||||
@@ -496,8 +495,8 @@ require (
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.45.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
@@ -505,12 +504,12 @@ require (
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260629224719-ba1bff0a7846 // indirect
|
||||
storj.io/drpc v1.0.0 // indirect
|
||||
storj.io/eventkit v0.0.0-20260716074419-6861a92e2aa5 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
storj.io/picobuf v0.0.4 // indirect
|
||||
storj.io/uplink v1.14.3 // indirect
|
||||
storj.io/uplink v1.14.0 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
@@ -6,8 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
|
||||
atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
|
||||
atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
|
||||
atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
|
||||
cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
|
||||
cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
|
||||
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiIS75iTVAKIUxrST5X2dVHMTptL4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||
@@ -591,18 +591,18 @@ github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4b
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194 h1:dtOFxSTWWRpkmvXa6ycNiw8dVDu1wkgzcXyVV1VafNc=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.194/go.mod h1:rl0WumSN9gSo775DgvQv+wMQ8rlb0ES/1hU5jkMtLXg=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 h1:yzIYdwuro811Z27D3T80Wkd3rqZzb0K43nner7Eh1yE=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 h1:2RfP0d2QgkFH64BjZSWd59aMsc28IyWsUuHqU0txBtY=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82/go.mod h1:IPk80dOmc7VFC0DJ85xMTPmre+8xoXX6kGHAkf5jRRw=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0 h1:cSjUzZ7KU8hicTgzaSv9NmSyM9fTVK3y5lsBUl3wOis=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk=
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1 h1:fxLusCG+GlGY1SM7BYAAiiSFhzKUs5zqs0P8knDqeFU=
|
||||
github.com/IBM/go-sdk-core/v5 v5.23.1/go.mod h1:yO+OQpByKDLTvpEcsFFexgzpeR8eRfCFWAYzxkAu4bk=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 h1:mJ5QbLPOm4g5qhZiVB6wbSllfpeUExftGoyPek2hk4M=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2/go.mod h1:ngpMgwkjur1VNUjqn11LPk3o5eCyOCRbcfg/0YAY7Hc=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1/go.mod h1:HbOjEdu/yzCJ/mjTF6wEOJNbAUpHfU2UOA2hVD4CNFg=
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
|
||||
@@ -631,12 +631,14 @@ github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
|
||||
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
|
||||
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
|
||||
github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI=
|
||||
github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk=
|
||||
github.com/ProtonMail/gopenpgp/v3 v3.4.1 h1:K7uUhSHSJxORZ+RuHpilTT6S4MA2whCRlXNwLqd0+ys=
|
||||
github.com/ProtonMail/gopenpgp/v3 v3.4.1/go.mod h1:bGdV9f6edhmd581wzXsQCTKdH8bXBbyhkgDKPjwPc6U=
|
||||
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s=
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/Sereal/Sereal/Go/sereal v0.0.0-20231009093132-b9187f1a92c6/go.mod h1:JwrycNnC8+sZPDyzM3MQ86LvaGzSpfxg885KOOwFRW4=
|
||||
github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
|
||||
github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
|
||||
@@ -673,20 +675,20 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/anchore/go-lzo v0.1.1 h1:IwL/fvkdtlIrYIXck6WxZ3nb8WjjHziYYmGxlooyOnM=
|
||||
github.com/anchore/go-lzo v0.1.1/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk=
|
||||
github.com/anchore/go-lzo v0.1.0 h1:NgAacnzqPeGH49Ky19QKLBZEuFRqtTG9cdaucc3Vncs=
|
||||
github.com/anchore/go-lzo v0.1.0/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk=
|
||||
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
|
||||
github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
|
||||
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
|
||||
github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY=
|
||||
github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A=
|
||||
github.com/apache/arrow-go/v18 v18.6.0 h1:GX/Jyd3R7mCLiECAwY9FWbbaYblie2WXBSz4Sw8fNpM=
|
||||
github.com/apache/arrow-go/v18 v18.6.0/go.mod h1:gm3MiPpY82fLYK5VKPB3WoJbsiLVDfT7flD5/vHReKw=
|
||||
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.2 h1:lu/p0Db2av18enHJvWJQoChLssI0P+AR06STq4VdvCc=
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.2/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
@@ -707,50 +709,50 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.2 h1:cl+IXwWb3qazClUcm08tGSsB6OiuV83JVJO9B0jQcPc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.2/go.mod h1:WEzLKBh/mEjXvx1FtQMWgSxMSTVqxQzjkRtk5fa3wkg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.33 h1:M1m/Q6f0OKDEDGwhiNOqx1OjTdrewe3v+GDbHmKczWk=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.33/go.mod h1:fGj1iQj2QpIZzp7jE4aQQ+71TE8cd4z9K4+xCd6EqmE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.32 h1:eNE0JnIblBo1NCvd3tqEYuZz9XDefn69R74CHd3nT7U=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.32/go.mod h1:yYJu+6tqKUYZuJSYcpSGjz/6sV/SUaAaKIufnWKx2OU=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.33 h1:MobhiR6KIerWxmO74Zit5I3379+mSc2DOdZ3DeRFB9w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.33/go.mod h1:xu02847OdZfNr/jAfZpHtyRk0b3v4d0kaoxNHxZGG/w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.25 h1:ACCejvStYoilgwrfegSt5ZntCbPrk52qfwyNcnl3omM=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.25/go.mod h1:LJyU8sDRbXUxFn8xMJIGP+v9QYYwveNLI8a/giAOiAs=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.33 h1:HAp1wLFZzch054uh3FK7rcVYg4v7J2FxVf3h3IGNZas=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.33/go.mod h1:mJk5fmqnF+WUlMdPG37pR2Fh3oh6r8F6ZGUgPKvzu0c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.33 h1:0YA0aCKgsJyno6xkFfaIgjE3/wK08+Qxo9nQfe1UrWM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.33/go.mod h1:UZqj4WIdTH+ga8Y/DgpAuy/8cGjM3h7gDCliJYGg2SE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.34 h1:HQYnjFnXpX8EbPW5M1QT8mXzesRPwly0HEPTcFlS02Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.34/go.mod h1:tGzj56niKYZBbDIRhwPGDqrULzmWv5b6uBQGqyNaFZw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.14 h1:SA43nfaY7+1jjMNIc2ywu99JLJLButtIdLP6j+bT870=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.14/go.mod h1:Du3llKcwbQvHsTXSLzTOGQz0DTDBMEzdg7DAGu7inrY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.33 h1:mqI7OrxN/DUH85F5OqVn3cIfuZ3+HVcebUm2N8mLlgQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.33/go.mod h1:eZ5jdEpvaaOU8nWWE4cTAJETSEA5FZoWxvNRao4piHY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2 h1:5C00eQYpTrgQXnp6V3P6P7zPElna3AXvlukbANE6nJI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.2 h1:EjI1CZzDcBxPkTa3j1BdtIrUDbqnOGssFMeyUS+6W0I=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.2/go.mod h1:vN3eb5H8MEAZ4dx0F5Wc9LT8eb3eW7bZZ5BjGJdbw9k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 h1:p8WdWDh5AwSZdp19Haa3XMyPCICi9Z375a/Nu3IIEZY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14/go.mod h1:NKVY7DER6VXHkt2I/ycmHakALNboi3Rqwt4eEf/1Cnk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 h1:JP2wjWGmUp8lTCZb13Dv0Eciyc1jbO8pd0HZVMHFlrc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24/go.mod h1:Ql9ziDutk8ERAN9HMaYANCW3lop451ppebkxEJMLCTM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.2 h1:zMP1FDFE08L7sM5f1QqkH/ZgKKg8Uc0Dz7KhSSYqWkw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.2/go.mod h1:0LoIZSUKjdo2BleHfT1hv/jlD33LQS00IrBlzoUsoUQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.2 h1:9eTqUYl+SyVmaRPMyBXSO9wwqC6TRwZB82pKENK2hdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.2/go.mod h1:DThweuz22kiLc7lGHop5vQ9c3bx5W6Azs/YqSHa2fu8=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.2 h1:EJd8vZO3E8SE6nmPqxuxlQ1NeSb8as50sf6eGdV4Saw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.2/go.mod h1:OgpPvKzsO2Ranjpli/20djMkg6UrV5mw4W3pZpq1Mqo=
|
||||
github.com/aws/smithy-go v1.27.5 h1:d1ro7KpYOYwP6m73YFa+Kc/A130VsAdX68SpsJwARMM=
|
||||
github.com/aws/smithy-go v1.27.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg=
|
||||
github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY=
|
||||
github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -773,6 +775,8 @@ github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVk
|
||||
github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
|
||||
github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8=
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og=
|
||||
github.com/broady/gogeohash v0.0.0-20120525094510-7b2c40d64042 h1:iEdmkrNMLXbM7ecffOAtZJQOQUTE4iMonxrb5opUgE4=
|
||||
github.com/broady/gogeohash v0.0.0-20120525094510-7b2c40d64042/go.mod h1:f1L9YvXvlt9JTa+A17trQjSMM6bV40f+tHjB+Pi+Fqk=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
@@ -785,8 +789,8 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
|
||||
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
|
||||
github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY=
|
||||
github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
|
||||
github.com/buger/jsonparser v1.2.0 h1:4EFcvK1kD4jyj6YqNK6skK6w+y7FHHBR+XBCtxwu/6g=
|
||||
github.com/buger/jsonparser v1.2.0/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
||||
github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
|
||||
github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
||||
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
|
||||
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
|
||||
@@ -821,12 +825,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
|
||||
github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U=
|
||||
github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.16.0 h1:0irPbKwRB6V6sdP9+a4P4D5sRUYHXriG5GfTjVq4YBI=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.16.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8 h1:+KC2I+emnT6ccWmo7IHbtchGc/vFqAxUuovo2eUgMuM=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20260701064823-d8c5600d73b8/go.mod h1:1pQXB0vAlzRlqcY7LYKOOZMw0wKfJPFxTLsJRF2Gswo=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 h1:iLoIwb7BJECHTbNcmIhYDsQhoZiACWGNvEpyqQy97Dk=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E=
|
||||
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
|
||||
@@ -918,8 +922,8 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/diskfs/go-diskfs v1.9.4 h1:0j2d7eG4IjyxL6+ChWbDPocdBCF6HQ4HBWU2WDYWVnc=
|
||||
github.com/diskfs/go-diskfs v1.9.4/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=
|
||||
github.com/diskfs/go-diskfs v1.7.0 h1:vonWmt5CMowXwUc79jWyGrf2DIMeoOjkLlMnQYGVOs8=
|
||||
github.com/diskfs/go-diskfs v1.7.0/go.mod h1:LhQyXqOugWFRahYUSw47NyZJPezFzB9UELwhpszLP/k=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
|
||||
@@ -943,8 +947,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
|
||||
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
|
||||
github.com/dromara/dongle v1.0.1 h1:si/7UP/EXxnFVZok1cNos70GiMGxInAYMilHQFP5dJs=
|
||||
github.com/dromara/dongle v1.0.1/go.mod h1:ebFhTaDgxaDIKppycENTWlBsxz8mWCPWOLnsEgDpMv4=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0 h1:OYMx56y2as2FIM6QuS5HGXC2AYtf5xv/2n6INaluF4Y=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.4.0/go.mod h1:gDXhl0OElhzYoDsYWHr1RXjpxjGeLzEvjYzH7sZV73k=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
|
||||
github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
|
||||
github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
@@ -956,18 +960,18 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
|
||||
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJtLmY22n99HaZTz+r2Z51xUPi01m3wg=
|
||||
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
|
||||
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
|
||||
github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
|
||||
github.com/elliotwutingfeng/asciiset v0.0.0-20260129054604-cfde2086bc57 h1:x5yxNrq8XffV/OoNUeFPM6hxHVi5OTspSTBxr/9pemg=
|
||||
github.com/elliotwutingfeng/asciiset v0.0.0-20260129054604-cfde2086bc57/go.mod h1:GLo/8fDswSAniFG+BFIaiSPcK610jyzgEhWYPQwuQdw=
|
||||
github.com/elliotwutingfeng/asciiset v0.0.0-20230602022725-51bbb787efab h1:h1UgjJdAAhj+uPL68n7XASS6bU+07ZX1WJvVS2eyoeY=
|
||||
github.com/elliotwutingfeng/asciiset v0.0.0-20230602022725-51bbb787efab/go.mod h1:GLo/8fDswSAniFG+BFIaiSPcK610jyzgEhWYPQwuQdw=
|
||||
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
||||
github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
|
||||
github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 h1:B9YK+Tck5mTccyDhtxBzWyqGYcFxLyB6+noMNW4/VgI=
|
||||
github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg=
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
|
||||
github.com/emmansun/gmsm v0.15.5/go.mod h1:2m4jygryohSWkaSduFErgCwQKab5BNjURoFrn2DNwyU=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
@@ -1007,8 +1011,8 @@ github.com/fanixk/geohash v0.0.0-20150324002647-c1f9b5fa157a/go.mod h1:UgNw+PTmm
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fluent/fluent-logger-golang v1.10.1 h1:wu54iN1O2afll5oQrtTjhgZRwWcfOeFFzwRsEkABfFQ=
|
||||
github.com/fluent/fluent-logger-golang v1.10.1/go.mod h1:qOuXG4ZMrXaSTk12ua+uAb21xfNYOzn0roAtp7mfGAE=
|
||||
github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
|
||||
@@ -1040,8 +1044,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
|
||||
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8=
|
||||
github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
@@ -1076,12 +1080,12 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I=
|
||||
github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w=
|
||||
github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM=
|
||||
github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc=
|
||||
github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
|
||||
github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
|
||||
github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
|
||||
github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ=
|
||||
github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8=
|
||||
github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls=
|
||||
github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54=
|
||||
github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
|
||||
github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
@@ -1090,8 +1094,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8=
|
||||
github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc=
|
||||
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
|
||||
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
|
||||
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
|
||||
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
|
||||
@@ -1202,6 +1206,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo=
|
||||
@@ -1368,8 +1373,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2 h1:ZeebPK9Bnpy40uTuneEeVMYaMMerol3qsmIvzD0EIAk=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2/go.mod h1:4jGLEnNHyWOVSGn89IeWUVqlKCEitOM3D32/XypGy/o=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 h1:PV6ipOJN0JIek4dqPqsTtenQmjI1SZntCUgPzhfk9gM=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160/go.mod h1:yRuPDnHgGmsbvopF0amMqXxr4n32GynzX5GTwFYDHaw=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -1446,13 +1451,13 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
||||
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
|
||||
github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
|
||||
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.14.1 h1:swE9kzyWXD/wVG+l5Pe8bWnQ0giIY7D1GjCBKk3kG2U=
|
||||
github.com/klauspost/reedsolomon v1.14.1/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
@@ -1492,31 +1497,31 @@ github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8
|
||||
github.com/lithammer/fuzzysearch v1.1.8/go.mod h1:IdqeyBClc3FFqSzYq/MXESsS4S0FsZ5ajtkr5xPLts4=
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8=
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
|
||||
github.com/lpar/calendar v0.2.0 h1:A1kxv6sbvBHFUkd2XotanIRqEXQGreQOeuGhkJqIaRA=
|
||||
github.com/lpar/calendar v0.2.0/go.mod h1:fsVJa4o2NvXYzaCE5RXMadqbZzWEhNCGOUPVHTIZdjc=
|
||||
github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 h1:YkjVPl/YH5XlJ+/NiwzJtPYXXKRcyjmEUhsDci6YK3c=
|
||||
github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
|
||||
github.com/lpar/date v1.0.0/go.mod h1:KjYe0dDyMQTgpqcUz4LEIeM5VZwhggjVx/V2dtc8NSo=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mailru/easyjson v0.9.2 h1:dX8U45hQsZpxd80nLvDGihsQ/OxlvTkVUXH2r/8cb2M=
|
||||
github.com/mailru/easyjson v0.9.2/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
||||
github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8=
|
||||
github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
||||
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ=
|
||||
github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
|
||||
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
|
||||
github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk=
|
||||
github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
|
||||
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
@@ -1604,8 +1609,6 @@ github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
|
||||
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s=
|
||||
github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
|
||||
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@@ -1626,12 +1629,12 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
|
||||
github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.121.0 h1:1J+5ARgrodrx8kzFy/hxznaoUzz43jr0EestCzEaOHw=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.121.0/go.mod h1:Pzy+BpgkDesvGZXEHgslwhIYobHCPHg6wRta1mWnlqQ=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 h1:eDkWg6ZN0uKwWzSekoFcQJhR+C+F/aVdTwr+lGHU9Qk=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0/go.mod h1:8ZzvzuEG/cFLFZhxg/Mg1w19KqyXBKO3c17QIc5PkGs=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
||||
github.com/panjf2000/ants/v2 v2.12.1 h1:BWvU2wHpyXWxhhNXsGB6JXLCNbshyLd1QxvoAmZnu10=
|
||||
github.com/panjf2000/ants/v2 v2.12.1/go.mod h1:tSQuaNQ6r6NRhPt+IZVUevvDyFMTs+eS4ztZc52uJTY=
|
||||
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
|
||||
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
|
||||
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
|
||||
github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
|
||||
@@ -1642,7 +1645,6 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=
|
||||
@@ -1710,8 +1712,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
|
||||
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
|
||||
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
|
||||
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI=
|
||||
github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
@@ -1723,8 +1725,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI=
|
||||
github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
@@ -1751,12 +1753,12 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.4 h1:uGQJRjQC1hVLd5kqLsXc6CWO6oqrVeLoKQYoHapEZDg=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.4/go.mod h1:VTPBYZotKAeDLlAzxU2O/s14NXk9FxUt9hn1jhH2iY8=
|
||||
github.com/rclone/go-proton-api v1.0.3 h1:3gBTzR+j0dYiTwtj9yKIdN/aV3W2a8KIPKp0GArojyQ=
|
||||
github.com/rclone/go-proton-api v1.0.3/go.mod h1:QAlkFfswzrBuxvCORWV8rZdddg52hahMN98CFWoFW1E=
|
||||
github.com/rclone/rclone v1.75.0 h1:3ARHem4jXWltvl+b0PvDAG8s6J/inHd5BRzfwMRb3W8=
|
||||
github.com/rclone/rclone v1.75.0/go.mod h1:PGLJUW/WSIJCysALqUcxmaCFyfMXUevf8CbuoOwsAdU=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
|
||||
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
|
||||
github.com/rclone/go-proton-api v1.0.2/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.74.3 h1:a2wln7pvEa0tS1WIZJKulEkVjxgC1DkCoyxYydkdiSY=
|
||||
github.com/rclone/rclone v1.74.3/go.mod h1:t5Mh86PO49DD7xlPt0trnK/aNf2Z3M0uip4l1Jqwiv8=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
@@ -1784,8 +1786,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
|
||||
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
||||
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
|
||||
@@ -1796,8 +1798,8 @@ github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDj
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM=
|
||||
github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
||||
github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
|
||||
github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0=
|
||||
@@ -1806,12 +1808,12 @@ github.com/schollz/progressbar/v3 v3.19.1 h1:iv8BgwOvdML/S3p84uBpy/IMigv4U9594vP
|
||||
github.com/schollz/progressbar/v3 v3.19.1/go.mod h1:LFL7jqimKxfhero4K1eCkUr/6R39AgQeiPCJtlTWIW8=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.4 h1:ACyloiuopdhRSjdLLeSWbsVaemMPskORaRF01TY6GyM=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.4/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3 h1:rJufGrHImTx7yoGUmetUi+To4LrmTQJROJnBHWt92ic=
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/seaweedfs/raft v1.2.0 h1:Ez4Hw9ifBbTT7wg54DvGHBjw1vRlTb4roH0TKl0Oj9Y=
|
||||
github.com/seaweedfs/raft v1.2.0/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/seaweedfs/raft v1.1.8 h1:a5f+wSnriRjINLqLqMY7DDNNfU1+kv6TKBuOZa0KDwU=
|
||||
github.com/seaweedfs/raft v1.1.8/go.mod h1:fgs/rAVEzjQ7e04XMzG3eJhwZZRmBW+2uRtjakeCGeU=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
@@ -1843,8 +1845,6 @@ github.com/snabb/httpreaderat v1.0.1 h1:whlb+vuZmyjqVop8x1EKOg05l2NE4z9lsMMXjmSU
|
||||
github.com/snabb/httpreaderat v1.0.1/go.mod h1:lpbGrKDWF37yvRbtRvQsbesS6Ty5c83t8ztannPoMsA=
|
||||
github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ=
|
||||
github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
|
||||
github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg=
|
||||
github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 h1:GyYC5Ntqk/yy9lEIGE7chdIvt4zP44taycwd9YDSGdc=
|
||||
@@ -1863,8 +1863,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E=
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.1/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo=
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
@@ -1894,14 +1894,14 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/substrait-io/substrait v0.87.0 h1:40rP4LejyK6SNQlWz7NX6kQELf8cmScWMBGruWhN4io=
|
||||
github.com/substrait-io/substrait v0.87.0/go.mod h1:MPFNw6sToJgpD5Z2rj0rQrdP/Oq8HG7Z2t3CAEHtkHw=
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.1 h1:XL7CqVVOQkXP9FRSIm2ylJCjqRASg1yBRcn5nRCRwss=
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.1/go.mod h1:6GLz9k21udB64g4lLKq8632TKfQCRAVfhuU3NSXtZWY=
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.0 h1:58nSM6Qsctke4DIHU2y4aR8FN5YC7syo1mk4zFKAo4U=
|
||||
github.com/substrait-io/substrait-go/v8 v8.1.0/go.mod h1:6GLz9k21udB64g4lLKq8632TKfQCRAVfhuU3NSXtZWY=
|
||||
github.com/substrait-io/substrait-protobuf/go v0.85.0 h1:zk6MtNWLtDSl8a7qCZRFH0+EIIXVrrd/hsgYK/SQTgM=
|
||||
github.com/substrait-io/substrait-protobuf/go v0.85.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 h1:s30A8dMuZ55lUOi5xUTh1hlfLqHpVBsRMfpk2iKCayk=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c h1:dtcOwRimeiBFrlutmF6K94l0rxYFARNFMA+lSQ41C+M=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/tarantool/go-iproto v1.1.0 h1:HULVOIHsiehI+FnHfM7wMDntuzUddO09DKqu2WnFQ5A=
|
||||
github.com/tarantool/go-iproto v1.1.0/go.mod h1:LNCtdyZxojUed8SbOiYHoc3v9NvaZTB7p96hUySMlIo=
|
||||
@@ -1931,12 +1931,12 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GR
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA=
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g=
|
||||
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
|
||||
github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ=
|
||||
github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU=
|
||||
github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI=
|
||||
github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4=
|
||||
github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg=
|
||||
github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s=
|
||||
github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
|
||||
github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323 h1:r0p7fK56l8WPequOaR3i9LBqfPtEdXIQbUTzT55iqT4=
|
||||
@@ -1998,8 +1998,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286 h1:tw5GqRXqExB/xghPoPLtVujBe9w9Pg1G78tvXCJNJAA=
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
@@ -2036,8 +2034,8 @@ github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2 h1:e2nGQPGC5OEPBWlMnLPFpd
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.146.3 h1:0ybsLjWJ25M8XXGXk7NfjhO+s8bqBcGGWt8iB2DQiHQ=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.146.3/go.mod h1:b9NEO6mgaiqsnOMkS003uS82XsKh6GL+ZTFfPqXWz+c=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.144.5 h1:FRDqbsrIUBjhgP0laLYiw+22kGT0JqPdeDY5Yejo6SE=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.144.5/go.mod h1:b9NEO6mgaiqsnOMkS003uS82XsKh6GL+ZTFfPqXWz+c=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
|
||||
@@ -2071,8 +2069,8 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||
go.einride.tech/aip v0.83.0 h1:TI21IdeOnLTwZEJ3BxtImIZk6bsN2Q+sd0x99SLiQ+M=
|
||||
go.einride.tech/aip v0.83.0/go.mod h1:E8+wdTApA70odnpFzJgsGogHozC2JCIhFJBKPr8bVig=
|
||||
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
|
||||
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
|
||||
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
|
||||
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
|
||||
go.etcd.io/etcd/api/v3 v3.6.12 h1:OLOZUKEuAA36TR48F0cIaa8FdzrWygjyfrJxXg4iDgs=
|
||||
go.etcd.io/etcd/api/v3 v3.6.12/go.mod h1:p14EIQXHbuOQbVvL/WEes5uqKnxP9AgKJgpjbMVvzvE=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.12 h1:36zzB+pQOdHbhN+kH2iJz/K8bJn0ZLtLfPPO7jozTDo=
|
||||
@@ -2092,14 +2090,14 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0 h1:2pn7OzMewmYRiNtv1doZnLo3gONcnMHlFnmOR8Vgt+8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0/go.mod h1:rjbQTDEPQymPE0YnRQp9/NuPwwtL0sesz/fnqRW/v84=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
|
||||
go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk=
|
||||
@@ -2182,6 +2180,9 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -2199,8 +2200,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g=
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
@@ -2214,8 +2215,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I=
|
||||
golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY=
|
||||
golang.org/x/image v0.43.0 h1:FLxcP4ec2350nTfOC8ysKtqYSIFbk/QGjw1ZHNP4tsY=
|
||||
golang.org/x/image v0.43.0/go.mod h1:rrpelvGFt+kLPAjPM4HeWPgrl0FtafueU//e5N0qk/Q=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2247,8 +2248,10 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2313,6 +2316,9 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@@ -2363,6 +2369,9 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -2466,8 +2475,12 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2480,6 +2493,9 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -2500,6 +2516,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -2578,8 +2596,9 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -2794,10 +2813,10 @@ google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 h1:YJjbgu+dkp5kUJLfpMyCLfBIWZb/FcJyuLeo1gVBOuo=
|
||||
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800 h1:admdQBe8jR3VWhBsUrAOaF2Qw6K/+p5pSm1GN8+6Fw4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260706201446-f0a921348800/go.mod h1:FPk7EXUKMtImne7AmknoYjT4QXqKIzzRbeQIXzLk6fQ=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d h1:Jkpk39hlTZOIp3RbfvNX9R8Hv+Sw0X89nlU/xFOErsc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260715232425-e75dac1f907d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -2838,8 +2857,8 @@ google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsA
|
||||
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
|
||||
google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
|
||||
google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
|
||||
google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6 h1:HfjjkdGIa8u9sP9EW5WCygy0kQDuTI/Tax4j//t24Fo=
|
||||
google.golang.org/grpc v1.84.0-dev.0.20260723093437-b6eac429d7b6/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0=
|
||||
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
|
||||
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
|
||||
@@ -2873,8 +2892,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw=
|
||||
gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
@@ -2984,17 +3003,17 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
|
||||
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
|
||||
storj.io/common v0.0.0-20260629224719-ba1bff0a7846 h1:TRtVGWm/Y4KAi4coWMxVpydPvBQKYVhkKMar7wu9Zow=
|
||||
storj.io/common v0.0.0-20260629224719-ba1bff0a7846/go.mod h1:1GZnCZNGbzBzaqhG0cUypQeLfNhNIeNY9JVMMiCF14M=
|
||||
storj.io/drpc v1.0.0 h1:1Xf1KCXXbV1viIfN56eqdJ3cNwpAL7OKwQkpS6ksing=
|
||||
storj.io/drpc v1.0.0/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
|
||||
storj.io/eventkit v0.0.0-20260716074419-6861a92e2aa5 h1:JPguPTktHaha9TBEHvuxZmB+mCkiMPgEMOLfosDoE9o=
|
||||
storj.io/eventkit v0.0.0-20260716074419-6861a92e2aa5/go.mod h1:RdWvp249AGm9FZnVKP88y3hFa7vlx9Jf3Cy1P+4EXIU=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a h1:7gSBQY3vhQMIqi3vfaEXR7mreRjcBLfVsYY0rHIN7P0=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a/go.mod h1:kyxwKwlfH4paBZCZt/szvRB770ieAIJF73iDy2DpEHw=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 h1:8OE12DvUnB9lfZcHe7IDGsuhjrY9GBAr964PVHmhsro=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 h1:5MZ0CyMbG6Pi0rRzUWVG6dvpXjbBYEX2oyXuj+tT+sk=
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156/go.mod h1:CpnM6kfZV58dcq3lpbo/IQ4/KoutarnTSHY0GYVwnYw=
|
||||
storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q=
|
||||
storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs=
|
||||
storj.io/picobuf v0.0.4 h1:qswHDla+YZ2TovGtMnU4astjvrADSIz84FXRn0qgP6o=
|
||||
storj.io/picobuf v0.0.4/go.mod h1:hSMxmZc58MS/2qSLy1I0idovlO7+6K47wIGUyRZa6mg=
|
||||
storj.io/uplink v1.14.3 h1:b89nziD1JgiF6oya+kDSBsqWp9Dzp5ADCUdhA2e2zvs=
|
||||
storj.io/uplink v1.14.3/go.mod h1:jAe47qR+gRnHW2+3THe4cWJBAj8ARZv4P9mguJRo7C4=
|
||||
storj.io/uplink v1.14.0 h1:J1yXlt0aRr6kgLTHWXOWosNCFVfbamlcyd+CSxyIczo=
|
||||
storj.io/uplink v1.14.0/go.mod h1:2ysmjzd/1Xtz4VKoErNcSqBQz3UC9WKTVuLMV1cNu6E=
|
||||
tags.cncf.io/container-device-interface v1.1.0 h1:RnxNhxF1JOu6CJUVpetTYvrXHdxw9j9jFYgZpI+anSY=
|
||||
tags.cncf.io/container-device-interface v1.1.0/go.mod h1:76Oj0Yqp9FwTx/pySDc8Bxjpg+VqXfDb50cKAXVJ34Q=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.41"
|
||||
appVersion: "4.40"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.41.0
|
||||
version: 4.40.0
|
||||
|
||||
@@ -404,7 +404,7 @@ Two things worth knowing before you turn this on:
|
||||
- **Monitoring stops.** The metrics ports are admitted from release pods like every other port, so with `global.seaweedfs.monitoring.enabled` the ServiceMonitors keep scraping targets a Prometheus in another namespace can no longer reach. Nothing reports it; add the scraper's namespace to `extraIngress`.
|
||||
- **The resize hook's policy is a Helm hook.** Its Job runs before the release manifest is applied, so the policy has to be a `pre-install` hook too. Helm does not garbage-collect hook resources, so on an upgrade that grows a volume PVC the policy is created and then left behind on uninstall - delete `<release>-seaweedfs-volume-resize-hook` by hand if it bothers you.
|
||||
|
||||
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
|
||||
The DNS selectors default to CoreDNS as kubeadm, kind, EKS, GKE and AKS install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
|
||||
|
||||
## OpenShift Support
|
||||
|
||||
|
||||
@@ -356,7 +356,6 @@ spec:
|
||||
- name: db-schema-config-volume
|
||||
configMap:
|
||||
name: {{ default (printf "%s-db-init-config" (include "seaweedfs.fullname" .)) .Values.filer.dbInitConfigName }}
|
||||
optional: true
|
||||
{{- if and .Values.filer.s3.enabled .Values.filer.s3.enableAuth }}
|
||||
- name: config-users
|
||||
secret:
|
||||
|
||||
@@ -12,10 +12,6 @@
|
||||
{{- end }}
|
||||
{{- $bucketsFolder = default $bucketsFolder (get $bucketEnvVars "WEED_FILER_BUCKETS_FOLDER") }}
|
||||
{{- $bucketsFolder = trimSuffix "/" $bucketsFolder }}
|
||||
{{- $clusterAlias := default "sw" (get $bucketEnvVars "WEED_CLUSTER_DEFAULT") }}
|
||||
{{- $clusterUpper := upper $clusterAlias }}
|
||||
{{- $clusterMasterKey := printf "WEED_CLUSTER_%s_MASTER" $clusterUpper }}
|
||||
{{- $clusterFilerKey := printf "WEED_CLUSTER_%s_FILER" $clusterUpper }}
|
||||
|
||||
{{- /* Check allInOne mode first */}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
@@ -80,6 +76,12 @@ spec:
|
||||
image: {{ template "seaweedfs.master.image" . }}
|
||||
imagePullPolicy: {{ $.Values.global.seaweedfs.imagePullPolicy | default "IfNotPresent" }}
|
||||
env:
|
||||
- name: WEED_CLUSTER_DEFAULT
|
||||
value: "sw"
|
||||
- name: WEED_CLUSTER_SW_MASTER
|
||||
value: {{ include "seaweedfs.cluster.masterAddress" . | quote }}
|
||||
- name: WEED_CLUSTER_SW_FILER
|
||||
value: {{ include "seaweedfs.cluster.filerAddress" . | quote }}
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
@@ -94,28 +96,6 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- /* Carries the JWT signing keys when they are set as environment
|
||||
overrides rather than in security.toml. */}}
|
||||
{{- range $key := keys $bucketEnvVars | sortAlpha }}
|
||||
{{- $value := index $bucketEnvVars $key }}
|
||||
{{- if not (has $key (list "WEED_CLUSTER_DEFAULT" $clusterMasterKey $clusterFilerKey)) }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 14 | trim }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- /* Computed, as in all-in-one: the values pick the cluster alias,
|
||||
the chart still owns the endpoints the script waits on. */}}
|
||||
- name: WEED_CLUSTER_DEFAULT
|
||||
value: {{ $clusterAlias | quote }}
|
||||
- name: {{ $clusterMasterKey }}
|
||||
value: {{ include "seaweedfs.cluster.masterAddress" . | quote }}
|
||||
- name: {{ $clusterFilerKey }}
|
||||
value: {{ include "seaweedfs.cluster.filerAddress" . | quote }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -140,11 +120,11 @@ spec:
|
||||
exit 1
|
||||
}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
wait_for_service "http://${{ $clusterMasterKey }}{{ .Values.allInOne.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://${{ $clusterFilerKey }}{{ .Values.filer.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.allInOne.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
|
||||
{{- else }}
|
||||
wait_for_service "http://${{ $clusterMasterKey }}{{ .Values.master.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://${{ $clusterFilerKey }}{{ .Values.filer.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
{{- $bucketName := .name }}
|
||||
@@ -202,19 +182,11 @@ spec:
|
||||
/usr/bin/weed shell
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if or $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
|
||||
{{- if $enableAuth }}
|
||||
volumeMounts:
|
||||
{{- if $enableAuth }}
|
||||
- name: config-users
|
||||
mountPath: /etc/sw
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
readOnly: true
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.master.port }}
|
||||
@@ -232,9 +204,8 @@ spec:
|
||||
{{- if .Values.filer.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
|
||||
{{- if $enableAuth }}
|
||||
volumes:
|
||||
{{- if $enableAuth }}
|
||||
- name: config-users
|
||||
secret:
|
||||
defaultMode: 420
|
||||
@@ -243,11 +214,5 @@ spec:
|
||||
{{- else }}
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-s3-secret
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -1881,9 +1881,8 @@ networkPolicy:
|
||||
# Every component resolves its peers by DNS name, so this is required
|
||||
# for the release to function at all.
|
||||
#
|
||||
# The defaults below are CoreDNS as kubeadm, kind and the managed offerings
|
||||
# from AWS, Google and Azure install it. OpenShift runs its resolver
|
||||
# elsewhere and needs both overridden:
|
||||
# The defaults below are CoreDNS as kubeadm, kind, EKS, GKE and AKS install
|
||||
# it. OpenShift runs its resolver elsewhere and needs both overridden:
|
||||
# dnsNamespaceSelector:
|
||||
# matchLabels:
|
||||
# kubernetes.io/metadata.name: openshift-dns
|
||||
|
||||
@@ -761,8 +761,7 @@ message FilerConf {
|
||||
string data_node = 11;
|
||||
uint32 max_file_name_length = 12;
|
||||
bool disable_chunk_deletion = 13;
|
||||
// unset inherits from the enclosing path rule, set overrides it
|
||||
optional bool worm = 14;
|
||||
bool worm = 14;
|
||||
uint64 worm_grace_period_seconds = 15;
|
||||
uint64 worm_retention_time_seconds = 16;
|
||||
}
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use tracing::warn;
|
||||
|
||||
@@ -122,8 +122,6 @@ impl DiskLocation {
|
||||
// Scan for .dat files
|
||||
let entries = fs::read_dir(&self.directory)?;
|
||||
let mut dat_files: Vec<(String, VolumeId)> = Vec::new();
|
||||
let mut to_load: Vec<(VolumeId, Vec<String>)> = Vec::new();
|
||||
let mut queued: HashMap<VolumeId, usize> = HashMap::new();
|
||||
let mut seen = HashSet::new();
|
||||
|
||||
for entry in entries {
|
||||
@@ -203,7 +201,6 @@ impl DiskLocation {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// Load existing data only; never create a phantom `.dat`. A lone
|
||||
// `.vif`/`.idx` (e.g. an EC sidecar whose `.ecx` is on a sibling
|
||||
// disk) would otherwise have Volume::new write an 8-byte stub that
|
||||
@@ -224,22 +221,30 @@ impl DiskLocation {
|
||||
continue;
|
||||
}
|
||||
|
||||
match queued.get(&vid) {
|
||||
Some(&i) => to_load[i].1.push(collection),
|
||||
None => {
|
||||
queued.insert(vid, to_load.len());
|
||||
to_load.push((vid, vec![collection]));
|
||||
match Volume::new(
|
||||
&self.directory,
|
||||
&self.idx_directory,
|
||||
&collection,
|
||||
vid,
|
||||
needle_map_kind,
|
||||
None, // replica placement read from superblock
|
||||
None, // TTL read from superblock
|
||||
0, // no preallocate on load
|
||||
Version::current(),
|
||||
) {
|
||||
Ok(mut v) => {
|
||||
v.location_disk_space_low = self.is_disk_space_low.clone();
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&collection, "volume"])
|
||||
.inc();
|
||||
self.volumes.insert(vid, v);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(volume_id = vid.0, error = %e, "failed to load volume");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (collection, vid, v) in self.open_volumes(to_load, needle_map_kind) {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&collection, "volume"])
|
||||
.inc();
|
||||
self.volumes.insert(vid, v);
|
||||
}
|
||||
|
||||
// After regular volumes, auto-discover EC shards on disk so a
|
||||
// fresh restart picks up shards without an explicit
|
||||
// VolumeEcShardsMount RPC. Mirrors Go's loadExistingVolumes
|
||||
@@ -252,60 +257,6 @@ impl DiskLocation {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Opening a volume is dominated by reading its .idx, so they load on
|
||||
/// max(cores, 10) workers, as Go's concurrentLoadingVolumes does. An id is
|
||||
/// only spoken for once one of its candidates opens.
|
||||
fn open_volumes(
|
||||
&self,
|
||||
to_load: Vec<(VolumeId, Vec<String>)>,
|
||||
needle_map_kind: NeedleMapKind,
|
||||
) -> Vec<(String, VolumeId, Volume)> {
|
||||
if to_load.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let workers = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(1)
|
||||
.max(10)
|
||||
.min(to_load.len());
|
||||
|
||||
let next = AtomicUsize::new(0);
|
||||
let opened = Mutex::new(Vec::with_capacity(to_load.len()));
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..workers {
|
||||
scope.spawn(|| loop {
|
||||
let i = next.fetch_add(1, Ordering::Relaxed);
|
||||
let Some((vid, collections)) = to_load.get(i) else {
|
||||
return;
|
||||
};
|
||||
for collection in collections {
|
||||
match Volume::new(
|
||||
&self.directory,
|
||||
&self.idx_directory,
|
||||
collection,
|
||||
*vid,
|
||||
needle_map_kind,
|
||||
None, // replica placement read from superblock
|
||||
None, // TTL read from superblock
|
||||
0, // no preallocate on load
|
||||
Version::current(),
|
||||
) {
|
||||
Ok(mut v) => {
|
||||
v.location_disk_space_low = self.is_disk_space_low.clone();
|
||||
opened.lock().unwrap().push((collection.clone(), *vid, v));
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(volume_id = vid.0, error = %e, "failed to load volume");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
opened.into_inner().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// Directory pre-pass that recovers interrupted compaction commits. Collects
|
||||
/// every volume id that still has a .cpc commit marker or a leftover
|
||||
/// .cpd/.cpx temp file across the data and idx directories, then rolls the
|
||||
@@ -1542,58 +1493,6 @@ mod tests {
|
||||
assert!(ids.contains(&VolumeId(2)));
|
||||
}
|
||||
|
||||
// Two collections can name the same volume id on one disk.
|
||||
#[test]
|
||||
fn test_open_volumes_falls_back_past_a_corrupt_candidate() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
{
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
loc.create_volume(
|
||||
VolumeId(9),
|
||||
"good",
|
||||
NeedleMapKind::InMemory,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
Version::current(),
|
||||
)
|
||||
.unwrap();
|
||||
loc.close();
|
||||
}
|
||||
|
||||
let mut bad = vec![0u8; 16];
|
||||
bad[0] = 9; // unsupported version
|
||||
std::fs::write(format!("{}/bad_9.dat", dir), &bad).unwrap();
|
||||
|
||||
let loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let opened = loc.open_volumes(
|
||||
vec![(VolumeId(9), vec!["bad".to_string(), "good".to_string()])],
|
||||
NeedleMapKind::InMemory,
|
||||
);
|
||||
|
||||
assert_eq!(opened.len(), 1, "the good candidate should still open");
|
||||
assert_eq!(opened[0].0, "good");
|
||||
assert_eq!(opened[0].1, VolumeId(9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disk_location_delete_volume() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
@@ -1970,12 +1970,11 @@ impl Volume {
|
||||
}
|
||||
|
||||
/// Verify the live needle at the .dat tail: its header must match the
|
||||
/// .idx entry and, on v3, the .dat must end exactly at its on-disk end.
|
||||
/// Extra bytes mean an unindexed trailing record (a torn append);
|
||||
/// appending after one would place the next needle at an offset the
|
||||
/// 8-byte .idx encoding may not represent, so the volume is quarantined
|
||||
/// read-only instead. Mirrors Go's verifyNeedleIntegrity, including its
|
||||
/// v3-only tail check.
|
||||
/// .idx entry and the .dat must end exactly at its on-disk end. Extra
|
||||
/// bytes mean an unindexed trailing record (a torn append); appending
|
||||
/// after one would place the next needle at an offset the 8-byte .idx
|
||||
/// encoding may not represent, so the volume is quarantined read-only
|
||||
/// instead. Mirrors Go's verifyNeedleIntegrity.
|
||||
fn verify_needle_integrity(
|
||||
&mut self,
|
||||
actual_offset: i64,
|
||||
@@ -2014,16 +2013,15 @@ impl Volume {
|
||||
)));
|
||||
}
|
||||
|
||||
if version != VERSION_3 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let ts_offset = checked_offset as u64 + NEEDLE_HEADER_SIZE as u64 + size.0 as u64 + 4; // skip checksum
|
||||
let mut ts_buf = [0u8; 8];
|
||||
self.read_exact_at_backend(&mut ts_buf, ts_offset)?;
|
||||
let ts = u64::from_be_bytes(ts_buf);
|
||||
if ts > 0 {
|
||||
self.last_append_at_ns = ts;
|
||||
if version == VERSION_3 {
|
||||
let ts_offset =
|
||||
checked_offset as u64 + NEEDLE_HEADER_SIZE as u64 + size.0 as u64 + 4; // skip checksum
|
||||
let mut ts_buf = [0u8; 8];
|
||||
self.read_exact_at_backend(&mut ts_buf, ts_offset)?;
|
||||
let ts = u64::from_be_bytes(ts_buf);
|
||||
if ts > 0 {
|
||||
self.last_append_at_ns = ts;
|
||||
}
|
||||
}
|
||||
|
||||
self.verify_dat_ends_at(checked_offset + get_actual_size(size, version))
|
||||
@@ -2741,29 +2739,6 @@ impl Volume {
|
||||
return Err(VolumeError::ReadOnly);
|
||||
}
|
||||
|
||||
// size indexes the needle and places the v3 append timestamp, so a caller using
|
||||
// the payload-only DataSize corrupts both, silently until the needle is read back.
|
||||
if needle_blob.len() < NEEDLE_HEADER_SIZE {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!(
|
||||
"needle {} blob of {} bytes is shorter than a needle header",
|
||||
needle_id.0,
|
||||
needle_blob.len()
|
||||
),
|
||||
)));
|
||||
}
|
||||
let (_, _, header_size) = Needle::parse_header(needle_blob);
|
||||
if header_size != size {
|
||||
return Err(VolumeError::Io(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!(
|
||||
"needle {} size {} does not match its blob header size {}",
|
||||
needle_id.0, size.0, header_size.0
|
||||
),
|
||||
)));
|
||||
}
|
||||
|
||||
// Dedup check: if the same needle already exists with matching content, skip the write.
|
||||
// Matches Go's WriteNeedleBlob which reads existing needle and compares cookie+checksum+data.
|
||||
if let Some(nm) = &self.nm {
|
||||
@@ -3629,7 +3604,7 @@ fn get_append_at_ns(last: u64) -> u64 {
|
||||
/// durable, propagating a sync failure so the commit path can abort rather than
|
||||
/// proceed with an undurable rename or marker. A path with no openable parent is
|
||||
/// tolerated; directory fsync is unsupported on Windows, so it is a no-op there
|
||||
/// (matching the Go fsyncDir helper, which skips it too).
|
||||
/// (matching the Go fsyncDir helper, which ignores that error).
|
||||
pub(crate) fn fsync_dir(path: &str) -> io::Result<()> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
@@ -4117,53 +4092,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// Go's tail check is v3-only, so it serves a v1/v2 volume with an unindexed
|
||||
// tail read-write; checking it here flipped whole legacy disks read-only.
|
||||
#[test]
|
||||
fn test_integrity_skips_dat_tail_check_before_v3() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
{
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
"",
|
||||
VolumeId(1),
|
||||
NeedleMapKind::InMemory,
|
||||
None,
|
||||
None,
|
||||
0,
|
||||
VERSION_2,
|
||||
)
|
||||
.unwrap();
|
||||
for i in 1..=3u64 {
|
||||
let data = format!("data {}", i);
|
||||
let mut n = Needle {
|
||||
id: NeedleId(i),
|
||||
cookie: Cookie(i as u32),
|
||||
data: data.as_bytes().to_vec(),
|
||||
data_size: data.len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut n, true).unwrap();
|
||||
}
|
||||
v.sync_to_disk().unwrap();
|
||||
}
|
||||
|
||||
let dat_path = volume_file_name(dir, "", VolumeId(1)) + ".dat";
|
||||
let mut f = OpenOptions::new().append(true).open(&dat_path).unwrap();
|
||||
f.write_all(&[0xFFu8; 96]).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
drop(f);
|
||||
|
||||
let v = reload_volume(dir);
|
||||
assert_eq!(v.version(), VERSION_2, "volume should reload as v2");
|
||||
assert!(
|
||||
!v.is_no_write_or_delete(),
|
||||
"v2 volume with unindexed .dat tail must stay writable, as in Go"
|
||||
);
|
||||
}
|
||||
|
||||
// Same torn tail, but with a deletion tombstone as the last indexed
|
||||
// record — the tombstone path must also verify the .dat ends at it.
|
||||
#[test]
|
||||
@@ -4517,39 +4445,6 @@ mod tests {
|
||||
assert!(matches!(err, VolumeError::CookieMismatch(_)));
|
||||
}
|
||||
|
||||
// A size disagreeing with the blob's own header indexes the needle at the wrong
|
||||
// length and, on v3, stamps the append timestamp into the middle of the needle.
|
||||
#[test]
|
||||
fn test_write_needle_blob_rejects_size_mismatch() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut v = make_test_volume(dir);
|
||||
|
||||
let mut n = Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x12345678),
|
||||
data: b"the merged payload".to_vec(),
|
||||
data_size: 18,
|
||||
..Needle::default()
|
||||
};
|
||||
n.checksum = CRC::new(&n.data);
|
||||
let (offset, _, _) = v.write_needle(&mut n, true).unwrap();
|
||||
let blob = v.read_needle_blob(offset as i64, n.size).unwrap();
|
||||
|
||||
let dat_size_before = v.dat_file_size().unwrap();
|
||||
|
||||
// Size(n.data_size) is what append reports, and what a caller following the
|
||||
// payload-size convention would send.
|
||||
let err = v
|
||||
.write_needle_blob_and_index(NeedleId(2), &blob, Size(n.data_size as i32))
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, VolumeError::Io(_)), "got {err:?}");
|
||||
assert_eq!(v.dat_file_size().unwrap(), dat_size_before);
|
||||
|
||||
v.write_needle_blob_and_index(NeedleId(2), &blob, n.size)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_volume_destroy() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
+11
-15
@@ -8,7 +8,7 @@ A privacy-respecting telemetry system for SeaweedFS that collects cluster-level
|
||||
- **Prometheus Integration**: Native Prometheus metrics for monitoring and alerting
|
||||
- **Grafana Dashboards**: Pre-built dashboards for data visualization
|
||||
- **Protocol Buffers**: Efficient binary data transmission for optimal performance
|
||||
- **Opt-out**: On by default, turned off with a single flag
|
||||
- **Opt-in Only**: Disabled by default, requires explicit configuration
|
||||
- **Docker Compose**: Complete monitoring stack deployment
|
||||
- **Automatic Cleanup**: Configurable data retention policies
|
||||
|
||||
@@ -51,7 +51,7 @@ message TelemetryData {
|
||||
- **No Personal Data**: No hostnames, IP addresses, or user information
|
||||
- **In-Memory IDs**: Cluster IDs are generated in-memory and change on restart
|
||||
- **Aggregated Data**: Only cluster-level statistics, no individual file/user data
|
||||
- **Opt-out Anytime**: `-telemetry=false` on the master stops all reporting
|
||||
- **Opt-in Only**: Telemetry is disabled by default
|
||||
- **Transparent**: Open source implementation, clear data collection policy
|
||||
|
||||
## Collected Data
|
||||
@@ -86,15 +86,14 @@ go run . -port=8080 -dashboard=true
|
||||
### 2. Configure SeaweedFS
|
||||
|
||||
```bash
|
||||
# Reporting to telemetry.seaweedfs.com is on by default
|
||||
weed master
|
||||
# Enable telemetry in SeaweedFS master (uses default telemetry.seaweedfs.com)
|
||||
weed master -telemetry=true
|
||||
|
||||
# Send to your own telemetry server instead
|
||||
weed master -telemetry.url=http://localhost:8080/api/collect
|
||||
# Or in server mode
|
||||
weed server -telemetry=true
|
||||
|
||||
# Turn reporting off
|
||||
weed master -telemetry=false
|
||||
weed server -master.telemetry=false
|
||||
# Or specify custom telemetry server
|
||||
weed master -telemetry=true -telemetry.url=http://localhost:8080/api/collect
|
||||
```
|
||||
|
||||
### 3. Access Dashboards
|
||||
@@ -108,15 +107,13 @@ weed server -master.telemetry=false
|
||||
### SeaweedFS Master/Server
|
||||
|
||||
```bash
|
||||
# Disable telemetry (enabled by default)
|
||||
-telemetry=false
|
||||
# Enable telemetry
|
||||
-telemetry=true
|
||||
|
||||
# Set custom telemetry server URL (optional, defaults to telemetry.seaweedfs.com)
|
||||
-telemetry.url=http://your-telemetry-server:8080/api/collect
|
||||
```
|
||||
|
||||
In `weed server` and `weed mini` the flags are prefixed: `-master.telemetry=false` and `-master.telemetry.url=...`.
|
||||
|
||||
### Telemetry Server
|
||||
|
||||
```bash
|
||||
@@ -181,8 +178,7 @@ GET /api/metrics?days=30
|
||||
# Get one cluster's daily usage history (disk bytes, volumes, volume servers)
|
||||
GET /api/history?cluster_id=<uuid>&days=90
|
||||
|
||||
# Get per-cluster disk usage and volume servers over time, largest first,
|
||||
# the rest summed as "other"
|
||||
# Get per-cluster disk usage over time, largest first, the rest summed as "other"
|
||||
GET /api/cluster-sizes?days=30&limit=20
|
||||
```
|
||||
|
||||
|
||||
@@ -151,6 +151,18 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
<canvas id="osChart" width="400" height="200"></canvas>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="chart-container">
|
||||
<div class="chart-title">Volume Servers Over Time</div>
|
||||
<canvas id="serverChart" width="400" height="200"></canvas>
|
||||
</div>
|
||||
|
||||
<div class="chart-container">
|
||||
<div class="chart-title">Total Disk Usage Over Time</div>
|
||||
<canvas id="diskChart" width="400" height="200"></canvas>
|
||||
</div>
|
||||
|
||||
<div class="chart-container">
|
||||
<div class="chart-title">Cluster Sizes Over Time</div>
|
||||
<div class="chart-subtitle" id="clusterSizesTotal"></div>
|
||||
@@ -159,14 +171,6 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chart-container">
|
||||
<div class="chart-title">Volume Servers Over Time</div>
|
||||
<div class="chart-subtitle" id="clusterServersTotal"></div>
|
||||
<div style="position: relative; height: 420px;">
|
||||
<canvas id="serverChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chart-container">
|
||||
<div class="chart-title">Per-Cluster History</div>
|
||||
<div class="cluster-lookup">
|
||||
@@ -192,13 +196,17 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
// Load stats
|
||||
const statsResponse = await fetch('/api/stats');
|
||||
const stats = await statsResponse.json();
|
||||
|
||||
// Load metrics
|
||||
const metricsResponse = await fetch('/api/metrics?days=30');
|
||||
const metrics = await metricsResponse.json();
|
||||
|
||||
// Load per-cluster sizes over time
|
||||
const sizesResponse = await fetch('/api/cluster-sizes?days=30&limit=20');
|
||||
const sizes = await sizesResponse.json();
|
||||
|
||||
updateStats(stats);
|
||||
updateCharts(stats);
|
||||
updateCharts(stats, metrics);
|
||||
updateClusterSizes(sizes);
|
||||
|
||||
document.getElementById('loading').style.display = 'none';
|
||||
@@ -217,9 +225,25 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
document.getElementById('totalOS').textContent = Object.keys(stats.os_distribution || {}).length;
|
||||
}
|
||||
|
||||
function updateCharts(stats) {
|
||||
function updateCharts(stats, metrics) {
|
||||
// Version chart
|
||||
createPieChart('versionChart', 'Version Distribution', stats.versions || {});
|
||||
|
||||
// OS chart
|
||||
createPieChart('osChart', 'Operating System Distribution', stats.os_distribution || {});
|
||||
|
||||
|
||||
|
||||
// Server count over time
|
||||
if (metrics.dates && metrics.server_counts) {
|
||||
createLineChart('serverChart', 'Volume Servers', metrics.dates, metrics.server_counts, '#2196F3');
|
||||
}
|
||||
|
||||
// Disk usage over time
|
||||
if (metrics.dates && metrics.disk_usage) {
|
||||
const diskUsageGB = metrics.disk_usage.map(bytes => Math.round(bytes / (1024 * 1024 * 1024)));
|
||||
createLineChart('diskChart', 'Disk Usage (GB)', metrics.dates, diskUsageGB, '#4CAF50');
|
||||
}
|
||||
}
|
||||
|
||||
function createPieChart(canvasId, title, data) {
|
||||
@@ -298,50 +322,37 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
return (unit === 0 ? value : value.toFixed(value >= 100 ? 0 : 1)) + ' ' + units[unit];
|
||||
}
|
||||
|
||||
// Disk usage and volume servers both drawn as one stacked band per
|
||||
// cluster over time: the band is that cluster's share, the top of the
|
||||
// stack is the fleet total. Clusters beyond the requested limit are
|
||||
// summed into a trailing "other" band so the stack still adds up.
|
||||
// One stacked band per cluster over time: the band is that cluster's
|
||||
// size, the top of the stack is the fleet total. Clusters beyond the
|
||||
// requested limit are summed into a trailing "other" band so the stack
|
||||
// still adds up.
|
||||
function updateClusterSizes(series) {
|
||||
const clusters = series.clusters || [];
|
||||
const dates = series.dates || [];
|
||||
const count = series.cluster_count || 0;
|
||||
const servers = series.total_servers || 0;
|
||||
const across = ' across ' + count + ' cluster' + (count === 1 ? '' : 's') +
|
||||
|
||||
document.getElementById('clusterSizesTotal').textContent =
|
||||
formatBytes(series.total_disk) + ' across ' + count + ' cluster' + (count === 1 ? '' : 's') +
|
||||
' on ' + (dates[dates.length - 1] || 'no data');
|
||||
|
||||
document.getElementById('clusterSizesTotal').textContent = formatBytes(series.total_disk) + across;
|
||||
document.getElementById('clusterServersTotal').textContent =
|
||||
servers + ' volume server' + (servers === 1 ? '' : 's') + across;
|
||||
|
||||
clusterSizeIds = clusters.map(c => c.cluster_id);
|
||||
if (series.other) {
|
||||
clusterSizeIds.push(null);
|
||||
}
|
||||
|
||||
stackedChart('clusterSizeChart', dates, clusters, series.other, 'disk', formatBytes);
|
||||
stackedChart('serverChart', dates, clusters, series.other, 'servers', value => value);
|
||||
}
|
||||
|
||||
// The stacks share one cluster order, so a cluster keeps its colour and
|
||||
// its legend entry across both of them.
|
||||
function stackedChart(canvasId, dates, clusters, other, key, format) {
|
||||
const datasets = clusters.map((c, i) => {
|
||||
// Evenly spaced hues keep neighbouring bands distinguishable.
|
||||
const hue = Math.round(i * 360 / clusters.length);
|
||||
return band(c.cluster_id.slice(0, 8), c[key],
|
||||
return band(c.cluster_id.slice(0, 8), c.disk,
|
||||
'hsl(' + hue + ', 65%, 45%)', 'hsla(' + hue + ', 65%, 55%, 0.75)');
|
||||
});
|
||||
if (other) {
|
||||
datasets.push(band('other (' + other.count + ')', other[key],
|
||||
if (series.other) {
|
||||
clusterSizeIds.push(null);
|
||||
datasets.push(band('other (' + series.other.count + ')', series.other.disk,
|
||||
'#9E9E9E', 'rgba(158, 158, 158, 0.6)'));
|
||||
}
|
||||
|
||||
const ctx = document.getElementById(canvasId).getContext('2d');
|
||||
if (charts[canvasId]) {
|
||||
charts[canvasId].destroy();
|
||||
const ctx = document.getElementById('clusterSizeChart').getContext('2d');
|
||||
if (charts.clusterSizeChart) {
|
||||
charts.clusterSizeChart.destroy();
|
||||
}
|
||||
charts[canvasId] = new Chart(ctx, {
|
||||
charts.clusterSizeChart = new Chart(ctx, {
|
||||
type: 'line',
|
||||
data: { labels: dates, datasets: datasets },
|
||||
options: {
|
||||
@@ -361,7 +372,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
callbacks: {
|
||||
label: item => {
|
||||
const id = clusterSizeIds[item.datasetIndex];
|
||||
return (id || item.dataset.label) + ': ' + format(item.raw);
|
||||
return (id || item.dataset.label) + ': ' + formatBytes(item.raw);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -371,7 +382,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
|
||||
y: {
|
||||
stacked: true,
|
||||
beginAtZero: true,
|
||||
ticks: { precision: 0, callback: value => format(value) }
|
||||
ticks: { callback: value => formatBytes(value) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/seaweedfs/seaweedfs/telemetry/server
|
||||
go 1.25.8
|
||||
|
||||
require (
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/prometheus/client_golang v1.24.0
|
||||
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
@@ -13,7 +13,7 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/common v0.70.0 // indirect
|
||||
github.com/prometheus/procfs v0.21.1 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
)
|
||||
|
||||
@@ -6,20 +6,20 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
|
||||
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
|
||||
github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI=
|
||||
github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI=
|
||||
github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY=
|
||||
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
|
||||
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
|
||||
@@ -40,25 +40,6 @@ func (s *PrometheusStorage) appendHistory(data *proto.TelemetryData, receivedAt
|
||||
s.histories[data.TopologyId] = h
|
||||
}
|
||||
|
||||
// seriesHistories picks the clusters the fleet-wide series are built from: the
|
||||
// confirmed ones. A cluster that only ever reported on one day is usually a CI
|
||||
// or test cluster that lived for a minute, and those arrive faster than they
|
||||
// age out, so counting them makes every fleet total climb forever. Falls back to
|
||||
// all clusters while none is confirmed yet, so a fresh server still draws its
|
||||
// charts. Callers must hold s.mu.
|
||||
func (s *PrometheusStorage) seriesHistories() map[string][]HistorySample {
|
||||
confirmed := make(map[string][]HistorySample, len(s.histories))
|
||||
for id, history := range s.histories {
|
||||
if len(history) >= confirmDays {
|
||||
confirmed[id] = history
|
||||
}
|
||||
}
|
||||
if len(confirmed) == 0 {
|
||||
return s.histories
|
||||
}
|
||||
return confirmed
|
||||
}
|
||||
|
||||
func sameUTCDay(a, b int64) bool {
|
||||
ta, tb := time.Unix(a, 0).UTC(), time.Unix(b, 0).UTC()
|
||||
return ta.Year() == tb.Year() && ta.YearDay() == tb.YearDay()
|
||||
|
||||
@@ -70,7 +70,7 @@ func TestGetMetricsWindowStartsAtOldestSample(t *testing.T) {
|
||||
}
|
||||
|
||||
// History reaching past the requested window still clips to the window.
|
||||
seedSamples(s, "old", HistorySample{TotalDiskBytes: 50, VolumeServerCount: 1}, -40, -39)
|
||||
seedSamples(s, "old", HistorySample{TotalDiskBytes: 50, VolumeServerCount: 1}, -40)
|
||||
metrics, err = s.GetMetrics(10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -80,13 +80,13 @@ func TestGetMetricsWindowStartsAtOldestSample(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// /api/metrics is the fleet total that /api/cluster-sizes breaks down per
|
||||
// cluster, so their last day has to add up to the same number.
|
||||
// "Total Disk Usage Over Time" and the stacked "Cluster Sizes Over Time" sit on
|
||||
// the same dashboard, so their last day has to add up to the same number.
|
||||
func TestGetMetricsAgreesWithClusterSizes(t *testing.T) {
|
||||
s := newPrometheusStorage(prometheus.NewRegistry())
|
||||
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
|
||||
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
|
||||
seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2}, -3, -2)
|
||||
seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2}, -2)
|
||||
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
|
||||
|
||||
metrics, err := s.GetMetrics(10)
|
||||
@@ -104,45 +104,6 @@ func TestGetMetricsAgreesWithClusterSizes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Short-lived clusters -- a CI run, a docker-compose demo -- report once under a
|
||||
// fresh raft topology id and never again. Held forward for the whole active
|
||||
// window they would stack up into a fleet that grows every day, so they stay out
|
||||
// of the totals until they are confirmed.
|
||||
func TestGetMetricsExcludesUnconfirmedClusters(t *testing.T) {
|
||||
s := newPrometheusStorage(prometheus.NewRegistry())
|
||||
|
||||
seedSamples(s, "real", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3}, -3, -2, -1, 0)
|
||||
for _, id := range []string{"ci-1", "ci-2", "ci-3"} {
|
||||
seedSamples(s, id, HistorySample{TotalDiskBytes: 5, VolumeServerCount: 14}, -1)
|
||||
}
|
||||
|
||||
metrics, err := s.GetMetrics(4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := metrics["server_counts"].([]int64); !equalInt64(got, []int64{3, 3, 3, 3}) {
|
||||
t.Errorf("server_counts = %v, want the confirmed cluster alone", got)
|
||||
}
|
||||
if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{300, 300, 300, 300}) {
|
||||
t.Errorf("disk_usage = %v, want the confirmed cluster alone", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Until any cluster has two days of history the charts fall back to every
|
||||
// cluster, so a fresh server doesn't serve empty series.
|
||||
func TestGetMetricsFallsBackWhenNoneConfirmed(t *testing.T) {
|
||||
s := newPrometheusStorage(prometheus.NewRegistry())
|
||||
seedSamples(s, "new", HistorySample{TotalDiskBytes: 100, VolumeServerCount: 2}, 0)
|
||||
|
||||
metrics, err := s.GetMetrics(7)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := metrics["server_counts"].([]int64); !equalInt64(got, []int64{2}) {
|
||||
t.Errorf("server_counts = %v, want today's only cluster", got)
|
||||
}
|
||||
}
|
||||
|
||||
func equalInt64(a, b []int64) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
|
||||
@@ -164,22 +164,21 @@ func (s *PrometheusStorage) GetInstances(limit int) ([]*telemetryData, error) {
|
||||
return instances, nil
|
||||
}
|
||||
|
||||
// GetMetrics returns fleet-wide daily totals across confirmed clusters for the
|
||||
// last `days` days, in the parallel-array shape the dashboard charts expect.
|
||||
// Totals come from the daily histories rather than from s.instances, which holds
|
||||
// only each cluster's most recent report and so would credit every cluster to
|
||||
// the single day it last reported on.
|
||||
// GetMetrics returns fleet-wide daily totals for the last `days` days, in the
|
||||
// parallel-array shape the dashboard charts expect. Totals come from the daily
|
||||
// histories rather than from s.instances, which holds only each cluster's most
|
||||
// recent report and so would credit every cluster to the single day it last
|
||||
// reported on.
|
||||
func (s *PrometheusStorage) GetMetrics(days int) (map[string]interface{}, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
histories := s.seriesHistories()
|
||||
axis := newDailySeries(days, histories)
|
||||
axis := newDailySeries(days, s.histories)
|
||||
activeSince := time.Now().UTC().AddDate(0, 0, -activeDays).Unix()
|
||||
|
||||
diskUsage := make([]uint64, len(axis.dates))
|
||||
serverCounts := make([]int64, len(axis.dates))
|
||||
for _, history := range histories {
|
||||
for _, history := range s.histories {
|
||||
if disk, ok := axis.align(history, activeSince, diskBytes); ok {
|
||||
for i, v := range disk {
|
||||
diskUsage[i] += v
|
||||
|
||||
@@ -5,57 +5,48 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ClusterSeries is one cluster's daily disk usage and volume server count,
|
||||
// aligned to the shared date axis of the enclosing ClusterSizeSeries.
|
||||
// ClusterSeries is one cluster's daily disk usage, aligned to the shared date
|
||||
// axis of the enclosing ClusterSizeSeries.
|
||||
type ClusterSeries struct {
|
||||
ClusterId string `json:"cluster_id"`
|
||||
Disk []uint64 `json:"disk"`
|
||||
Servers []uint64 `json:"servers"`
|
||||
}
|
||||
|
||||
// OtherSeries is the clusters beyond the caller's limit, summed per day so a
|
||||
// stacked chart still adds up to the fleet total.
|
||||
type OtherSeries struct {
|
||||
Count int `json:"count"`
|
||||
Disk []uint64 `json:"disk"`
|
||||
Servers []uint64 `json:"servers"`
|
||||
Count int `json:"count"`
|
||||
Disk []uint64 `json:"disk"`
|
||||
}
|
||||
|
||||
// ClusterSizeSeries is per-cluster disk usage and volume server count over
|
||||
// time: one value per cluster per day, largest cluster first, ranked by disk on
|
||||
// their most recent day. Both metrics share one ranking so a cluster keeps its
|
||||
// place, and its colour, across the charts drawn from this.
|
||||
// ClusterSizeSeries is per-cluster disk usage over time: one value per cluster
|
||||
// per day, largest cluster first, ranked by their most recent day.
|
||||
type ClusterSizeSeries struct {
|
||||
Dates []string `json:"dates"`
|
||||
Clusters []ClusterSeries `json:"clusters"`
|
||||
Other *OtherSeries `json:"other,omitempty"`
|
||||
ClusterCount int `json:"cluster_count"`
|
||||
TotalDisk uint64 `json:"total_disk"` // across all clusters on the last day
|
||||
TotalServers uint64 `json:"total_servers"` // across all clusters on the last day
|
||||
TotalDisk uint64 `json:"total_disk"` // across all clusters on the last day
|
||||
}
|
||||
|
||||
// GetClusterSizeSeries returns the last `days` days of per-cluster disk usage
|
||||
// and volume server counts across confirmed clusters. Clusters beyond `limit`
|
||||
// are folded into Other.
|
||||
// GetClusterSizeSeries returns the last `days` days of per-cluster disk usage.
|
||||
// Clusters beyond `limit` are folded into Other.
|
||||
func (s *PrometheusStorage) GetClusterSizeSeries(days, limit int) ClusterSizeSeries {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
histories := s.seriesHistories()
|
||||
axis := newDailySeries(days, histories)
|
||||
axis := newDailySeries(days, s.histories)
|
||||
activeSince := time.Now().UTC().AddDate(0, 0, -activeDays).Unix()
|
||||
last := len(axis.dates) - 1
|
||||
series := ClusterSizeSeries{Dates: axis.dates}
|
||||
|
||||
for id, history := range histories {
|
||||
for id, history := range s.histories {
|
||||
disk, ok := axis.align(history, activeSince, diskBytes)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
servers, _ := axis.align(history, activeSince, serverCount)
|
||||
series.Clusters = append(series.Clusters, ClusterSeries{ClusterId: id, Disk: disk, Servers: servers})
|
||||
series.Clusters = append(series.Clusters, ClusterSeries{ClusterId: id, Disk: disk})
|
||||
series.TotalDisk += disk[last]
|
||||
series.TotalServers += servers[last]
|
||||
}
|
||||
series.ClusterCount = len(series.Clusters)
|
||||
|
||||
@@ -71,17 +62,13 @@ func (s *PrometheusStorage) GetClusterSizeSeries(days, limit int) ClusterSizeSer
|
||||
|
||||
if limit > 0 && len(series.Clusters) > limit {
|
||||
other := OtherSeries{
|
||||
Count: len(series.Clusters) - limit,
|
||||
Disk: make([]uint64, len(axis.dates)),
|
||||
Servers: make([]uint64, len(axis.dates)),
|
||||
Count: len(series.Clusters) - limit,
|
||||
Disk: make([]uint64, len(axis.dates)),
|
||||
}
|
||||
for _, c := range series.Clusters[limit:] {
|
||||
for i, v := range c.Disk {
|
||||
other.Disk[i] += v
|
||||
}
|
||||
for i, v := range c.Servers {
|
||||
other.Servers[i] += v
|
||||
}
|
||||
}
|
||||
series.Clusters = series.Clusters[:limit]
|
||||
series.Other = &other
|
||||
|
||||
@@ -8,7 +8,12 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/telemetry/proto"
|
||||
)
|
||||
|
||||
// seedSamples gives a cluster one sample per listed day offset (0 is today).
|
||||
// seedHistory gives a cluster one sample per listed day offset (0 is today).
|
||||
func seedHistory(s *PrometheusStorage, id string, disk uint64, dayOffsets ...int) {
|
||||
seedSamples(s, id, HistorySample{TotalDiskBytes: disk}, dayOffsets...)
|
||||
}
|
||||
|
||||
// seedSamples is seedHistory for tests that care about more than disk usage.
|
||||
// The sample's Ts is filled in per day offset.
|
||||
func seedSamples(s *PrometheusStorage, id string, sample HistorySample, dayOffsets ...int) {
|
||||
s.mu.Lock()
|
||||
@@ -23,15 +28,12 @@ func TestClusterSizeSeries(t *testing.T) {
|
||||
s := newPrometheusStorage(prometheus.NewRegistry())
|
||||
|
||||
// Reported every day of the window.
|
||||
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
|
||||
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
|
||||
seedHistory(s, "daily", 300, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
|
||||
// Reported two days ago and not since: still active, so its size is held
|
||||
// to the right edge instead of dropping out of the stack.
|
||||
seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2}, -3, -2)
|
||||
seedHistory(s, "lagging", 200, -2)
|
||||
// Stopped reporting past the active window: its own days only.
|
||||
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
|
||||
// One day of history only: unconfirmed, so it stays out of the stack.
|
||||
seedSamples(s, "oneshot", HistorySample{TotalDiskBytes: 400, VolumeServerCount: 4}, -1)
|
||||
seedHistory(s, "gone", 900, -9, -8)
|
||||
|
||||
series := s.GetClusterSizeSeries(10, 0)
|
||||
if len(series.Dates) != 10 {
|
||||
@@ -41,41 +43,24 @@ func TestClusterSizeSeries(t *testing.T) {
|
||||
t.Fatalf("cluster_count = %d, want 3", series.ClusterCount)
|
||||
}
|
||||
|
||||
byId := map[string]ClusterSeries{}
|
||||
byId := map[string][]uint64{}
|
||||
for _, c := range series.Clusters {
|
||||
byId[c.ClusterId] = c
|
||||
byId[c.ClusterId] = c.Disk
|
||||
}
|
||||
if got := byId["daily"].Disk; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {
|
||||
if got := byId["daily"]; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {
|
||||
t.Errorf("daily = %v, want 300 every day", got)
|
||||
}
|
||||
if got := byId["lagging"].Disk; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 200, 200, 200, 200}) {
|
||||
if got := byId["lagging"]; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 0, 200, 200, 200}) {
|
||||
t.Errorf("lagging = %v, want its size carried to the right edge", got)
|
||||
}
|
||||
if got := byId["gone"].Disk; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {
|
||||
if got := byId["gone"]; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {
|
||||
t.Errorf("gone = %v, want no capacity after its last report", got)
|
||||
}
|
||||
if _, ok := byId["oneshot"]; ok {
|
||||
t.Errorf("unconfirmed cluster in the stack: %+v", byId["oneshot"])
|
||||
}
|
||||
|
||||
// Volume servers ride the same axis and the same hold-forward rule.
|
||||
if got := byId["daily"].Servers; !equal(got, []uint64{3, 3, 3, 3, 3, 3, 3, 3, 3, 3}) {
|
||||
t.Errorf("daily servers = %v, want 3 every day", got)
|
||||
}
|
||||
if got := byId["lagging"].Servers; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 2, 2, 2, 2}) {
|
||||
t.Errorf("lagging servers = %v, want carried to the right edge", got)
|
||||
}
|
||||
if got := byId["gone"].Servers; !equal(got, []uint64{9, 9, 0, 0, 0, 0, 0, 0, 0, 0}) {
|
||||
t.Errorf("gone servers = %v, want none after its last report", got)
|
||||
}
|
||||
|
||||
// The totals are the last day's stack height: daily + lagging, not gone.
|
||||
// The total is the last day's stack height: daily + lagging, not gone.
|
||||
if series.TotalDisk != 500 {
|
||||
t.Errorf("total_disk = %d, want 500", series.TotalDisk)
|
||||
}
|
||||
if series.TotalServers != 5 {
|
||||
t.Errorf("total_servers = %d, want 5", series.TotalServers)
|
||||
}
|
||||
// Largest on the last day comes first so the stack reads top-down.
|
||||
if series.Clusters[0].ClusterId != "daily" {
|
||||
t.Errorf("order = %s first, want daily", series.Clusters[0].ClusterId)
|
||||
@@ -89,15 +74,11 @@ func TestClusterSizeSeries(t *testing.T) {
|
||||
if series.Other == nil || series.Other.Count != 2 {
|
||||
t.Fatalf("other = %+v, want 2 clusters", series.Other)
|
||||
}
|
||||
if !equal(series.Other.Disk, []uint64{900, 900, 0, 0, 0, 0, 200, 200, 200, 200}) {
|
||||
if !equal(series.Other.Disk, []uint64{900, 900, 0, 0, 0, 0, 0, 200, 200, 200}) {
|
||||
t.Errorf("other = %v, want lagging+gone summed per day", series.Other.Disk)
|
||||
}
|
||||
if !equal(series.Other.Servers, []uint64{9, 9, 0, 0, 0, 0, 2, 2, 2, 2}) {
|
||||
t.Errorf("other servers = %v, want lagging+gone summed per day", series.Other.Servers)
|
||||
}
|
||||
if series.ClusterCount != 3 || series.TotalDisk != 500 || series.TotalServers != 5 {
|
||||
t.Errorf("limit changed totals: count=%d disk=%d servers=%d, want 3/500/5",
|
||||
series.ClusterCount, series.TotalDisk, series.TotalServers)
|
||||
if series.ClusterCount != 3 || series.TotalDisk != 500 {
|
||||
t.Errorf("limit changed totals: count=%d disk=%d, want 3/500", series.ClusterCount, series.TotalDisk)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,17 +88,15 @@ func TestClusterSizeSeriesUsesLatestDailySample(t *testing.T) {
|
||||
s := newPrometheusStorage(prometheus.NewRegistry())
|
||||
|
||||
data := &proto.TelemetryData{
|
||||
TopologyId: "aaaaaaaa-0000-0000-0000-000000000001",
|
||||
Version: "4.40",
|
||||
Os: "linux/amd64",
|
||||
TotalDiskBytes: 100,
|
||||
VolumeServerCount: 4,
|
||||
TopologyId: "aaaaaaaa-0000-0000-0000-000000000001",
|
||||
Version: "4.40",
|
||||
Os: "linux/amd64",
|
||||
TotalDiskBytes: 100,
|
||||
}
|
||||
if err := s.StoreTelemetry(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data.TotalDiskBytes = 700
|
||||
data.VolumeServerCount = 6
|
||||
if err := s.StoreTelemetry(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -134,11 +113,8 @@ func TestClusterSizeSeriesUsesLatestDailySample(t *testing.T) {
|
||||
if got := series.Clusters[0].Disk; !equal(got, []uint64{700}) {
|
||||
t.Errorf("disk = %v, want today's latest sample only", got)
|
||||
}
|
||||
if got := series.Clusters[0].Servers; !equal(got, []uint64{6}) {
|
||||
t.Errorf("servers = %v, want today's latest sample only", got)
|
||||
}
|
||||
if series.TotalDisk != 700 || series.TotalServers != 6 {
|
||||
t.Errorf("totals = %d disk / %d servers, want 700/6", series.TotalDisk, series.TotalServers)
|
||||
if series.TotalDisk != 700 {
|
||||
t.Errorf("total_disk = %d, want 700", series.TotalDisk)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,8 +24,8 @@ type FuseTestFramework struct {
|
||||
mountPoint string
|
||||
dataDir string
|
||||
logDir string
|
||||
miniProcess *managedProcess
|
||||
mountProcess *managedProcess
|
||||
miniProcess *os.Process
|
||||
mountProcess *os.Process
|
||||
filerAddr string
|
||||
filerPort int
|
||||
weedBinary string
|
||||
@@ -146,7 +146,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error {
|
||||
}
|
||||
|
||||
// Wait for filer to be ready (mini starts all services on filerPort)
|
||||
if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil {
|
||||
if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil {
|
||||
f.dumpLog("mini")
|
||||
return fmt.Errorf("weed mini not ready: %v", err)
|
||||
}
|
||||
@@ -173,11 +173,16 @@ func (f *FuseTestFramework) Cleanup() {
|
||||
f.DumpLogs()
|
||||
}
|
||||
|
||||
if f.mountProcess != nil {
|
||||
f.unmountFuse()
|
||||
}
|
||||
|
||||
// Stop processes in reverse order
|
||||
f.unmountFuse()
|
||||
if f.miniProcess != nil {
|
||||
f.miniProcess.stop()
|
||||
f.miniProcess = nil
|
||||
for _, proc := range []*os.Process{f.mountProcess, f.miniProcess} {
|
||||
if proc != nil {
|
||||
proc.Signal(syscall.SIGTERM)
|
||||
proc.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
f.copyLogsForCI()
|
||||
@@ -204,43 +209,9 @@ func (f *FuseTestFramework) GetFilerAddr() string {
|
||||
return f.filerAddr
|
||||
}
|
||||
|
||||
// managedProcess is a started weed sub-command whose exit is watched, so a wait
|
||||
// for it to come up ends the moment it dies instead of burning its full timeout.
|
||||
type managedProcess struct {
|
||||
cmd *exec.Cmd
|
||||
done chan struct{}
|
||||
err error // exit error, read only after done is closed
|
||||
}
|
||||
|
||||
// exited returns the exit error once the process is gone, nil while it runs.
|
||||
func (p *managedProcess) exited() error {
|
||||
select {
|
||||
case <-p.done:
|
||||
if p.err != nil {
|
||||
return p.err
|
||||
}
|
||||
return fmt.Errorf("exit status 0")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// stop asks the process to terminate and waits for it to go away.
|
||||
func (p *managedProcess) stop() {
|
||||
// Signal fails with os.ErrProcessDone when the child is already gone, which
|
||||
// is exactly the case the select below handles.
|
||||
_ = p.cmd.Process.Signal(syscall.SIGTERM)
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(10 * time.Second):
|
||||
p.cmd.Process.Kill()
|
||||
<-p.done
|
||||
}
|
||||
}
|
||||
|
||||
// startProcess is a helper that starts a weed sub-command with output captured
|
||||
// to a log file in f.logDir.
|
||||
func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) {
|
||||
func (f *FuseTestFramework) startProcess(name string, args []string) (*os.Process, error) {
|
||||
logFile, err := os.Create(filepath.Join(f.logDir, name+".log"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create log file: %v", err)
|
||||
@@ -255,13 +226,7 @@ func (f *FuseTestFramework) startProcess(name string, args []string) (*managedPr
|
||||
}
|
||||
// Close the file handle — the child process inherited it.
|
||||
logFile.Close()
|
||||
|
||||
p := &managedProcess{cmd: cmd, done: make(chan struct{})}
|
||||
go func() {
|
||||
p.err = cmd.Wait()
|
||||
close(p.done)
|
||||
}()
|
||||
return p, nil
|
||||
return cmd.Process, nil
|
||||
}
|
||||
|
||||
// dumpLog prints the last lines of a process log file to the test output
|
||||
@@ -361,7 +326,8 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
|
||||
// unmountFuse unmounts the FUSE filesystem
|
||||
func (f *FuseTestFramework) unmountFuse() error {
|
||||
if f.mountProcess != nil {
|
||||
f.mountProcess.stop()
|
||||
f.mountProcess.Signal(syscall.SIGTERM)
|
||||
f.mountProcess.Wait()
|
||||
f.mountProcess = nil
|
||||
}
|
||||
|
||||
@@ -372,7 +338,7 @@ func (f *FuseTestFramework) unmountFuse() error {
|
||||
}
|
||||
|
||||
// waitForService waits for a service to be available
|
||||
func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error {
|
||||
func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
|
||||
@@ -380,9 +346,6 @@ func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, ti
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
if exitErr := proc.exited(); exitErr != nil {
|
||||
return fmt.Errorf("process exited (%v) before %s accepted connections", exitErr, addr)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("service at %s not ready within timeout", addr)
|
||||
@@ -405,11 +368,6 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// A mount that cannot mount at all (no /dev/fuse, fusermount not setuid)
|
||||
// dies within a second; reporting that beats waiting out the timeout.
|
||||
if exitErr := f.mountProcess.exited(); exitErr != nil {
|
||||
return fmt.Errorf("mount process exited (%v)", exitErr)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("mount point not ready within timeout")
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
package acl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func createOwnershipTestBucket(t *testing.T, client *s3.Client) string {
|
||||
bucketName := "test-ownership-" + strings.ToLower(strings.ReplaceAll(time.Now().Format("2006-01-02-15-04-05.000"), ":", "-"))
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return bucketName
|
||||
}
|
||||
|
||||
func getObjectOwnership(t *testing.T, client *s3.Client, bucketName string) types.ObjectOwnership {
|
||||
t.Helper()
|
||||
resp, err := client.GetBucketOwnershipControls(context.TODO(), &s3.GetBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp.OwnershipControls)
|
||||
require.Len(t, resp.OwnershipControls.Rules, 1)
|
||||
return resp.OwnershipControls.Rules[0].ObjectOwnership
|
||||
}
|
||||
|
||||
func putObjectOwnership(t *testing.T, client *s3.Client, bucketName string, ownership types.ObjectOwnership) {
|
||||
t.Helper()
|
||||
_, err := client.PutBucketOwnershipControls(context.TODO(), &s3.PutBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
OwnershipControls: &types.OwnershipControls{
|
||||
Rules: []types.OwnershipControlsRule{{ObjectOwnership: ownership}},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestGetBucketOwnershipControlsDefault(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := createOwnershipTestBucket(t, client)
|
||||
defer cleanupTestBucket(t, client, bucketName)
|
||||
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
}
|
||||
|
||||
func TestBucketOwnershipControlsLifecycle(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := createOwnershipTestBucket(t, client)
|
||||
defer cleanupTestBucket(t, client, bucketName)
|
||||
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipObjectWriter)
|
||||
assert.Equal(t, types.ObjectOwnershipObjectWriter, getObjectOwnership(t, client, bucketName))
|
||||
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
|
||||
_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
|
||||
}
|
||||
|
||||
// A bucket with nothing stored already reports BucketOwnerEnforced, so this put has
|
||||
// to persist anyway or the delete below finds nothing to remove.
|
||||
func TestPutBucketOwnershipControlsDefaultOnNewBucket(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := createOwnershipTestBucket(t, client)
|
||||
defer cleanupTestBucket(t, client, bucketName)
|
||||
|
||||
putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
|
||||
|
||||
_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "OwnershipControlsNotFound")
|
||||
}
|
||||
@@ -36,65 +36,12 @@ func TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort(t *testing.T) {
|
||||
|
||||
waitForLifecycleWorkerReady(t, adminEndpoint)
|
||||
|
||||
payload := runLifecycleJob(t, adminEndpoint, 30)
|
||||
|
||||
require.GreaterOrEqual(t, jsonNumber(t, payload, "detected_count"), 1)
|
||||
require.Equal(t, 0, jsonNumber(t, payload, "error_count"),
|
||||
"dispatched job errored — likely filer_grpc_address was raw host:httpPort.grpcPort")
|
||||
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(oldKey),
|
||||
})
|
||||
return err != nil
|
||||
}, 30*time.Second, 500*time.Millisecond,
|
||||
"expected %s/%s to be deleted after admin-dispatched lifecycle run", bucket, oldKey)
|
||||
}
|
||||
|
||||
// A worker-dispatched pass with rules in place but nothing due: the
|
||||
// invariant is that a pass returns on its own, so the admin never has to
|
||||
// cancel it and the executor slot is free for the next one.
|
||||
//
|
||||
// Not a regression test for the wedge this scenario comes from. Before
|
||||
// the subscription was bounded a pass ended only when some write landed
|
||||
// past its boundary, and on a shared test cluster something usually
|
||||
// does — verified: the whole suite passes on the unfixed build. The
|
||||
// deterministic guards are the dailyrun unit tests. This covers the
|
||||
// worker path end-to-end and would catch a pass that hangs
|
||||
// unconditionally.
|
||||
func TestLifecycleAdminDispatchCompletesWithNothingDue(t *testing.T) {
|
||||
adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
|
||||
|
||||
c := s3Client(t)
|
||||
bucket := uniqueBucket("admin-idle")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putExpirationLifecycle(t, c, bucket, "expire/", 1)
|
||||
// Deliberately not backdated: the rule matches the prefix but nothing
|
||||
// is due, so the pass dispatches nothing.
|
||||
putObject(t, c, bucket, "expire/fresh.txt", "fresh")
|
||||
|
||||
waitForLifecycleWorkerReady(t, adminEndpoint)
|
||||
|
||||
// Let the writes above settle below the pass boundary, or one of them
|
||||
// ends the pass and the wedge is masked.
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
payload := runLifecycleJob(t, adminEndpoint, 30)
|
||||
|
||||
require.Equal(t, 0, jsonNumber(t, payload, "canceled_count"),
|
||||
"pass did not return on its own; the admin cancelled it at the run timeout")
|
||||
require.GreaterOrEqual(t, jsonNumber(t, payload, "success_count"), 1,
|
||||
"expected the dispatched pass to complete")
|
||||
require.Equal(t, 0, jsonNumber(t, payload, "skipped_active_count"),
|
||||
"a previous pass still held the executor slot")
|
||||
}
|
||||
|
||||
// runLifecycleJob drives the admin's run endpoint and returns its decoded
|
||||
// response. timeoutSeconds bounds the run admin-side: a pass that never
|
||||
// returns comes back as canceled_count=1 after that long.
|
||||
func runLifecycleJob(t *testing.T, adminEndpoint string, timeoutSeconds int) map[string]any {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(map[string]any{"timeout_seconds": timeoutSeconds})
|
||||
// Lifecycle is a long-running batch; the run endpoint cancels it at
|
||||
// this timeout, which converts a healthy run into canceled_count=1.
|
||||
const runTimeoutSeconds = 30
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"timeout_seconds": runTimeoutSeconds,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
req, err := http.NewRequestWithContext(
|
||||
context.Background(), http.MethodPost,
|
||||
@@ -112,7 +59,18 @@ func runLifecycleJob(t *testing.T, adminEndpoint string, timeoutSeconds int) map
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&payload))
|
||||
t.Logf("admin /api/plugin/job-types/s3_lifecycle/run response: %v", payload)
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "admin run endpoint failed: %v", payload)
|
||||
return payload
|
||||
|
||||
require.GreaterOrEqual(t, jsonNumber(t, payload, "detected_count"), 1)
|
||||
require.Equal(t, 0, jsonNumber(t, payload, "error_count"),
|
||||
"dispatched job errored — likely filer_grpc_address was raw host:httpPort.grpcPort")
|
||||
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(oldKey),
|
||||
})
|
||||
return err != nil
|
||||
}, 30*time.Second, 500*time.Millisecond,
|
||||
"expected %s/%s to be deleted after admin-dispatched lifecycle run", bucket, oldKey)
|
||||
}
|
||||
|
||||
func jsonNumber(t *testing.T, payload map[string]any, key string) int {
|
||||
|
||||
@@ -3,19 +3,14 @@ package s3api
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -34,152 +29,6 @@ func suspendVersioning(t *testing.T, client *s3.Client, bucketName string) {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// vacuumVolumes asks the master to compact away the needles a delete tombstoned.
|
||||
// Tests that assert a surviving object still has its data need this: deleting an
|
||||
// entry only tombstones the needles it points at, so a shared chunk list reads
|
||||
// fine right up until the vacuum makes the loss permanent. A vacuum that does not
|
||||
// run leaves those tests asserting nothing, so treat every failure as fatal.
|
||||
func vacuumVolumes(t *testing.T) {
|
||||
t.Helper()
|
||||
require.NotEmpty(t, defaultConfig.MasterEndpoint, "vacuum needs a master endpoint; set MASTER_ENDPOINT")
|
||||
endpoint := strings.TrimRight(defaultConfig.MasterEndpoint, "/") + "/vol/vacuum?garbageThreshold=0.001"
|
||||
httpClient := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := httpClient.Get(endpoint)
|
||||
require.NoError(t, err, "vacuum request to %s", endpoint)
|
||||
defer resp.Body.Close()
|
||||
_, err = io.Copy(io.Discard, resp.Body)
|
||||
require.NoError(t, err, "reading the vacuum response")
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "vacuum request to %s", endpoint)
|
||||
}
|
||||
|
||||
func requireVersionBody(t *testing.T, client *s3.Client, bucketName, objectKey, versionId string, want []byte, msg string) {
|
||||
t.Helper()
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(versionId),
|
||||
})
|
||||
require.NoError(t, err, msg)
|
||||
defer getResp.Body.Close()
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err, msg)
|
||||
require.Equal(t, len(want), len(body), msg)
|
||||
require.True(t, bytes.Equal(want, body), msg)
|
||||
}
|
||||
|
||||
// chunkedTestContent returns a body large enough to land in volume needles rather
|
||||
// than inline in the filer entry, so a copy that reuses the source fids is visible
|
||||
// once those needles are freed.
|
||||
func chunkedTestContent(size int) []byte {
|
||||
content := make([]byte, size)
|
||||
for i := range content {
|
||||
content[i] = byte(i * 31 % 251)
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
// TestVersioningSelfCopyMetadataReplaceKeepsChunksIndependent covers the copy that
|
||||
// only rewrites metadata: it used to hand the source version's chunk fids to the
|
||||
// new version, so nothing owned those needles and deleting either version freed
|
||||
// the survivor's data (silently, once a vacuum ran).
|
||||
func TestVersioningSelfCopyMetadataReplaceKeepsChunksIndependent(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
objectKey := "self-copy-chunk-ownership.bin"
|
||||
content := chunkedTestContent(6 << 20)
|
||||
|
||||
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
copyResp, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(versioningCopySource(bucketName, objectKey)),
|
||||
Metadata: map[string]string{"mtime": "1653465360"},
|
||||
MetadataDirective: types.MetadataDirectiveReplace,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, copyResp.VersionId)
|
||||
require.NotEqual(t, *putResp.VersionId, *copyResp.VersionId)
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// The filer frees a deleted entry's chunks asynchronously, so re-check across
|
||||
// a few vacuum rounds instead of racing a single one.
|
||||
for round := 0; round < 4; round++ {
|
||||
time.Sleep(time.Second)
|
||||
vacuumVolumes(t)
|
||||
requireVersionBody(t, client, bucketName, objectKey, *copyResp.VersionId, content,
|
||||
"the surviving version must keep its own data after the other version is deleted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuspendedSelfCopyMetadataReplaceKeepsChunksIndependent is the same defect on
|
||||
// a suspended bucket: the null version the copy writes sits beside a .versions/
|
||||
// entry that stays live, so the two must not share needles either.
|
||||
func TestSuspendedSelfCopyMetadataReplaceKeepsChunksIndependent(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
objectKey := "suspended-self-copy-chunk-ownership.bin"
|
||||
content := chunkedTestContent(6 << 20)
|
||||
|
||||
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
suspendVersioning(t, client, bucketName)
|
||||
|
||||
_, err = client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(versioningCopySource(bucketName, objectKey)),
|
||||
Metadata: map[string]string{"mtime": "1653465360"},
|
||||
MetadataDirective: types.MetadataDirectiveReplace,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Drop the version the copy read from; the null version it wrote must survive.
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
for round := 0; round < 4; round++ {
|
||||
time.Sleep(time.Second)
|
||||
vacuumVolumes(t)
|
||||
requireVersionBody(t, client, bucketName, objectKey, "null", content,
|
||||
"the null version must keep its own data after the version it was copied from is deleted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersioningSelfCopyMetadataReplaceCreatesNewVersion(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
@@ -296,162 +145,3 @@ func TestVersioningSelfCopyMetadataReplaceSuspendedKeepsNullVersion(t *testing.T
|
||||
assert.Equal(t, "null", *versionsResp.Versions[0].VersionId, "Suspended self-copy should preserve null-version semantics")
|
||||
assert.True(t, *versionsResp.Versions[0].IsLatest, "Null version should remain latest")
|
||||
}
|
||||
|
||||
func TestVersioningSelfCopyCreatesNewVersion(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
objectKey := "self-copy-no-directive.txt"
|
||||
firstContent := []byte("first")
|
||||
secondContent := []byte("second")
|
||||
|
||||
firstPut, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(firstContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, firstPut.VersionId)
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(secondContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Copying an earlier version onto its own key is how AWS restores that version.
|
||||
copyResp, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s?versionId=%s", versioningCopySource(bucketName, objectKey), *firstPut.VersionId)),
|
||||
})
|
||||
require.NoError(t, err, "Self-copy of an earlier version should succeed on a versioned bucket")
|
||||
require.NotNil(t, copyResp.VersionId)
|
||||
assert.NotEqual(t, *firstPut.VersionId, *copyResp.VersionId, "Restore should write a new version")
|
||||
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, firstContent, body, "Restored version should serve the earlier body")
|
||||
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, versionsResp.Versions, 3, "Restore should append to the version history")
|
||||
}
|
||||
|
||||
func TestSelfCopyWithoutVersioningIsRejected(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "self-copy-unversioned.txt"
|
||||
putObject(t, client, bucketName, objectKey, "content")
|
||||
|
||||
_, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(versioningCopySource(bucketName, objectKey)),
|
||||
})
|
||||
require.Error(t, err, "Self-copy without a metadata change is a no-op on an unversioned bucket")
|
||||
var apiErr smithy.APIError
|
||||
if assert.True(t, errors.As(err, &apiErr), "Expected a smithy.APIError, but got %T", err) {
|
||||
assert.Equal(t, "InvalidRequest", apiErr.ErrorCode())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelfCopyWithSuspendedVersioningIsRejected(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
suspendVersioning(t, client, bucketName)
|
||||
|
||||
objectKey := "self-copy-suspended-no-directive.txt"
|
||||
putObject(t, client, bucketName, objectKey, "content")
|
||||
|
||||
_, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(versioningCopySource(bucketName, objectKey)),
|
||||
})
|
||||
require.Error(t, err, "Suspended versioning overwrites the null version in place, so the copy changes nothing")
|
||||
var apiErr smithy.APIError
|
||||
if assert.True(t, errors.As(err, &apiErr), "Expected a smithy.APIError, but got %T", err) {
|
||||
assert.Equal(t, "InvalidRequest", apiErr.ErrorCode())
|
||||
}
|
||||
}
|
||||
|
||||
// A suspended-versioning CopyObject writes the null version at the regular path, so
|
||||
// like PutObject and multipart completion it has to retire the null delete marker a
|
||||
// preceding DELETE left in .versions. While the regular-path object owns the null
|
||||
// slot the leftover marker is shadowed, but it resurfaces as a phantom delete the
|
||||
// moment that null version goes away.
|
||||
func TestSuspendedCopyRetiresDeleteMarker(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
sourceKey := "suspended-copy-source.txt"
|
||||
objectKey := "suspended-copy-dest.txt"
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
putObject(t, client, bucketName, objectKey, "pre-suspension-content")
|
||||
suspendVersioning(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, sourceKey, "source-content")
|
||||
putObject(t, client, bucketName, objectKey, "null-version-content")
|
||||
deleteKey(t, client, bucketName, objectKey)
|
||||
|
||||
_, err := client.CopyObject(context.TODO(), &s3.CopyObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(versioningCopySource(bucketName, sourceKey)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "source-content", string(body))
|
||||
|
||||
// Drop the null version the copy just wrote; a retired marker leaves nothing behind.
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String("null"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, listResp.DeleteMarkers, "the copy should have retired the null delete marker")
|
||||
}
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Backup clients probe and retract lock keys constantly, so they delete keys and
|
||||
// versions that may already be gone. S3 makes those deletes succeed; answering an
|
||||
// error instead turns routine lock arbitration into a job failure. Object lock is
|
||||
// enabled here because the retention check runs before the delete and is the most
|
||||
// likely place for a missing object to be reported as an error.
|
||||
|
||||
func TestDeleteMissingObjectSucceeds(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// A key that never existed.
|
||||
resp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("never-written.lock"),
|
||||
})
|
||||
require.NoError(t, err, "deleting a key that never existed must succeed")
|
||||
require.NotNil(t, resp.DeleteMarker, "a versioned bucket records the delete as a marker")
|
||||
assert.True(t, *resp.DeleteMarker)
|
||||
require.NotNil(t, resp.VersionId)
|
||||
|
||||
// The response can claim a marker the backend never stored, so confirm it is
|
||||
// really in the version history under the id the response handed back.
|
||||
versions, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String("never-written.lock"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, m := range versions.DeleteMarkers {
|
||||
if m.Key != nil && *m.Key == "never-written.lock" && m.VersionId != nil && *m.VersionId == *resp.VersionId {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "the delete marker reported by the response must be persisted")
|
||||
}
|
||||
|
||||
func TestDeleteAbsentVersionSucceeds(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "arbitration.lock"
|
||||
put, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("held")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, put.VersionId)
|
||||
|
||||
// Well-formed version id that names no stored version. Deleting it is a no-op,
|
||||
// not an error, and must not disturb the version that does exist.
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String("deadbeefdeadbeefdeadbeefdeadbeef"),
|
||||
})
|
||||
require.NoError(t, err, "deleting a version that does not exist must succeed")
|
||||
|
||||
head, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: put.VersionId,
|
||||
})
|
||||
require.NoError(t, err, "the real version must survive the no-op delete")
|
||||
require.NotNil(t, head.VersionId)
|
||||
assert.Equal(t, *put.VersionId, *head.VersionId)
|
||||
|
||||
// Repeating the delete of the real version is also a no-op rather than an error.
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: put.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Confirm the first delete actually removed the version. Without this a
|
||||
// backend that ignored every version-specific delete would satisfy both calls
|
||||
// and the idempotency the test claims to cover would never be exercised.
|
||||
_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: put.VersionId,
|
||||
})
|
||||
require.Error(t, err, "the version must be gone after the first delete")
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: put.VersionId,
|
||||
})
|
||||
require.NoError(t, err, "repeating a version delete must stay a no-op")
|
||||
}
|
||||
|
||||
// A missing key in the middle of a batch must not consume another key's result
|
||||
// slot: every request key has to come back under its own name.
|
||||
func TestMultiObjectDeleteAttributesMissingKeys(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
present := []string{"batch-1.lock", "batch-3.lock"}
|
||||
for _, key := range present {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("held")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
requested := []string{"batch-1.lock", "batch-2-missing.lock", "batch-3.lock"}
|
||||
objects := make([]types.ObjectIdentifier, 0, len(requested))
|
||||
for _, key := range requested {
|
||||
objects = append(objects, types.ObjectIdentifier{Key: aws.String(key)})
|
||||
}
|
||||
|
||||
resp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Delete: &types.Delete{Objects: objects},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, resp.Errors, "a missing key is not a batch error")
|
||||
require.Len(t, resp.Deleted, len(requested))
|
||||
|
||||
got := make(map[string]bool, len(resp.Deleted))
|
||||
for _, d := range resp.Deleted {
|
||||
require.NotNil(t, d.Key, "every result row must carry the key it belongs to")
|
||||
assert.NotEmpty(t, *d.Key, "an empty key means a row lost its identity")
|
||||
got[*d.Key] = true
|
||||
}
|
||||
for _, key := range requested {
|
||||
assert.True(t, got[key], "no result row for %s", key)
|
||||
}
|
||||
}
|
||||
@@ -1,224 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDeletedDirectoryMarkerDisappears covers the rclone directory_markers flow: a key
|
||||
// created with PutObject on "m2/" is deleted, and stops being a key everywhere. The
|
||||
// directory that carried it goes with it once nothing is left underneath.
|
||||
func TestDeletedDirectoryMarkerDisappears(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, "m2/", "")
|
||||
putObject(t, client, bucketName, "m2/f.txt", "hi")
|
||||
assert.Equal(t, []string{"m2/", "m2/f.txt"}, listKeys(t, client, bucketName, ""))
|
||||
|
||||
deleteKey(t, client, bucketName, "m2/f.txt")
|
||||
deleteKey(t, client, bucketName, "m2/")
|
||||
|
||||
assert.Empty(t, listKeys(t, client, bucketName, ""), "the deleted marker is not a key")
|
||||
assert.Empty(t, listPrefixes(t, client, bucketName, ""), "and it names no prefix")
|
||||
|
||||
// The file's own version history is untouched by deleting the directory key.
|
||||
versions, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for _, v := range versions.Versions {
|
||||
assert.NotEqual(t, "m2/", *v.Key, "a directory marker is not a versioned object")
|
||||
}
|
||||
for _, m := range versions.DeleteMarkers {
|
||||
assert.NotEqual(t, "m2/", *m.Key, "deleting one writes no delete marker")
|
||||
}
|
||||
assert.Len(t, versions.Versions, 1, "m2/f.txt keeps its version")
|
||||
assert.Len(t, versions.DeleteMarkers, 1, "and its delete marker")
|
||||
|
||||
// Re-creating the marker brings the key back.
|
||||
putObject(t, client, bucketName, "m2/", "")
|
||||
assert.Equal(t, []string{"m2/"}, listKeys(t, client, bucketName, ""))
|
||||
_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestDeletedDirectoryMarkerKeepsItsSubtree pins the boundary: deleting the key "m2/"
|
||||
// says nothing about the objects under it, which keep listing and keep the prefix.
|
||||
func TestDeletedDirectoryMarkerKeepsItsSubtree(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, "m2/", "")
|
||||
putObject(t, client, bucketName, "m2/keep.txt", "still here")
|
||||
|
||||
deleteKey(t, client, bucketName, "m2/")
|
||||
|
||||
assert.Equal(t, []string{"m2/keep.txt"}, listKeys(t, client, bucketName, ""), "the marker key is gone")
|
||||
assert.Equal(t, []string{"m2/"}, listPrefixes(t, client, bucketName, ""),
|
||||
"a live child keeps the prefix")
|
||||
assert.Equal(t, []string{"m2/keep.txt"}, listKeys(t, client, bucketName, "m2/"))
|
||||
|
||||
// The surviving object is still readable through the prefix that no longer has a key.
|
||||
got, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/keep.txt"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, got.Body.Close())
|
||||
}
|
||||
|
||||
// TestDeletedDirectoryMarkerIsGoneFromReads checks the read surfaces once nothing is
|
||||
// left under the deleted key: the directory goes with it, so the path is not there.
|
||||
func TestDeletedDirectoryMarkerIsGoneFromReads(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, "m2/", "")
|
||||
deleteKey(t, client, bucketName, "m2/")
|
||||
|
||||
_, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
require.Error(t, err, "HEAD on a deleted directory marker must not answer 200")
|
||||
_, err = client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
requireAPIError(t, err, "NoSuchKey")
|
||||
assert.Empty(t, listKeys(t, client, bucketName, "m2/"))
|
||||
}
|
||||
|
||||
// TestDirectoryMarkerDeleteMatchesUnversioned pins the point of the change: a directory
|
||||
// marker is deleted the same way whether or not the bucket is versioned.
|
||||
func TestDirectoryMarkerDeleteMatchesUnversioned(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
|
||||
for _, versioned := range []bool{false, true} {
|
||||
bucketName := getNewBucketName()
|
||||
createBucket(t, client, bucketName)
|
||||
if versioned {
|
||||
enableVersioning(t, client, bucketName)
|
||||
}
|
||||
|
||||
putObject(t, client, bucketName, "m/", "")
|
||||
putObject(t, client, bucketName, "m/child.txt", "x")
|
||||
deleteKey(t, client, bucketName, "m/")
|
||||
|
||||
assert.Equal(t, []string{"m/child.txt"}, listKeys(t, client, bucketName, ""),
|
||||
"versioned=%v: the marker key is gone, the child stays", versioned)
|
||||
assert.Equal(t, []string{"m/"}, listPrefixes(t, client, bucketName, ""),
|
||||
"versioned=%v: the prefix survives its live child", versioned)
|
||||
|
||||
deleteBucket(t, client, bucketName)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteDirectoryMarkerSparesAPromotedFile guards the sharp edge of storing a key
|
||||
// on a directory entry: writing under an existing object turns that object's entry into
|
||||
// a directory while it keeps its data, so "m2/" and "m2" end up on the same entry. They
|
||||
// are still different keys, and deleting one must not destroy the other.
|
||||
func TestDeleteDirectoryMarkerSparesAPromotedFile(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
|
||||
for _, versioned := range []bool{false, true} {
|
||||
bucketName := getNewBucketName()
|
||||
createBucket(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, "m2", "important data")
|
||||
if versioned {
|
||||
enableVersioning(t, client, bucketName)
|
||||
}
|
||||
putObject(t, client, bucketName, "m2/child.txt", "child")
|
||||
|
||||
deleteKey(t, client, bucketName, "m2/")
|
||||
|
||||
got, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2"),
|
||||
})
|
||||
require.NoError(t, err, "versioned=%v: deleting m2/ must not delete m2", versioned)
|
||||
body, err := io.ReadAll(got.Body)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, got.Body.Close())
|
||||
assert.Equal(t, "important data", string(body), "versioned=%v", versioned)
|
||||
|
||||
deleteBucket(t, client, bucketName)
|
||||
}
|
||||
}
|
||||
|
||||
func listObjects(t *testing.T, client *s3.Client, bucketName, prefix, delimiter string) *s3.ListObjectsV2Output {
|
||||
t.Helper()
|
||||
input := &s3.ListObjectsV2Input{Bucket: aws.String(bucketName)}
|
||||
if prefix != "" {
|
||||
input.Prefix = aws.String(prefix)
|
||||
}
|
||||
if delimiter != "" {
|
||||
input.Delimiter = aws.String(delimiter)
|
||||
}
|
||||
resp, err := client.ListObjectsV2(context.TODO(), input)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
// listKeys lists without a delimiter, so a directory marker shows as the key it is
|
||||
// instead of rolling up into its own CommonPrefix.
|
||||
func listKeys(t *testing.T, client *s3.Client, bucketName, prefix string) []string {
|
||||
t.Helper()
|
||||
resp := listObjects(t, client, bucketName, prefix, "")
|
||||
keys := make([]string, 0, len(resp.Contents))
|
||||
for _, c := range resp.Contents {
|
||||
keys = append(keys, *c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func listPrefixes(t *testing.T, client *s3.Client, bucketName, prefix string) []string {
|
||||
t.Helper()
|
||||
resp := listObjects(t, client, bucketName, prefix, "/")
|
||||
prefixes := make([]string, 0, len(resp.CommonPrefixes))
|
||||
for _, p := range resp.CommonPrefixes {
|
||||
prefixes = append(prefixes, *p.Prefix)
|
||||
}
|
||||
return prefixes
|
||||
}
|
||||
|
||||
func deleteKey(t *testing.T, client *s3.Client, bucketName, key string) {
|
||||
t.Helper()
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func requireAPIError(t *testing.T, err error, code string) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
var apiErr smithy.APIError
|
||||
require.ErrorAs(t, err, &apiErr)
|
||||
assert.Equal(t, code, apiErr.ErrorCode())
|
||||
}
|
||||
@@ -1,179 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A listing walks entries and drops the ones whose current version is a delete
|
||||
// marker. When a whole run of consecutive entries drops out, the page it was
|
||||
// filling can come back empty — and an empty page is easily mistaken for the end
|
||||
// of the listing. Everything after the run then never appears, and the caller is
|
||||
// told the objects do not exist. Backup repositories hit this shape constantly:
|
||||
// they retract a batch of keys under one prefix and keep writing under the next.
|
||||
|
||||
func putObjectVersioned(t *testing.T, client *s3.Client, bucket, key string) {
|
||||
t.Helper()
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("x")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func listAllKeys(t *testing.T, client *s3.Client, bucket string, maxKeys int32) []string {
|
||||
t.Helper()
|
||||
var keys []string
|
||||
var token *string
|
||||
for {
|
||||
in := &s3.ListObjectsV2Input{Bucket: aws.String(bucket), ContinuationToken: token}
|
||||
if maxKeys > 0 {
|
||||
in.MaxKeys = aws.Int32(maxKeys)
|
||||
}
|
||||
resp, err := client.ListObjectsV2(context.TODO(), in)
|
||||
require.NoError(t, err)
|
||||
for _, o := range resp.Contents {
|
||||
require.NotNil(t, o.Key)
|
||||
keys = append(keys, *o.Key)
|
||||
}
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
require.NotNil(t, resp.NextContinuationToken, "a truncated listing must carry a continuation token")
|
||||
token = resp.NextContinuationToken
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func TestListingSurvivesRunOfHiddenVersions(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// A long run of retracted keys, sorting before the live ones.
|
||||
const hidden = 25
|
||||
for i := 0; i < hidden; i++ {
|
||||
key := fmt.Sprintf("aretracted/%03d.lock", i)
|
||||
putObjectVersioned(t, client, bucketName, key)
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
var live []string
|
||||
for i := 0; i < 5; i++ {
|
||||
key := fmt.Sprintf("blive/%03d.blk", i)
|
||||
putObjectVersioned(t, client, bucketName, key)
|
||||
live = append(live, key)
|
||||
}
|
||||
sort.Strings(live)
|
||||
|
||||
// Unpaginated: the retracted run must not hide what follows it.
|
||||
assert.Equal(t, live, listAllKeys(t, client, bucketName, 0),
|
||||
"a run of retracted keys must not truncate the listing")
|
||||
|
||||
// Paginated with pages smaller than the retracted run, so at least one page
|
||||
// is filled entirely from entries that get dropped.
|
||||
for _, pageSize := range []int32{1, 2, 3, 5, 10} {
|
||||
t.Run(fmt.Sprintf("maxKeys=%d", pageSize), func(t *testing.T) {
|
||||
assert.Equal(t, live, listAllKeys(t, client, bucketName, pageSize),
|
||||
"pagination across a retracted run must not lose the keys after it")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The same shape with the retracted run in the middle, so both a preceding and a
|
||||
// following key have to survive it.
|
||||
func TestListingSurvivesHiddenVersionsBetweenLiveKeys(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
putObjectVersioned(t, client, bucketName, "a-first.blk")
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
key := fmt.Sprintf("m-retracted/%03d.lock", i)
|
||||
putObjectVersioned(t, client, bucketName, key)
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
putObjectVersioned(t, client, bucketName, "z-last.blk")
|
||||
|
||||
want := []string{"a-first.blk", "z-last.blk"}
|
||||
for _, pageSize := range []int32{0, 1, 2, 3, 5, 10} {
|
||||
t.Run(fmt.Sprintf("maxKeys=%d", pageSize), func(t *testing.T) {
|
||||
assert.Equal(t, want, listAllKeys(t, client, bucketName, pageSize),
|
||||
"keys on both sides of a retracted run must both be listed")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ListObjectVersions sees the same namespace from the other side: the retracted
|
||||
// keys are still there as versions plus delete markers, and must all be reported.
|
||||
func TestListObjectVersionsReportsRetractedKeys(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
const retracted = 10
|
||||
for i := 0; i < retracted; i++ {
|
||||
key := fmt.Sprintf("retracted/%03d.lock", i)
|
||||
putObjectVersioned(t, client, bucketName, key)
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Paginated with a page far smaller than the run, so the continuation markers
|
||||
// have to carry the walk across it — the same path the plain listing exercises
|
||||
// above, but on the version side.
|
||||
var versions, markers int
|
||||
var keyMarker, versionIDMarker *string
|
||||
for {
|
||||
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
MaxKeys: aws.Int32(3),
|
||||
KeyMarker: keyMarker,
|
||||
VersionIdMarker: versionIDMarker,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
versions += len(resp.Versions)
|
||||
markers += len(resp.DeleteMarkers)
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
require.NotNil(t, resp.NextKeyMarker, "a truncated version listing must carry a key marker")
|
||||
keyMarker, versionIDMarker = resp.NextKeyMarker, resp.NextVersionIdMarker
|
||||
}
|
||||
|
||||
assert.Equal(t, retracted, versions, "every written version must still be reported")
|
||||
assert.Equal(t, retracted, markers, "every retraction must be reported as a delete marker")
|
||||
|
||||
// And the current-version view of the same namespace is empty.
|
||||
assert.Empty(t, listAllKeys(t, client, bucketName, 0),
|
||||
"no key should be current once every one has been retracted")
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Version ids that can never name a stored version. A request carrying one has
|
||||
// to be refused: resolving it to the null or latest version instead would let a
|
||||
// caller destroy a live version by asking for one that does not exist.
|
||||
var unusableVersionIDs = []string{"a/b", `a\b`, "..", "."}
|
||||
|
||||
func statusOf(t *testing.T, err error) int {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
var respErr *smithyhttp.ResponseError
|
||||
require.True(t, errors.As(err, &respErr), "expected an HTTP response error, got %v", err)
|
||||
return respErr.HTTPStatusCode()
|
||||
}
|
||||
|
||||
func TestUnusableVersionIDIsRefusedNotResolved(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "arbitration.lock"
|
||||
put, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("held")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, put.VersionId)
|
||||
|
||||
for _, versionID := range unusableVersionIDs {
|
||||
t.Run("delete "+versionID, func(t *testing.T) {
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(versionID),
|
||||
})
|
||||
// 400, not merely "some 4xx": the id is malformed, which is a bad
|
||||
// argument rather than a missing resource. A 5xx would be the real
|
||||
// damage — it invites endless retries of a request that can never
|
||||
// succeed — but pinning the exact code keeps the contract honest.
|
||||
assert.Equal(t, 400, statusOf(t, err))
|
||||
})
|
||||
}
|
||||
|
||||
// The point of the whole test: none of those requests may have been resolved
|
||||
// to the version that does exist.
|
||||
head, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: put.VersionId,
|
||||
})
|
||||
require.NoError(t, err, "the stored version must survive every refused request")
|
||||
require.NotNil(t, head.VersionId)
|
||||
assert.Equal(t, *put.VersionId, *head.VersionId)
|
||||
|
||||
get, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "the latest version must still resolve")
|
||||
defer get.Body.Close()
|
||||
}
|
||||
|
||||
// Read paths must agree with the write path: the same id is unusable everywhere,
|
||||
// so a caller cannot be told the object is missing by one verb and served by another.
|
||||
func TestUnusableVersionIDRefusedOnReadPaths(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "arbitration.lock"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("held")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, versionID := range unusableVersionIDs {
|
||||
t.Run("get "+versionID, func(t *testing.T) {
|
||||
_, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(versionID),
|
||||
})
|
||||
assert.Equal(t, 400, statusOf(t, err))
|
||||
})
|
||||
t.Run("head "+versionID, func(t *testing.T) {
|
||||
_, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(versionID),
|
||||
})
|
||||
assert.Equal(t, 400, statusOf(t, err))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ListObjects and ListObjectVersions walk the same namespace and must describe it
|
||||
// the same way. They are separate code paths, so a client that navigates by
|
||||
// versioned listings can see a different tree than one navigating by plain
|
||||
// listings — and concludes keys are missing that are plainly there. Testing the
|
||||
// two paths independently never catches that; only comparing them does.
|
||||
|
||||
// listingParityTree is shaped after a backup repository: nested prefixes, a file
|
||||
// beside a prefix at the same level, and a key that is both an object and the
|
||||
// parent of other keys.
|
||||
var listingParityTree = []string{
|
||||
"backup/archive/001/data.blk",
|
||||
"backup/archive/002/data.blk",
|
||||
"backup/archive/summary.xml",
|
||||
"backup/meta",
|
||||
"backup/meta/nested.txt",
|
||||
"backup/index",
|
||||
}
|
||||
|
||||
func currentKeysAndPrefixes(t *testing.T, client *s3.Client, bucket, prefix, delimiter string) ([]string, []string) {
|
||||
t.Helper()
|
||||
in := &s3.ListObjectsV2Input{Bucket: aws.String(bucket)}
|
||||
if prefix != "" {
|
||||
in.Prefix = aws.String(prefix)
|
||||
}
|
||||
if delimiter != "" {
|
||||
in.Delimiter = aws.String(delimiter)
|
||||
}
|
||||
resp, err := client.ListObjectsV2(context.TODO(), in)
|
||||
require.NoError(t, err)
|
||||
// One page only. A truncated response would silently compare partial results
|
||||
// against partial results and report parity that was never checked.
|
||||
require.False(t, aws.ToBool(resp.IsTruncated), "test data must fit one page")
|
||||
|
||||
var keys, prefixes []string
|
||||
for _, o := range resp.Contents {
|
||||
require.NotNil(t, o.Key)
|
||||
keys = append(keys, *o.Key)
|
||||
}
|
||||
for _, p := range resp.CommonPrefixes {
|
||||
require.NotNil(t, p.Prefix)
|
||||
prefixes = append(prefixes, *p.Prefix)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
sort.Strings(prefixes)
|
||||
return keys, prefixes
|
||||
}
|
||||
|
||||
func currentKeysAndPrefixesFromVersions(t *testing.T, client *s3.Client, bucket, prefix, delimiter string) ([]string, []string) {
|
||||
t.Helper()
|
||||
in := &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)}
|
||||
if prefix != "" {
|
||||
in.Prefix = aws.String(prefix)
|
||||
}
|
||||
if delimiter != "" {
|
||||
in.Delimiter = aws.String(delimiter)
|
||||
}
|
||||
resp, err := client.ListObjectVersions(context.TODO(), in)
|
||||
require.NoError(t, err)
|
||||
require.False(t, aws.ToBool(resp.IsTruncated), "test data must fit one page")
|
||||
|
||||
// Reduce the version view to the same thing ListObjects reports: keys whose
|
||||
// current version is real content, not a delete marker.
|
||||
//
|
||||
// Delete markers are only visible here as individual entries. Under a
|
||||
// delimiter, keys grouped into a common prefix do not appear individually, so
|
||||
// this cannot tell that every key beneath a prefix is delete-marked — such a
|
||||
// prefix stays in the returned list. Comparisons that need that case must
|
||||
// query the prefix directly rather than relying on the grouped view.
|
||||
deleted := make(map[string]bool)
|
||||
for _, m := range resp.DeleteMarkers {
|
||||
if m.Key != nil && m.IsLatest != nil && *m.IsLatest {
|
||||
deleted[*m.Key] = true
|
||||
}
|
||||
}
|
||||
var keys, prefixes []string
|
||||
for _, v := range resp.Versions {
|
||||
if v.Key == nil || v.IsLatest == nil || !*v.IsLatest {
|
||||
continue
|
||||
}
|
||||
if deleted[*v.Key] {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, *v.Key)
|
||||
}
|
||||
for _, p := range resp.CommonPrefixes {
|
||||
require.NotNil(t, p.Prefix)
|
||||
prefixes = append(prefixes, *p.Prefix)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
sort.Strings(prefixes)
|
||||
return keys, prefixes
|
||||
}
|
||||
|
||||
func TestListingParityAcrossPrefixesAndDelimiter(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
for _, key := range listingParityTree {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("x")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
prefix string
|
||||
delimiter string
|
||||
}{
|
||||
{"root delimited", "", "/"},
|
||||
{"root undelimited", "", ""},
|
||||
{"one level delimited", "backup/", "/"},
|
||||
{"one level undelimited", "backup/", ""},
|
||||
{"two levels delimited", "backup/archive/", "/"},
|
||||
{"leaf prefix delimited", "backup/archive/001/", "/"},
|
||||
// The prefix names an object exactly. Both listings must return that
|
||||
// object and must not descend into anything below it.
|
||||
{"prefix is exactly an object", "backup/archive/summary.xml", "/"},
|
||||
{"prefix is exactly an object undelimited", "backup/archive/summary.xml", ""},
|
||||
// A key that is also a parent: "backup/meta" is an object and
|
||||
// "backup/meta/" is a prefix. The two listings must not disagree about
|
||||
// which of those the client is looking at.
|
||||
{"object that is also a prefix", "backup/meta", "/"},
|
||||
{"prefix form of that object", "backup/meta/", "/"},
|
||||
{"partial key fragment", "backup/ind", "/"},
|
||||
{"prefix matching nothing", "backup/nothing/", "/"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
plainKeys, plainPrefixes := currentKeysAndPrefixes(t, client, bucketName, tc.prefix, tc.delimiter)
|
||||
versionKeys, versionPrefixes := currentKeysAndPrefixesFromVersions(t, client, bucketName, tc.prefix, tc.delimiter)
|
||||
|
||||
assert.Equal(t, plainKeys, versionKeys,
|
||||
"prefix=%q delimiter=%q: ListObjectVersions reports a different set of current keys than ListObjects", tc.prefix, tc.delimiter)
|
||||
assert.Equal(t, plainPrefixes, versionPrefixes,
|
||||
"prefix=%q delimiter=%q: ListObjectVersions reports different common prefixes than ListObjects", tc.prefix, tc.delimiter)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting every key under a prefix must retire the prefix from both listings
|
||||
// together. A prefix that lingers in one view sends a client looking for keys the
|
||||
// other view says are gone.
|
||||
func TestListingParityAfterPrefixEmptied(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
keys := []string{"backup/archive/001/data.blk", "backup/archive/002/data.blk"}
|
||||
versionIDs := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
put, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("x")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, put.VersionId)
|
||||
versionIDs = append(versionIDs, *put.VersionId)
|
||||
}
|
||||
|
||||
plainKeys, plainPrefixes := currentKeysAndPrefixes(t, client, bucketName, "backup/archive/", "/")
|
||||
versionKeys, versionPrefixes := currentKeysAndPrefixesFromVersions(t, client, bucketName, "backup/archive/", "/")
|
||||
require.Equal(t, plainKeys, versionKeys)
|
||||
require.Equal(t, plainPrefixes, versionPrefixes)
|
||||
require.Len(t, plainPrefixes, 2, "both leaf prefixes should be visible while populated")
|
||||
|
||||
// Remove the versions outright rather than layering delete markers, so the
|
||||
// containers are left genuinely empty.
|
||||
for i, key := range keys {
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: aws.String(versionIDs[i]),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
plainKeys, plainPrefixes = currentKeysAndPrefixes(t, client, bucketName, "backup/archive/", "/")
|
||||
versionKeys, versionPrefixes = currentKeysAndPrefixesFromVersions(t, client, bucketName, "backup/archive/", "/")
|
||||
|
||||
// Both views must still describe the same namespace. Whether an emptied
|
||||
// prefix is retired promptly is a separate question from whether the two
|
||||
// listings agree about it, and only the latter is under test here.
|
||||
assert.Equal(t, plainKeys, versionKeys, "listings disagree on remaining keys after the prefix was emptied")
|
||||
assert.Equal(t, plainPrefixes, versionPrefixes, "listings disagree on remaining common prefixes after the prefix was emptied")
|
||||
// Asserted on both sides rather than leaning on the equality above to catch a
|
||||
// divergence in whichever direction it happens.
|
||||
assert.Empty(t, plainKeys, "no current keys should remain once every version is gone")
|
||||
assert.Empty(t, versionKeys, "no current keys should remain once every version is gone")
|
||||
}
|
||||
|
||||
// The other way a key stops being current: a delete marker rather than removal of
|
||||
// the version. Both listings must drop the key while its history stays intact.
|
||||
func TestListingParityAfterDeleteMarker(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
const marked = "backup/archive/001/data.blk"
|
||||
const kept = "backup/archive/002/data.blk"
|
||||
for _, key := range []string{marked, kept} {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("x")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// No version id: this layers a delete marker instead of removing the version.
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(marked),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, tc := range []struct{ prefix, delimiter string }{
|
||||
{"backup/archive/", ""},
|
||||
{"backup/archive/001/", "/"},
|
||||
{"backup/archive/001/", ""},
|
||||
} {
|
||||
t.Run(tc.prefix+"|"+tc.delimiter, func(t *testing.T) {
|
||||
plainKeys, plainPrefixes := currentKeysAndPrefixes(t, client, bucketName, tc.prefix, tc.delimiter)
|
||||
versionKeys, versionPrefixes := currentKeysAndPrefixesFromVersions(t, client, bucketName, tc.prefix, tc.delimiter)
|
||||
|
||||
assert.Equal(t, plainKeys, versionKeys,
|
||||
"prefix=%q delimiter=%q: listings disagree on current keys after a delete marker", tc.prefix, tc.delimiter)
|
||||
assert.Equal(t, plainPrefixes, versionPrefixes,
|
||||
"prefix=%q delimiter=%q: listings disagree on common prefixes after a delete marker", tc.prefix, tc.delimiter)
|
||||
assert.NotContains(t, plainKeys, marked, "a delete-marked key is not current")
|
||||
})
|
||||
}
|
||||
|
||||
// The history the marker hides must still be there.
|
||||
versions, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(marked),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, versions.Versions, 1, "the superseded version must survive its delete marker")
|
||||
assert.Len(t, versions.DeleteMarkers, 1)
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Backup clients arbitrate ownership by writing and retracting small keys under a
|
||||
// fixed prefix, then re-probing them. Each probe uses a different verb, and the
|
||||
// client trusts them to agree: a key reported present by one and absent by another
|
||||
// makes it either spin or conclude the repository is corrupt. The keys are written
|
||||
// and immediately deleted by version, which is the cycle that leaves versioning
|
||||
// residue behind, so parity has to hold on both sides of the delete.
|
||||
|
||||
var lockArbitrationKeys = []string{
|
||||
"Metadata/Lock/create.checkpoint/lock",
|
||||
"Metadata/Lock/create.checkpoint/try",
|
||||
"Metadata/Lock/create.checkpoint/declare",
|
||||
"Metadata/Lock/delete.checkpoint/lock",
|
||||
}
|
||||
|
||||
// existence reports what each verb believes about a key, so a disagreement names
|
||||
// the verbs rather than just failing.
|
||||
type existence struct {
|
||||
get bool
|
||||
head bool
|
||||
listObjects bool
|
||||
listVersion bool
|
||||
}
|
||||
|
||||
func (e existence) agree() bool {
|
||||
return e.get == e.head && e.head == e.listObjects && e.listObjects == e.listVersion
|
||||
}
|
||||
|
||||
func notFound(t *testing.T, err error) bool {
|
||||
t.Helper()
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var respErr *smithyhttp.ResponseError
|
||||
if errors.As(err, &respErr) {
|
||||
require.Less(t, respErr.HTTPStatusCode(), 500, "probing a lock key must never fault the server: %v", err)
|
||||
return respErr.HTTPStatusCode() == 404
|
||||
}
|
||||
t.Fatalf("unexpected error probing lock key: %v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
func probe(t *testing.T, client *s3.Client, bucket, key string) existence {
|
||||
t.Helper()
|
||||
var e existence
|
||||
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
if err == nil {
|
||||
// Drain before closing so the connection goes back to the pool; these
|
||||
// probes run in tight loops and a discarded body forces a new connection.
|
||||
_, _ = io.Copy(io.Discard, getResp.Body)
|
||||
getResp.Body.Close()
|
||||
e.get = true
|
||||
} else {
|
||||
require.True(t, notFound(t, err), "GET must answer present or 404, got %v", err)
|
||||
}
|
||||
|
||||
if _, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
}); err == nil {
|
||||
e.head = true
|
||||
} else {
|
||||
require.True(t, notFound(t, err), "HEAD must answer present or 404, got %v", err)
|
||||
}
|
||||
|
||||
listed, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket), Prefix: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for _, obj := range listed.Contents {
|
||||
if obj.Key != nil && *obj.Key == key {
|
||||
e.listObjects = true
|
||||
}
|
||||
}
|
||||
|
||||
versions, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucket), Prefix: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for _, v := range versions.Versions {
|
||||
// A version is only "the object exists" if it is the current one; a
|
||||
// superseded version coexists with a delete marker on a live key.
|
||||
if v.Key != nil && *v.Key == key && v.IsLatest != nil && *v.IsLatest {
|
||||
e.listVersion = true
|
||||
}
|
||||
}
|
||||
|
||||
return e
|
||||
}
|
||||
|
||||
func TestLockKeyVerbParityWhilePresent(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
for _, key := range lockArbitrationKeys {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("held")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
for _, key := range lockArbitrationKeys {
|
||||
t.Run(strings.ReplaceAll(key, "/", "_"), func(t *testing.T) {
|
||||
e := probe(t, client, bucketName, key)
|
||||
assert.True(t, e.agree(), "verbs disagree on a present lock key: %+v", e)
|
||||
assert.True(t, e.get, "lock key should be present: %+v", e)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The retract half of the cycle: write, then delete by version id — the sequence
|
||||
// that empties a version container. Every verb must then agree the key is gone.
|
||||
func TestLockKeyVerbParityAfterVersionDelete(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
for _, key := range lockArbitrationKeys {
|
||||
put, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("held")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, put.VersionId)
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: put.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
for _, key := range lockArbitrationKeys {
|
||||
t.Run(strings.ReplaceAll(key, "/", "_"), func(t *testing.T) {
|
||||
e := probe(t, client, bucketName, key)
|
||||
assert.True(t, e.agree(), "verbs disagree on a retracted lock key: %+v", e)
|
||||
assert.False(t, e.get, "retracted lock key should be absent: %+v", e)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Re-acquiring after a retract is the steady-state loop. Residue left by the
|
||||
// previous cycle must not make the new key invisible to any verb.
|
||||
func TestLockKeyVerbParityAcrossReacquireCycles(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
for _, key := range lockArbitrationKeys {
|
||||
t.Run(strings.ReplaceAll(key, "/", "_"), func(t *testing.T) {
|
||||
for cycle := 0; cycle < 3; cycle++ {
|
||||
put, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("held")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, put.VersionId)
|
||||
|
||||
held := probe(t, client, bucketName, key)
|
||||
require.True(t, held.agree(), "cycle %d: verbs disagree while held: %+v", cycle, held)
|
||||
require.True(t, held.get, "cycle %d: key should be present while held: %+v", cycle, held)
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: put.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
released := probe(t, client, bucketName, key)
|
||||
require.True(t, released.agree(), "cycle %d: verbs disagree after release: %+v", cycle, released)
|
||||
require.False(t, released.get, "cycle %d: key should be absent after release: %+v", cycle, released)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// A listing on a versioned bucket is served from metadata cached on the .versions
|
||||
// directory entry so the whole listing is one scan. Anything the cache does not
|
||||
// carry falls back to a default, so a field missing from the cache makes the
|
||||
// listing disagree with HEAD about the same object. Storage class is one such
|
||||
// field, and clients that filter or tier on it act on the listing.
|
||||
|
||||
func TestStorageClassConsistentBetweenHeadAndListings(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
cases := []struct {
|
||||
key string
|
||||
class types.StorageClass
|
||||
want string
|
||||
}{
|
||||
{"default.blk", "", "STANDARD"},
|
||||
{"standard.blk", types.StorageClassStandard, "STANDARD"},
|
||||
{"reduced.blk", types.StorageClassReducedRedundancy, "REDUCED_REDUNDANCY"},
|
||||
{"glacier.blk", types.StorageClassGlacier, "GLACIER"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
in := &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(tc.key),
|
||||
Body: bytes.NewReader([]byte("x")),
|
||||
}
|
||||
if tc.class != "" {
|
||||
in.StorageClass = tc.class
|
||||
}
|
||||
_, err := client.PutObject(context.TODO(), in)
|
||||
require.NoError(t, err, "PUT %s with class %q", tc.key, tc.class)
|
||||
}
|
||||
|
||||
listed, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{Bucket: aws.String(bucketName)})
|
||||
require.NoError(t, err)
|
||||
byKeyV2 := make(map[string]string, len(listed.Contents))
|
||||
for _, o := range listed.Contents {
|
||||
require.NotNil(t, o.Key)
|
||||
byKeyV2[*o.Key] = string(o.StorageClass)
|
||||
}
|
||||
|
||||
versions, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{Bucket: aws.String(bucketName)})
|
||||
require.NoError(t, err)
|
||||
byKeyVersions := make(map[string]string, len(versions.Versions))
|
||||
for _, v := range versions.Versions {
|
||||
require.NotNil(t, v.Key)
|
||||
if v.IsLatest != nil && *v.IsLatest {
|
||||
byKeyVersions[*v.Key] = string(v.StorageClass)
|
||||
}
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.key, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, byKeyV2[tc.key],
|
||||
"ListObjectsV2 must report the class the object was stored with")
|
||||
assert.Equal(t, tc.want, byKeyVersions[tc.key],
|
||||
"ListObjectVersions must report the class the object was stored with")
|
||||
|
||||
head, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(tc.key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
// S3 omits the header for STANDARD and sends it otherwise; either way
|
||||
// it must not contradict the listing.
|
||||
if head.StorageClass != "" {
|
||||
assert.Equal(t, tc.want, string(head.StorageClass),
|
||||
"HEAD and the listings must agree on storage class")
|
||||
} else {
|
||||
assert.Equal(t, "STANDARD", tc.want,
|
||||
"HEAD omits the header only for STANDARD")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Overwriting refreshes the cached listing metadata; the class must follow the
|
||||
// new current version rather than stay pinned to the one it replaced.
|
||||
func TestStorageClassFollowsLatestVersion(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
key := "rewritten.blk"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("first")),
|
||||
StorageClass: types.StorageClassGlacier,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName), Key: aws.String(key),
|
||||
Body: bytes.NewReader([]byte("second")),
|
||||
StorageClass: types.StorageClassReducedRedundancy,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
listed, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName), Prefix: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, listed.Contents, 1)
|
||||
assert.Equal(t, "REDUCED_REDUNDANCY", string(listed.Contents[0].StorageClass),
|
||||
"the listing must report the current version's class, not the one it replaced")
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -81,93 +80,3 @@ func TestSuspendedDeleteCreatesDeleteMarker(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "versioned-content", string(body))
|
||||
}
|
||||
|
||||
// A suspended-versioning completion must drop the null delete marker a preceding
|
||||
// DELETE left in .versions, or it reports 200 and the object lists while HEAD/GET
|
||||
// keep resolving the marker and answer NoSuchKey.
|
||||
func TestSuspendedMultipartOverwritesDeleteMarker(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "suspended-multipart-after-delete.bin"
|
||||
partData := bytes.Repeat([]byte("a"), 5*1024*1024)
|
||||
|
||||
// The cleanup retires only the null version, never the key's real history.
|
||||
enableVersioning(t, client, bucketName)
|
||||
realVersion := putObject(t, client, bucketName, objectKey, "pre-suspension-content")
|
||||
require.NotNil(t, realVersion.VersionId)
|
||||
suspendVersioning(t, client, bucketName)
|
||||
|
||||
completeSuspendedMultipart := func() {
|
||||
t.Helper()
|
||||
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
uploadResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(partData),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{{ETag: uploadResp.ETag, PartNumber: aws.Int32(1)}},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
completeSuspendedMultipart()
|
||||
deleteKey(t, client, bucketName, objectKey)
|
||||
completeSuspendedMultipart()
|
||||
|
||||
headResp := headObject(t, client, bucketName, objectKey)
|
||||
require.NotNil(t, headResp.ContentLength)
|
||||
assert.Equal(t, int64(len(partData)), *headResp.ContentLength)
|
||||
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
written, err := io.Copy(io.Discard, getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(partData)), written)
|
||||
|
||||
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, listResp.DeleteMarkers)
|
||||
|
||||
var listedVersionIds []string
|
||||
for _, version := range listResp.Versions {
|
||||
listedVersionIds = append(listedVersionIds, aws.ToString(version.VersionId))
|
||||
}
|
||||
assert.ElementsMatch(t, []string{*realVersion.VersionId, "null"}, listedVersionIds)
|
||||
|
||||
realVersionResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: realVersion.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer realVersionResp.Body.Close()
|
||||
realVersionBody, err := io.ReadAll(realVersionResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "pre-suspension-content", string(realVersionBody))
|
||||
}
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestSuspendedVersioningNullOverwrite tests the scenario where:
|
||||
@@ -257,41 +255,3 @@ func TestEnabledVersioningReturnsVersionId(t *testing.T) {
|
||||
t.Logf(" Version %d: VersionId=%s, Size=%d, IsLatest=%v", i, *v.VersionId, v.Size, v.IsLatest)
|
||||
}
|
||||
}
|
||||
|
||||
// The suspended PUT retires the null delete marker a preceding DELETE left, the same
|
||||
// as the copy and multipart paths. While the regular-path object owns the null slot a
|
||||
// leftover marker is shadowed, so it only shows once that null version goes away.
|
||||
func TestSuspendedPutRetiresDeleteMarker(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
objectKey := "suspended-put-after-delete.txt"
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
putObject(t, client, bucketName, objectKey, "pre-suspension-content")
|
||||
suspendVersioning(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, objectKey, "null-version-content")
|
||||
deleteKey(t, client, bucketName, objectKey)
|
||||
putObject(t, client, bucketName, objectKey, "replacement-content")
|
||||
|
||||
headObject(t, client, bucketName, objectKey)
|
||||
|
||||
// Drop the null version just written; a retired marker leaves nothing behind.
|
||||
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String("null"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, listResp.DeleteMarkers, "the suspended PUT should have retired the null delete marker")
|
||||
}
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestVersionedListMarkerIsExclusive covers start-after and marker on a versioned bucket,
|
||||
// where an object is stored in a "<key>.versions" directory and so never matches the
|
||||
// marker by entry name.
|
||||
func TestVersionedListMarkerIsExclusive(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
for _, key := range []string{"file-0", "file-1", "file-2", "logs/file-0", "logs/file-1", "logs/file-2"} {
|
||||
putObject(t, client, bucketName, key, "content")
|
||||
}
|
||||
|
||||
listV2 := func(prefix, startAfter string) []string {
|
||||
t.Helper()
|
||||
resp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(prefix),
|
||||
StartAfter: aws.String(startAfter),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return contentKeys(resp.Contents)
|
||||
}
|
||||
listV1 := func(prefix, marker string) []string {
|
||||
t.Helper()
|
||||
resp, err := client.ListObjects(context.TODO(), &s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(prefix),
|
||||
Marker: aws.String(marker),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return contentKeys(resp.Contents)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{"file-2"}, listV2("file-", "file-1"))
|
||||
assert.Equal(t, []string{"file-2"}, listV1("file-", "file-1"))
|
||||
assert.Equal(t, []string{"logs/file-2"}, listV2("logs/", "logs/file-1"))
|
||||
assert.Equal(t, []string{"logs/file-2"}, listV1("logs/", "logs/file-1"))
|
||||
}
|
||||
|
||||
func contentKeys(contents []types.Object) []string {
|
||||
keys := make([]string, 0, len(contents))
|
||||
for _, c := range contents {
|
||||
keys = append(keys, *c.Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -38,9 +38,6 @@ type oauthTestEnv struct {
|
||||
weedCancel context.CancelFunc
|
||||
accessKey string
|
||||
secretKey string
|
||||
// s3ExternalURL, when set before start, is the S3 endpoint the catalog
|
||||
// advertises to clients in LoadTable FileIO config.
|
||||
s3ExternalURL string
|
||||
}
|
||||
|
||||
func newOAuthTestEnv(t *testing.T) *oauthTestEnv {
|
||||
@@ -111,7 +108,7 @@ func (env *oauthTestEnv) start(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
args := []string{"mini",
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
|
||||
"-master.port", fmt.Sprintf("%d", env.masterPort),
|
||||
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
|
||||
"-volume.port", fmt.Sprintf("%d", env.volumePort),
|
||||
@@ -126,12 +123,7 @@ func (env *oauthTestEnv) start(t *testing.T) {
|
||||
"-ip", env.bindIP,
|
||||
"-ip.bind", "0.0.0.0",
|
||||
"-dir", env.dataDir,
|
||||
}
|
||||
if env.s3ExternalURL != "" {
|
||||
args = append(args, "-s3.externalUrl", env.s3ExternalURL)
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, args...)
|
||||
)
|
||||
cmd.Dir = env.dataDir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
@@ -224,12 +216,8 @@ func TestOAuthTokenEndpoint(t *testing.T) {
|
||||
t.Run("bearer token auth on catalog endpoint", func(t *testing.T) {
|
||||
token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
|
||||
|
||||
// Point the call at a table bucket that exists, so anything but 200 is an
|
||||
// auth or routing fault rather than the catalog reporting a missing bucket.
|
||||
bucketName := "oauth-bearer-" + randomSuffix()
|
||||
createTableBucketViaShell(t, env, bucketName)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, env.icebergURL()+"/v1/"+bucketName+"/namespaces", nil)
|
||||
// Use the token to call the catalog
|
||||
req, err := http.NewRequest(http.MethodGet, env.icebergURL()+"/v1/namespaces", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("create request: %v", err)
|
||||
}
|
||||
@@ -237,12 +225,17 @@ func TestOAuthTokenEndpoint(t *testing.T) {
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("GET /v1/%s/namespaces with Bearer: %v", bucketName, err)
|
||||
t.Fatalf("GET /v1/namespaces with Bearer: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Auth should pass. We accept 200 (success) or 500 (missing warehouse bucket
|
||||
// is an internal error, not an auth error). Reject 401/403/404/405.
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK, http.StatusInternalServerError:
|
||||
t.Logf("Bearer token auth succeeded, status=%d", resp.StatusCode)
|
||||
default:
|
||||
t.Fatalf("Bearer auth failed unexpectedly: status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
// Tests for the credential-vending access pattern.
|
||||
//
|
||||
// DuckDB attaches a catalog with "X-Iceberg-Access-Delegation: vended-credentials"
|
||||
// and rebuilds its S3 credential out of whatever LoadTable returns in `config`,
|
||||
// dropping the S3 secret it was configured with. A catalog that advertises an
|
||||
// endpoint there but vends no credentials therefore leaves DuckDB sending
|
||||
// unsigned requests, and every metadata and data file comes back 403.
|
||||
//
|
||||
// These run against a weed mini started with -s3.externalUrl, which is what
|
||||
// makes the catalog advertise an endpoint at all; with a wildcard bind and no
|
||||
// external URL the config is empty and the bug cannot appear.
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
// duckDBImage tracks whatever DuckDB users are actually running, matching the
|
||||
// other DuckDB tests here: a client that changes how it handles vended
|
||||
// credentials is exactly what this suite exists to catch.
|
||||
const duckDBImage = "duckdb/duckdb:latest"
|
||||
|
||||
// requireDuckDBIceberg skips when the image cannot install the iceberg
|
||||
// extension -- no egress, or a build without it. Probing separately keeps the
|
||||
// round trip below free to fail on any error instead of having to guess which
|
||||
// DuckDB messages mean "no extension" and which mean "the catalog is broken".
|
||||
func requireDuckDBIceberg(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
const ready = "iceberg extension ready"
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--entrypoint", "duckdb",
|
||||
duckDBImage,
|
||||
"-c", fmt.Sprintf("INSTALL iceberg; LOAD iceberg; SELECT '%s' AS marker;", ready),
|
||||
)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil || !strings.Contains(string(output), ready) {
|
||||
t.Skipf("DuckDB image cannot load the iceberg extension: %v\n%s", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIcebergVendedCredentials(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := newOAuthTestEnv(t)
|
||||
// The DuckDB container reaches S3 through host.docker.internal, so advertise
|
||||
// the endpoint under the name the client can actually resolve.
|
||||
env.s3ExternalURL = fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
defer env.cleanup(t)
|
||||
env.start(t)
|
||||
|
||||
bucketName := "vendcreds-" + randomSuffix()
|
||||
namespace := "vendcreds_ns_" + randomSuffix()
|
||||
tableName := "vendcreds_tbl_" + randomSuffix()
|
||||
|
||||
createTableBucketViaShell(t, env, bucketName)
|
||||
token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
|
||||
createNamespaceWithToken(t, env, token, bucketName, namespace)
|
||||
createTableWithToken(t, env, token, bucketName, namespace, tableName)
|
||||
|
||||
t.Run("plain load table advertises the endpoint", func(t *testing.T) {
|
||||
config := loadTableConfig(t, env, token, bucketName, namespace, tableName, "")
|
||||
if config["s3.endpoint"] != env.s3ExternalURL {
|
||||
t.Fatalf("s3.endpoint = %q, want %q (clients that bring their own credentials still need the endpoint)",
|
||||
config["s3.endpoint"], env.s3ExternalURL)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("vended credentials request gets no half-configured storage", func(t *testing.T) {
|
||||
config := loadTableConfig(t, env, token, bucketName, namespace, tableName, "vended-credentials")
|
||||
if config["s3.endpoint"] == "" {
|
||||
return
|
||||
}
|
||||
// If the catalog ever does vend credentials, the endpoint may come back
|
||||
// -- but only alongside the credentials that make it usable.
|
||||
if config["s3.access-key-id"] == "" || config["s3.secret-access-key"] == "" {
|
||||
t.Fatalf("LoadTable config = %v: a client that asked for vended credentials drops its own and signs with what it gets, so an endpoint without credentials makes every data file read 403", config)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("duckdb writes and reads back through the catalog", func(t *testing.T) {
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("Docker not available, skipping DuckDB round trip")
|
||||
}
|
||||
requireDuckDBIceberg(t)
|
||||
duckDBRoundTrip(t, env, bucketName, namespace)
|
||||
})
|
||||
}
|
||||
|
||||
// duckDBRoundTrip runs the reporter's flow: attach the catalog, create a table
|
||||
// from a query, then read it back. Both halves need signed S3 access to the
|
||||
// table's metadata and data files.
|
||||
func duckDBRoundTrip(t *testing.T, env *oauthTestEnv, bucketName, namespace string) {
|
||||
t.Helper()
|
||||
|
||||
const marker = "vended-credentials round trip ok"
|
||||
sql := fmt.Sprintf(`
|
||||
INSTALL iceberg;
|
||||
LOAD iceberg;
|
||||
|
||||
CREATE SECRET iceberg_secret (
|
||||
TYPE ICEBERG,
|
||||
ENDPOINT 'http://host.docker.internal:%d',
|
||||
CLIENT_ID '%s',
|
||||
CLIENT_SECRET '%s'
|
||||
);
|
||||
|
||||
CREATE SECRET s3_secret (
|
||||
TYPE S3,
|
||||
KEY_ID '%s',
|
||||
SECRET '%s',
|
||||
ENDPOINT 'host.docker.internal:%d',
|
||||
URL_STYLE 'path',
|
||||
USE_SSL false
|
||||
);
|
||||
|
||||
ATTACH 's3://%s' AS vend_cat (
|
||||
TYPE ICEBERG,
|
||||
SECRET iceberg_secret,
|
||||
ENDPOINT 'http://host.docker.internal:%d',
|
||||
READ_ONLY false
|
||||
);
|
||||
|
||||
CREATE TABLE vend_cat.%s.round_trip AS SELECT 42 AS answer;
|
||||
|
||||
SELECT CASE WHEN sum(answer) = 42 THEN '%s' ELSE 'wrong answer' END AS marker
|
||||
FROM vend_cat.%s.round_trip;
|
||||
`,
|
||||
env.icebergPort, env.accessKey, env.secretKey,
|
||||
env.accessKey, env.secretKey, env.s3Port,
|
||||
bucketName, env.icebergPort,
|
||||
namespace, marker, namespace,
|
||||
)
|
||||
|
||||
sqlFile := filepath.Join(env.dataDir, "duckdb_vended_credentials.sql")
|
||||
if err := os.WriteFile(sqlFile, []byte(sql), 0644); err != nil {
|
||||
t.Fatalf("write SQL file: %v", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"-v", fmt.Sprintf("%s:/test", env.dataDir),
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-e", "AWS_REGION=us-east-1",
|
||||
"--entrypoint", "duckdb",
|
||||
duckDBImage,
|
||||
"-init", "/test/duckdb_vended_credentials.sql",
|
||||
"-c", "SELECT 1",
|
||||
)
|
||||
output, err := cmd.CombinedOutput()
|
||||
outputStr := string(output)
|
||||
t.Logf("DuckDB output:\n%s", outputStr)
|
||||
|
||||
// Nothing below skips: requireDuckDBIceberg already established that the
|
||||
// extension loads, so any failure from here is the catalog's.
|
||||
if strings.Contains(outputStr, "trying to refresh secret") {
|
||||
t.Fatalf("DuckDB fell back to refreshing the credential the catalog vended, which no stage-created table can satisfy:\n%s", outputStr)
|
||||
}
|
||||
// Match the phrasings rather than a bare "403", which a random port could
|
||||
// carry.
|
||||
for _, denial := range []string{"AccessDenied", "403 Forbidden", "code 403"} {
|
||||
if strings.Contains(outputStr, denial) {
|
||||
t.Fatalf("DuckDB signed its S3 requests with the catalog's credential-less config:\n%s", outputStr)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(outputStr, marker) {
|
||||
t.Fatalf("DuckDB did not write and read the table back (err=%v):\n%s", err, outputStr)
|
||||
}
|
||||
}
|
||||
|
||||
// loadTableConfig loads a table and returns the FileIO config the catalog
|
||||
// advertises, optionally asking for an access delegation mechanism.
|
||||
func loadTableConfig(t *testing.T, env *oauthTestEnv, token, bucketName, namespace, tableName, delegation string) map[string]string {
|
||||
t.Helper()
|
||||
|
||||
url := fmt.Sprintf("%s/v1/%s/namespaces/%s/tables/%s", env.icebergURL(), bucketName, namespace, tableName)
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("create request: %v", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
if delegation != "" {
|
||||
req.Header.Set("X-Iceberg-Access-Delegation", delegation)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("load table: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("load table failed: status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Config map[string]string `json:"config"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
t.Fatalf("decode LoadTableResult: %v", err)
|
||||
}
|
||||
return result.Config
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
# Known winfsp-tests failures for the SeaweedFS Windows mount.
|
||||
#
|
||||
# One test name per line, as winfsp-tests reports it. A trailing * matches a
|
||||
# prefix, so "stream_*" covers a whole group. Listed tests are excluded from
|
||||
# the run; a failure in anything NOT listed fails CI, which is what catches a
|
||||
# regression.
|
||||
#
|
||||
# Every test runs on its own in a clean directory, so an entry here is a real
|
||||
# defect rather than the wreckage of the test before it. The rdwr and flush
|
||||
# group is gone from this list for exactly that reason: it passes in isolation
|
||||
# and only failed as collateral. Every entry below is a
|
||||
# gap in the mount rather than a quirk of the suite, and the list is meant to
|
||||
# shrink. Keep a reason on each group — an entry with no reason cannot be told
|
||||
# apart from one nobody has looked at.
|
||||
|
||||
# Cannot pass by construction
|
||||
# ---------------------------
|
||||
# Alternate data streams: the filer stores one byte range per entry, and the
|
||||
# mount exposes no second stream.
|
||||
stream_*
|
||||
# Reparse points: Symlink is refused, so there is nothing to resolve.
|
||||
reparse_*
|
||||
# Byte-range locks live in the WinFsp driver, so they never reach the mount.
|
||||
lock_*
|
||||
|
||||
# Security descriptors
|
||||
# --------------------
|
||||
# The mount carries unix mode bits and no ACL, and Chown accepts and discards,
|
||||
# so a descriptor written here does not read back.
|
||||
create_sd_test
|
||||
getsecurity_test
|
||||
|
||||
# Memory-mapped IO
|
||||
# ----------------
|
||||
# rdwr-test.c:645 compares the mapped bytes against the pattern that was
|
||||
# written and finds them different. A real data mismatch, not a cascade: it
|
||||
# only surfaced once each test ran on its own and this one got far enough to
|
||||
# check. Worth its own investigation.
|
||||
rdwr_mmap_test
|
||||
|
||||
# Delete semantics
|
||||
# ----------------
|
||||
# WinFsp advertises POSIX unlink, so a file can be deleted while handles are
|
||||
# still open; the mount does not implement the delete-pending state.
|
||||
delete_access_test
|
||||
delete_ex_test
|
||||
delete_mmap_test
|
||||
delete_pending_test
|
||||
|
||||
# Rename semantics
|
||||
# ----------------
|
||||
# Rename over an open file, and rename of a directory holding open handles,
|
||||
# which WinFsp is stricter about than POSIX.
|
||||
rename_ex_test
|
||||
rename_mmap_test
|
||||
rename_open_test
|
||||
exec_rename_dir_test
|
||||
|
||||
# File information
|
||||
# ----------------
|
||||
# A regular file reports as archived now, but hidden, system and readonly are
|
||||
# still not round-tripped: create_fileattr gets past the archive check and then
|
||||
# asks for READONLY as well, which needs Chflags. Creation time is reported as
|
||||
# ctime because the raw protocol's Attr carries no field for it outside darwin.
|
||||
create_fileattr_test
|
||||
# getfileinfo asserts a file created a moment ago has an access time within ten
|
||||
# seconds of now, and it does not. Two causes have been fixed and neither was
|
||||
# it: Utimens no longer stores a zero timestamp, and the in-memory overlays are
|
||||
# dropped when an inode is released so a recycled number cannot inherit an old
|
||||
# time. Still unexplained, so it stays listed rather than pretended about.
|
||||
getfileinfo_test
|
||||
create_readonlydir_test
|
||||
setfileinfo_test
|
||||
|
||||
# Directory enumeration
|
||||
# ---------------------
|
||||
# querydir_namelen only: checkName caps at 255 bytes while WinFsp counts 255
|
||||
# characters. The other two are expected to pass now that "." and ".." are
|
||||
# reported for non-root directories, which is what WinFsp enumerates.
|
||||
querydir_namelen_test
|
||||
|
||||
# Name length
|
||||
# -----------
|
||||
# checkName caps at 255 bytes while WinFsp allows 255 characters, so a CJK
|
||||
# name that is legal to Windows is rejected here.
|
||||
create_namelen_test
|
||||
|
||||
# Sharing modes
|
||||
# -------------
|
||||
# Windows share-access checking on create, which the mount does not enforce.
|
||||
create_share_test
|
||||
create_test
|
||||
@@ -1,112 +0,0 @@
|
||||
# Runs WinFsp's own conformance suite against a mounted SeaweedFS drive.
|
||||
#
|
||||
# winfsp-tests is what WinFsp uses to check a filesystem behaves like NTFS,
|
||||
# and --fuse-external points it at somebody else's filesystem instead of the
|
||||
# bundled memfs. It is the Windows counterpart of the pjdfstest run the FUSE
|
||||
# mount already goes through, and it is scored the same way: anything failing
|
||||
# that is not in known_failures.txt is a regression.
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$MountPoint,
|
||||
[string]$KnownFailures = "$PSScriptRoot\known_failures.txt",
|
||||
[string]$WinFspTestsVersion = '2.1.25156'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# "S:" and "S:\" mean different things to Join-Path: without the separator the
|
||||
# result is relative to the drive's current directory, not its root.
|
||||
if ($MountPoint -notmatch '[\\/]$') { $MountPoint = $MountPoint + '\' }
|
||||
|
||||
# winfsp-tests links against winfsp-x64.dll, which the MSI puts somewhere the
|
||||
# loader does not look by default.
|
||||
foreach ($candidate in @("${env:ProgramFiles(x86)}\WinFsp\bin", "$env:ProgramFiles\WinFsp\bin")) {
|
||||
if (Test-Path $candidate) {
|
||||
$env:PATH = "$candidate;$env:PATH"
|
||||
Write-Host "using WinFsp binaries from $candidate"
|
||||
}
|
||||
}
|
||||
|
||||
$toolDir = Join-Path $env:TEMP 'winfsp-tests'
|
||||
$exe = Join-Path $toolDir 'winfsp-tests-x64.exe'
|
||||
if (-not (Test-Path $exe)) {
|
||||
# Shipped as its own archive rather than in the MSI.
|
||||
$url = "https://github.com/winfsp/winfsp/releases/download/v2.1/winfsp-tests-$WinFspTestsVersion.zip"
|
||||
$zip = Join-Path $env:TEMP 'winfsp-tests.zip'
|
||||
Write-Host "downloading $url"
|
||||
Invoke-WebRequest -Uri $url -OutFile $zip -UseBasicParsing
|
||||
Expand-Archive -LiteralPath $zip -DestinationPath $toolDir -Force
|
||||
}
|
||||
if (-not (Test-Path $exe)) {
|
||||
throw "winfsp-tests-x64.exe not found under $toolDir"
|
||||
}
|
||||
|
||||
# A missing or unreadable list would otherwise run with nothing excluded and
|
||||
# report that as normal, which reads like a pass with no known failures.
|
||||
if (-not (Test-Path $KnownFailures)) {
|
||||
throw "known failures list not found at $KnownFailures"
|
||||
}
|
||||
$excluded = @(Get-Content $KnownFailures |
|
||||
ForEach-Object { $_.Trim() } |
|
||||
Where-Object { $_ -and -not $_.StartsWith('#') })
|
||||
Write-Host "read $($excluded.Count) exclusions from $KnownFailures"
|
||||
if ($excluded.Count -eq 0) {
|
||||
throw "known failures list at $KnownFailures parsed to nothing"
|
||||
}
|
||||
|
||||
# The suite refuses to run anywhere but a drive, and works in the current
|
||||
# directory, so it has to be driven from inside the mount.
|
||||
$workDir = Join-Path $MountPoint 'winfsp-conformance'
|
||||
New-Item -ItemType Directory -Force -Path $workDir | Out-Null
|
||||
Push-Location $workDir
|
||||
try {
|
||||
# --fuse-external: a third-party FUSE filesystem, not the bundled memfs.
|
||||
# --resilient: tolerate operations this filesystem does not implement.
|
||||
#
|
||||
# One test per invocation, each in its own directory. Batched with
|
||||
# --no-abort, a test that failed part way left its files behind and the
|
||||
# next one failed creating them, reporting a cascade of failures that were
|
||||
# really one. This costs a process start per test and makes the list mean
|
||||
# what it says.
|
||||
$base = @('--fuse-external', '--resilient')
|
||||
$names = @(& $exe @base '--list' 2>&1 |
|
||||
ForEach-Object { if ($_ -match '^([a-z_0-9]+)\s*$') { $Matches[1] } })
|
||||
if ($names.Count -eq 0) { throw "could not list tests" }
|
||||
Write-Host "listed $($names.Count) tests"
|
||||
|
||||
$failed = @()
|
||||
$ran = 0
|
||||
foreach ($name in $names) {
|
||||
if ($excluded | Where-Object { $name -like $_ }) { continue }
|
||||
$ran++
|
||||
$caseDir = Join-Path $workDir $name
|
||||
# -Force creates the directory but leaves anything already in it, and
|
||||
# a leftover file is the very thing this isolation exists to avoid.
|
||||
Remove-Item $caseDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Force -Path $caseDir | Out-Null
|
||||
Push-Location $caseDir
|
||||
try {
|
||||
$out = & $exe @base $name 2>&1
|
||||
$out | ForEach-Object { Write-Host $_ }
|
||||
if ($out -match '\s+KO\s*$' -or $LASTEXITCODE -ne 0) { $failed += $name }
|
||||
} finally {
|
||||
Pop-Location
|
||||
Remove-Item $caseDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
Write-Host "ran $ran tests, $($failed.Count) failed"
|
||||
$code = if ($failed.Count -gt 0) { 1 } else { 0 }
|
||||
} finally {
|
||||
Pop-Location
|
||||
Remove-Item $workDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
if ($failed.Count -gt 0) {
|
||||
Write-Host "::error::winfsp-tests failures outside known_failures.txt: $($failed -join ', ')"
|
||||
exit 1
|
||||
}
|
||||
if ($code -ne 0) {
|
||||
Write-Host "::error::winfsp-tests exited $code with no failure reported"
|
||||
exit $code
|
||||
}
|
||||
Write-Host 'winfsp-tests passed'
|
||||
@@ -1,7 +0,0 @@
|
||||
package winfsp
|
||||
|
||||
import "errors"
|
||||
|
||||
// errWindowsOnly marks the platforms with no way to ask for delete-on-close,
|
||||
// so the test can tell that apart from the flag being refused.
|
||||
var errWindowsOnly = errors.New("delete-on-close is a windows flag")
|
||||
@@ -1,9 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
package winfsp
|
||||
|
||||
type handle uintptr
|
||||
|
||||
func createDeleteOnClose(path string) (handle, error) { return 0, errWindowsOnly }
|
||||
|
||||
func closeHandle(h handle) error { return errWindowsOnly }
|
||||
@@ -1,24 +0,0 @@
|
||||
package winfsp
|
||||
|
||||
import "golang.org/x/sys/windows"
|
||||
|
||||
// createDeleteOnClose opens a new file that Windows removes when the last
|
||||
// handle to it closes. Installers and editors use this for temporaries, and
|
||||
// os has no way to ask for it.
|
||||
func createDeleteOnClose(path string) (windows.Handle, error) {
|
||||
p, err := windows.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return windows.InvalidHandle, err
|
||||
}
|
||||
return windows.CreateFile(
|
||||
p,
|
||||
windows.GENERIC_READ|windows.GENERIC_WRITE,
|
||||
windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE,
|
||||
nil,
|
||||
windows.CREATE_NEW,
|
||||
windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_DELETE_ON_CLOSE,
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
func closeHandle(h windows.Handle) error { return windows.CloseHandle(h) }
|
||||
@@ -1,9 +0,0 @@
|
||||
//go:build !windows
|
||||
|
||||
package winfsp
|
||||
|
||||
import "errors"
|
||||
|
||||
func getDiskFreeSpace(path string, free, total, totalFree *uint64) error {
|
||||
return errors.New("windows only")
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package winfsp
|
||||
|
||||
import (
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func getDiskFreeSpace(path string, free, total, totalFree *uint64) error {
|
||||
p, err := windows.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return windows.GetDiskFreeSpaceEx(p, free, total, totalFree)
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
// Package winfsp exercises a live SeaweedFS mount on Windows.
|
||||
//
|
||||
// It runs against whatever is mounted at -mountpoint, so it needs a real
|
||||
// WinFsp mount and is skipped otherwise. The CI job drives it; run it by hand
|
||||
// with: go test ./test/winfsp -mountpoint=S:\
|
||||
package winfsp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var mountPoint = flag.String("mountpoint", "", "a mounted SeaweedFS drive, e.g. S:\\")
|
||||
|
||||
// entryCount is the directory width the test builds. The reported case that
|
||||
// motivated Windows support is 200k files in one folder; CI uses less so the
|
||||
// job stays inside its budget, but the read path is the same.
|
||||
var entryCount = flag.Int("entries", 5000, "how many files to put in one directory")
|
||||
|
||||
func testRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
if *mountPoint == "" {
|
||||
t.Skip("no -mountpoint given; this test needs a live WinFsp mount")
|
||||
}
|
||||
dir := filepath.Join(*mountPoint, "winfsp-test-"+t.Name())
|
||||
if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) {
|
||||
t.Fatalf("clean %s: %v", dir, err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", dir, err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestWriteReadRoundTrip(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
for _, size := range []int{0, 1, 4095, 4096, 1 << 20, 5 << 20} {
|
||||
t.Run(fmt.Sprintf("%dbytes", size), func(t *testing.T) {
|
||||
want := make([]byte, size)
|
||||
if _, err := rand.Read(want); err != nil {
|
||||
t.Fatalf("rand: %v", err)
|
||||
}
|
||||
path := filepath.Join(dir, fmt.Sprintf("file-%d", size))
|
||||
if err := os.WriteFile(path, want, 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("content differs: got %d bytes, want %d", len(got), len(want))
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
if info.Size() != int64(size) {
|
||||
t.Fatalf("stat size = %d, want %d", info.Size(), size)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeekWriteAndOverwrite(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "sparse")
|
||||
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if _, err := f.WriteAt([]byte("tail"), 4096); err != nil {
|
||||
t.Fatalf("write at offset: %v", err)
|
||||
}
|
||||
if _, err := f.WriteAt([]byte("head"), 0); err != nil {
|
||||
t.Fatalf("write at start: %v", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if len(got) != 4100 {
|
||||
t.Fatalf("size = %d, want 4100", len(got))
|
||||
}
|
||||
if string(got[:4]) != "head" || string(got[4096:]) != "tail" {
|
||||
t.Fatalf("content at the edges is wrong: %q ... %q", got[:4], got[4096:])
|
||||
}
|
||||
if !bytes.Equal(got[4:4096], make([]byte, 4092)) {
|
||||
t.Fatal("the hole between the two writes is not zero-filled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameAndDelete(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
src := filepath.Join(dir, "before")
|
||||
dst := filepath.Join(dir, "after")
|
||||
|
||||
if err := os.WriteFile(src, []byte("payload"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(src); !os.IsNotExist(err) {
|
||||
t.Fatalf("old name still resolves: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("read renamed: %v", err)
|
||||
}
|
||||
if string(got) != "payload" {
|
||||
t.Fatalf("content after rename = %q", got)
|
||||
}
|
||||
if err := os.Remove(dst); err != nil {
|
||||
t.Fatalf("remove: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(dst); !os.IsNotExist(err) {
|
||||
t.Fatalf("deleted file still resolves: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectoryTree(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
nested := filepath.Join(dir, "a", "b", "c")
|
||||
if err := os.MkdirAll(nested, 0755); err != nil {
|
||||
t.Fatalf("mkdirall: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(nested, "leaf"), []byte("x"), 0644); err != nil {
|
||||
t.Fatalf("write leaf: %v", err)
|
||||
}
|
||||
info, err := os.Stat(nested)
|
||||
if err != nil {
|
||||
t.Fatalf("stat dir: %v", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
t.Fatal("nested path is not reported as a directory")
|
||||
}
|
||||
// A non-empty directory must not be removable.
|
||||
if err := os.Remove(filepath.Join(dir, "a", "b")); err == nil {
|
||||
t.Fatal("removed a non-empty directory")
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(dir, "a")); err != nil {
|
||||
t.Fatalf("removeall: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWideDirectory is the case from the report: one folder holding far more
|
||||
// entries than Explorer or the Windows WebDAV client cope with.
|
||||
func TestWideDirectory(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
want := make([]string, 0, *entryCount)
|
||||
for i := 0; i < *entryCount; i++ {
|
||||
name := fmt.Sprintf("img-%06d.dat", i)
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte{byte(i)}, 0644); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
want = append(want, name)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("readdir: %v", err)
|
||||
}
|
||||
got := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
got = append(got, e.Name())
|
||||
}
|
||||
sort.Strings(got)
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("readdir returned %d entries, want %d", len(got), len(want))
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("entry %d = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
|
||||
// Reading the directory twice must be stable; a paging bug shows up here.
|
||||
again, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("second readdir: %v", err)
|
||||
}
|
||||
if len(again) != len(entries) {
|
||||
t.Fatalf("second readdir returned %d entries, first returned %d", len(again), len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentWriters(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
const writers = 8
|
||||
const perWriter = 32
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, writers)
|
||||
for w := 0; w < writers; w++ {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for i := 0; i < perWriter; i++ {
|
||||
name := filepath.Join(dir, fmt.Sprintf("w%d-%d", w, i))
|
||||
body := []byte(strings.Repeat(fmt.Sprintf("%d", w), 128))
|
||||
if err := os.WriteFile(name, body, 0644); err != nil {
|
||||
errs <- fmt.Errorf("writer %d: %w", w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("readdir: %v", err)
|
||||
}
|
||||
if len(entries) != writers*perWriter {
|
||||
t.Fatalf("got %d files, want %d", len(entries), writers*perWriter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatfsReportsCapacity(t *testing.T) {
|
||||
if *mountPoint == "" {
|
||||
t.Skip("no -mountpoint given; this test needs a live WinFsp mount")
|
||||
}
|
||||
// The drive has to report a size, or Explorer refuses to show it.
|
||||
var free, total, totalFree uint64
|
||||
if err := getDiskFreeSpace(*mountPoint, &free, &total, &totalFree); err != nil {
|
||||
t.Fatalf("GetDiskFreeSpaceEx: %v", err)
|
||||
}
|
||||
if total == 0 {
|
||||
t.Fatal("drive reports zero total bytes")
|
||||
}
|
||||
}
|
||||
@@ -1,186 +0,0 @@
|
||||
package winfsp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// phase splits the test across a remount: the write phase is run, the
|
||||
// mount is torn down and brought back, then the verify phase runs. Reading
|
||||
// back through the same live mount proves nothing about durability, since
|
||||
// the answer can come from the mount's own caches.
|
||||
phase = flag.String("phase", "", "write or verify; empty skips the persistence test")
|
||||
|
||||
// filerAddr enables a second check, that the bytes reached the filer and
|
||||
// are servable without the mount in the path at all.
|
||||
filerAddr = flag.String("filer", "", "filer host:port to cross-check through, e.g. localhost:8888")
|
||||
|
||||
// persistSubdir is where the fixtures live, under both the mount and the
|
||||
// filer's mount root.
|
||||
persistSubdir = flag.String("persistdir", "winfsp-persist", "directory under the mount to persist into")
|
||||
)
|
||||
|
||||
type fixture struct {
|
||||
relPath string
|
||||
size int
|
||||
}
|
||||
|
||||
// Sizes straddle the boundaries where the write path changes behavior: inline,
|
||||
// a single chunk, and several chunks.
|
||||
var fixtures = []fixture{
|
||||
{"small.txt", 11},
|
||||
{"medium.bin", 300 << 10},
|
||||
{"multichunk.bin", 9 << 20},
|
||||
{"nested/deep/leaf.bin", 65536},
|
||||
{"unicode-café-日本.txt", 64},
|
||||
}
|
||||
|
||||
// contentFor derives bytes from the name, so the write and verify phases agree
|
||||
// without carrying a manifest between them.
|
||||
func contentFor(relPath string, size int) []byte {
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(relPath))
|
||||
state := h.Sum64() | 1
|
||||
out := make([]byte, size)
|
||||
for i := range out {
|
||||
state ^= state << 13
|
||||
state ^= state >> 7
|
||||
state ^= state << 17
|
||||
out[i] = byte(state)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestPersistence(t *testing.T) {
|
||||
if *mountPoint == "" {
|
||||
t.Skip("no -mountpoint given; this test needs a live WinFsp mount")
|
||||
}
|
||||
switch *phase {
|
||||
case "write":
|
||||
persistenceWrite(t)
|
||||
case "verify":
|
||||
persistenceVerify(t)
|
||||
default:
|
||||
t.Skip("no -phase given; run with -phase=write before a remount and -phase=verify after")
|
||||
}
|
||||
}
|
||||
|
||||
func persistenceWrite(t *testing.T) {
|
||||
root := filepath.Join(*mountPoint, *persistSubdir)
|
||||
if err := os.RemoveAll(root); err != nil && !os.IsNotExist(err) {
|
||||
t.Fatalf("clean %s: %v", root, err)
|
||||
}
|
||||
if err := os.MkdirAll(root, 0755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", root, err)
|
||||
}
|
||||
|
||||
for _, f := range fixtures {
|
||||
target := filepath.Join(root, filepath.FromSlash(f.relPath))
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
t.Fatalf("mkdir for %s: %v", f.relPath, err)
|
||||
}
|
||||
// Sync before closing: the mount is killed rather than unmounted, so
|
||||
// anything still queued for flush is legitimately lost. Testing
|
||||
// durability means testing what survives after an explicit sync.
|
||||
if err := writeAndSync(target, contentFor(f.relPath, f.size)); err != nil {
|
||||
t.Fatalf("write %s: %v", f.relPath, err)
|
||||
}
|
||||
t.Logf("wrote %s (%d bytes)", f.relPath, f.size)
|
||||
}
|
||||
|
||||
// Closing the files should have pushed them to the filer. Check that
|
||||
// directly, so a failure here separates "never left the mount" from
|
||||
// "did not survive the remount".
|
||||
if *filerAddr == "" {
|
||||
return
|
||||
}
|
||||
for _, f := range fixtures {
|
||||
want := contentFor(f.relPath, f.size)
|
||||
got, err := fetchFromFiler(*filerAddr, path.Join(*persistSubdir, f.relPath))
|
||||
if err != nil {
|
||||
t.Errorf("fetch %s from filer: %v", f.relPath, err)
|
||||
continue
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("filer served %d bytes for %s, want %d", len(got), f.relPath, len(want))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func persistenceVerify(t *testing.T) {
|
||||
root := filepath.Join(*mountPoint, *persistSubdir)
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Fatalf("stat %s after remount: %v", root, err)
|
||||
}
|
||||
|
||||
for _, f := range fixtures {
|
||||
target := filepath.Join(root, filepath.FromSlash(f.relPath))
|
||||
got, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Errorf("read %s after remount: %v", f.relPath, err)
|
||||
continue
|
||||
}
|
||||
want := contentFor(f.relPath, f.size)
|
||||
if len(got) != len(want) {
|
||||
t.Errorf("%s is %d bytes after remount, want %d", f.relPath, len(got), len(want))
|
||||
continue
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Errorf("%s survived the remount with different content", f.relPath)
|
||||
}
|
||||
}
|
||||
|
||||
// The directory structure has to come back too, not just the files.
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatalf("readdir %s after remount: %v", root, err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatal("mount root directory is empty after remount")
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
t.Errorf("cleanup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAndSync(path string, content []byte) error {
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(content); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
func fetchFromFiler(addr, filerPath string) ([]byte, error) {
|
||||
endpoint := &url.URL{Scheme: "http", Host: addr, Path: "/" + filerPath}
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
resp, err := client.Get(endpoint.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("GET %s: %s", endpoint, resp.Status)
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
@@ -1,502 +0,0 @@
|
||||
package winfsp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTruncate(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "resize")
|
||||
original := bytes.Repeat([]byte("abcd"), 4096) // 16 KiB
|
||||
|
||||
if err := os.WriteFile(path, original, 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
// Shrink.
|
||||
if err := os.Truncate(path, 100); err != nil {
|
||||
t.Fatalf("truncate down: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read after shrink: %v", err)
|
||||
}
|
||||
if len(got) != 100 {
|
||||
t.Fatalf("size after shrink = %d, want 100", len(got))
|
||||
}
|
||||
if !bytes.Equal(got, original[:100]) {
|
||||
t.Fatal("the surviving prefix does not match the original")
|
||||
}
|
||||
|
||||
// Grow: the new space has to read back as zeros, not stale bytes.
|
||||
if err := os.Truncate(path, 8192); err != nil {
|
||||
t.Fatalf("truncate up: %v", err)
|
||||
}
|
||||
got, err = os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read after grow: %v", err)
|
||||
}
|
||||
if len(got) != 8192 {
|
||||
t.Fatalf("size after grow = %d, want 8192", len(got))
|
||||
}
|
||||
if !bytes.Equal(got[100:], make([]byte, 8192-100)) {
|
||||
t.Fatal("extension is not zero-filled")
|
||||
}
|
||||
|
||||
// Truncate to empty.
|
||||
if err := os.Truncate(path, 0); err != nil {
|
||||
t.Fatalf("truncate to zero: %v", err)
|
||||
}
|
||||
if info, err := os.Stat(path); err != nil || info.Size() != 0 {
|
||||
t.Fatalf("stat after zero truncate: size=%v err=%v", info, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppend(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "log")
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
t.Fatalf("open for append: %v", err)
|
||||
}
|
||||
if _, err := fmt.Fprintf(f, "line %d\n", i); err != nil {
|
||||
t.Fatalf("append: %v", err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
want := "line 0\nline 1\nline 2\nline 3\nline 4\n"
|
||||
if string(got) != want {
|
||||
t.Fatalf("appended content = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadPastEndOfFile(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "short")
|
||||
if err := os.WriteFile(path, []byte("12345"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
buf := make([]byte, 16)
|
||||
n, err := f.ReadAt(buf, 100)
|
||||
if n != 0 {
|
||||
t.Fatalf("read %d bytes past EOF, want 0", n)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("reading past EOF returned no error, want io.EOF")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorPaths(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
|
||||
t.Run("open missing file", func(t *testing.T) {
|
||||
_, err := os.Open(filepath.Join(dir, "does-not-exist"))
|
||||
if !os.IsNotExist(err) {
|
||||
t.Fatalf("got %v, want a not-exist error", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("exclusive create over existing", func(t *testing.T) {
|
||||
path := filepath.Join(dir, "taken")
|
||||
if err := os.WriteFile(path, []byte("x"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
_, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
|
||||
if !os.IsExist(err) {
|
||||
t.Fatalf("got %v, want an already-exists error", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("remove missing file", func(t *testing.T) {
|
||||
if err := os.Remove(filepath.Join(dir, "never-there")); !os.IsNotExist(err) {
|
||||
t.Fatalf("got %v, want a not-exist error", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mkdir under a file", func(t *testing.T) {
|
||||
path := filepath.Join(dir, "regular")
|
||||
if err := os.WriteFile(path, []byte("x"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(path, "child"), 0755); err == nil {
|
||||
t.Fatal("created a directory underneath a regular file")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("readdir of a missing directory", func(t *testing.T) {
|
||||
if _, err := os.ReadDir(filepath.Join(dir, "no-such-dir")); err == nil {
|
||||
t.Fatal("listed a directory that does not exist")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRenameOverExisting(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
src := filepath.Join(dir, "src")
|
||||
dst := filepath.Join(dir, "dst")
|
||||
|
||||
if err := os.WriteFile(src, []byte("new"), 0644); err != nil {
|
||||
t.Fatalf("write src: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(dst, []byte("old"), 0644); err != nil {
|
||||
t.Fatalf("write dst: %v", err)
|
||||
}
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
t.Fatalf("rename over existing: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if string(got) != "new" {
|
||||
t.Fatalf("target holds %q, want the source content", got)
|
||||
}
|
||||
if _, err := os.Stat(src); !os.IsNotExist(err) {
|
||||
t.Fatal("source survived the rename")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameAcrossDirectories(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
from := filepath.Join(dir, "from")
|
||||
to := filepath.Join(dir, "to")
|
||||
for _, d := range []string{from, to} {
|
||||
if err := os.Mkdir(d, 0755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
src := filepath.Join(from, "file")
|
||||
dst := filepath.Join(to, "file")
|
||||
if err := os.WriteFile(src, []byte("moved"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
t.Fatalf("rename across directories: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if string(got) != "moved" {
|
||||
t.Fatalf("content after move = %q", got)
|
||||
}
|
||||
|
||||
// A directory should move too, children and all.
|
||||
nested := filepath.Join(from, "sub")
|
||||
if err := os.Mkdir(nested, 0755); err != nil {
|
||||
t.Fatalf("mkdir sub: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(nested, "child"), []byte("c"), 0644); err != nil {
|
||||
t.Fatalf("write child: %v", err)
|
||||
}
|
||||
if err := os.Rename(nested, filepath.Join(to, "sub")); err != nil {
|
||||
t.Fatalf("rename directory: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(to, "sub", "child")); err != nil {
|
||||
t.Fatalf("child did not come along: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAwkwardNames covers what has to survive the UTF-16 boundary and the
|
||||
// shapes Windows software tends to produce.
|
||||
func TestAwkwardNames(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
names := []string{
|
||||
"café.txt",
|
||||
"日本語のファイル.dat",
|
||||
"emoji-🐟.bin",
|
||||
"with space.txt",
|
||||
"with.many.dots.txt",
|
||||
"UPPER and lower.TXT",
|
||||
"dash-and_underscore.txt",
|
||||
"'quoted'.txt",
|
||||
"(parens).txt",
|
||||
"#hash&.txt",
|
||||
strings.Repeat("x", 200) + ".txt",
|
||||
}
|
||||
for _, name := range names {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
path := filepath.Join(dir, name)
|
||||
want := []byte(name)
|
||||
if err := os.WriteFile(path, want, 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("content = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("readdir: %v", err)
|
||||
}
|
||||
found := make(map[string]bool, len(entries))
|
||||
for _, e := range entries {
|
||||
found[e.Name()] = true
|
||||
}
|
||||
for _, name := range names {
|
||||
if !found[name] {
|
||||
t.Errorf("%q did not come back from readdir", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeepDirectoryNesting(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
deep := dir
|
||||
for i := 0; i < 24; i++ {
|
||||
deep = filepath.Join(deep, fmt.Sprintf("level%02d", i))
|
||||
}
|
||||
if err := os.MkdirAll(deep, 0755); err != nil {
|
||||
t.Fatalf("mkdirall depth 24: %v", err)
|
||||
}
|
||||
leaf := filepath.Join(deep, "leaf.txt")
|
||||
if err := os.WriteFile(leaf, []byte("bottom"), 0644); err != nil {
|
||||
t.Fatalf("write leaf: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(leaf)
|
||||
if err != nil {
|
||||
t.Fatalf("read leaf: %v", err)
|
||||
}
|
||||
if string(got) != "bottom" {
|
||||
t.Fatalf("leaf content = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrentSameFile is the interleaving that matters: several handles on
|
||||
// one file at once, rather than one writer each on separate files.
|
||||
func TestConcurrentSameFile(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "shared")
|
||||
|
||||
const writers = 4
|
||||
const blockSize = 4096
|
||||
if err := os.WriteFile(path, make([]byte, writers*blockSize), 0644); err != nil {
|
||||
t.Fatalf("preallocate: %v", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, writers)
|
||||
for w := 0; w < writers; w++ {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
f, err := os.OpenFile(path, os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("writer %d open: %w", w, err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
block := bytes.Repeat([]byte{byte('A' + w)}, blockSize)
|
||||
if _, err := f.WriteAt(block, int64(w*blockSize)); err != nil {
|
||||
errs <- fmt.Errorf("writer %d write: %w", w, err)
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
if len(got) != writers*blockSize {
|
||||
t.Fatalf("size = %d, want %d", len(got), writers*blockSize)
|
||||
}
|
||||
for w := 0; w < writers; w++ {
|
||||
want := bytes.Repeat([]byte{byte('A' + w)}, blockSize)
|
||||
if !bytes.Equal(got[w*blockSize:(w+1)*blockSize], want) {
|
||||
t.Fatalf("block %d was not written whole", w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentReaders(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "read-me")
|
||||
want := contentFor("concurrent-readers", 1<<20)
|
||||
if err := os.WriteFile(path, want, 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 8)
|
||||
for r := 0; r < 8; r++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
errs <- fmt.Errorf("reader saw %d bytes, want %d", len(got), len(want))
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Hard links are not something WinFsp offers, so the mount has to refuse them
|
||||
// rather than appear to succeed.
|
||||
func TestHardLinkUnsupported(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
target := filepath.Join(dir, "original")
|
||||
if err := os.WriteFile(target, []byte("x"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if err := os.Link(target, filepath.Join(dir, "hardlink")); err == nil {
|
||||
t.Fatal("hard link creation reported success")
|
||||
}
|
||||
}
|
||||
|
||||
// Symlinks are refused rather than half-supported: WinFsp needs a reparse
|
||||
// point to follow one, and an entry it cannot follow reads back empty.
|
||||
func TestSymlinkUnsupported(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
target := filepath.Join(dir, "target.txt")
|
||||
if err := os.WriteFile(target, []byte("pointed at"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if err := os.Symlink(target, filepath.Join(dir, "link.txt")); err == nil {
|
||||
t.Fatal("symlink creation reported success")
|
||||
}
|
||||
}
|
||||
|
||||
func TestModTimeAdvances(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "touched")
|
||||
if err := os.WriteFile(path, []byte("one"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
first, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
if err := os.WriteFile(path, []byte("two"), 0644); err != nil {
|
||||
t.Fatalf("rewrite: %v", err)
|
||||
}
|
||||
second, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat again: %v", err)
|
||||
}
|
||||
if !second.ModTime().After(first.ModTime()) {
|
||||
t.Fatalf("mtime did not advance: %v then %v", first.ModTime(), second.ModTime())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChtimes(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "dated")
|
||||
if err := os.WriteFile(path, []byte("x"), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
want := time.Date(2020, 3, 4, 5, 6, 7, 0, time.UTC)
|
||||
if err := os.Chtimes(path, want, want); err != nil {
|
||||
t.Fatalf("chtimes: %v", err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat: %v", err)
|
||||
}
|
||||
if diff := info.ModTime().UTC().Sub(want); diff > 2*time.Second || diff < -2*time.Second {
|
||||
t.Fatalf("mtime = %v, want about %v", info.ModTime().UTC(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatfsIsSelfConsistent(t *testing.T) {
|
||||
if *mountPoint == "" {
|
||||
t.Skip("no -mountpoint given; this test needs a live WinFsp mount")
|
||||
}
|
||||
var free, total, totalFree uint64
|
||||
if err := getDiskFreeSpace(*mountPoint, &free, &total, &totalFree); err != nil {
|
||||
t.Fatalf("GetDiskFreeSpaceEx: %v", err)
|
||||
}
|
||||
if total == 0 {
|
||||
t.Fatal("drive reports zero total bytes")
|
||||
}
|
||||
if free > total {
|
||||
t.Fatalf("free (%d) exceeds total (%d)", free, total)
|
||||
}
|
||||
if totalFree > total {
|
||||
t.Fatalf("total free (%d) exceeds total (%d)", totalFree, total)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteOnClose covers the pattern Windows software uses for temporaries:
|
||||
// the file goes away when the last handle closes, with no explicit delete. A
|
||||
// filesystem that ignores the flag leaves the name behind, and the next
|
||||
// program to create it gets a collision.
|
||||
func TestDeleteOnClose(t *testing.T) {
|
||||
dir := testRoot(t)
|
||||
path := filepath.Join(dir, "ephemeral.tmp")
|
||||
|
||||
h, err := createDeleteOnClose(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, errWindowsOnly) {
|
||||
t.Skip("delete-on-close needs the Win32 create call")
|
||||
}
|
||||
t.Fatalf("open with delete-on-close: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
closeHandle(h)
|
||||
t.Fatalf("file is not visible while its handle is open: %v", err)
|
||||
}
|
||||
if err := closeHandle(h); err != nil {
|
||||
t.Fatalf("close: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
t.Fatal("file outlived its last handle")
|
||||
} else if !os.IsNotExist(err) {
|
||||
t.Fatalf("stat after close: %v", err)
|
||||
}
|
||||
// The name has to be free again, which is what the conformance suite
|
||||
// tripped over when an aborted test left its file behind.
|
||||
h, err = createDeleteOnClose(path)
|
||||
if err != nil {
|
||||
t.Fatalf("recreating the same name failed: %v", err)
|
||||
}
|
||||
if err := closeHandle(h); err != nil {
|
||||
t.Fatalf("close after recreate: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ const (
|
||||
type AdminData struct {
|
||||
Username string `json:"username"`
|
||||
TotalVolumes int `json:"total_volumes"`
|
||||
TotalChunks int64 `json:"total_chunks"`
|
||||
TotalFiles int64 `json:"total_files"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
VolumeSizeLimitMB uint64 `json:"volume_size_limit_mb"`
|
||||
MasterNodes []MasterNode `json:"master_nodes"`
|
||||
@@ -199,7 +199,7 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
|
||||
adminData := AdminData{
|
||||
Username: username,
|
||||
TotalVolumes: topology.TotalVolumes,
|
||||
TotalChunks: topology.TotalChunks,
|
||||
TotalFiles: topology.TotalFiles,
|
||||
TotalSize: topology.TotalSize,
|
||||
VolumeSizeLimitMB: volumeSizeLimitMB,
|
||||
MasterNodes: masterNodes,
|
||||
|
||||
@@ -1928,20 +1928,9 @@ type ecVolumeCounts struct {
|
||||
deleteCount uint64
|
||||
}
|
||||
|
||||
// volumeLiveCount is the live chunk count of one regular volume. Replicas
|
||||
// mirror each other's needles and their deletes, so the fullest report is the
|
||||
// volume's count — dividing each report by the copy count instead would lose
|
||||
// a chunk to integer truncation and would halve a volume whose second replica
|
||||
// has not reported yet.
|
||||
type volumeLiveCount struct {
|
||||
collection string
|
||||
live uint64
|
||||
}
|
||||
|
||||
func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]collectionStats {
|
||||
collectionMap := make(map[string]collectionStats)
|
||||
ecVolumeAgg := make(map[uint32]*ecVolumeCounts)
|
||||
volumeAgg := make(map[uint32]*volumeLiveCount)
|
||||
for _, dc := range topologyInfo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
for _, node := range rack.DataNodeInfos {
|
||||
@@ -1962,18 +1951,10 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col
|
||||
if volInfo.Size >= volInfo.DeletedByteCount {
|
||||
data.LogicalSize += int64(volInfo.Size-volInfo.DeletedByteCount) / replicaCount
|
||||
}
|
||||
collectionMap[collection] = data
|
||||
|
||||
if volInfo.FileCount >= volInfo.DeleteCount {
|
||||
agg, ok := volumeAgg[volInfo.Id]
|
||||
if !ok {
|
||||
agg = &volumeLiveCount{collection: collection}
|
||||
volumeAgg[volInfo.Id] = agg
|
||||
}
|
||||
if live := volInfo.FileCount - volInfo.DeleteCount; live > agg.live {
|
||||
agg.live = live
|
||||
}
|
||||
data.FileCount += int64(volInfo.FileCount-volInfo.DeleteCount) / replicaCount
|
||||
}
|
||||
collectionMap[collection] = data
|
||||
}
|
||||
for _, ecShardInfo := range diskInfo.EcShardInfos {
|
||||
collection := ecShardInfo.Collection
|
||||
@@ -2006,14 +1987,6 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col
|
||||
}
|
||||
}
|
||||
|
||||
// Fold the per-volume live counts in, one entry per volume id no matter
|
||||
// how many replicas reported it.
|
||||
for _, agg := range volumeAgg {
|
||||
data := collectionMap[agg.collection]
|
||||
data.FileCount += int64(agg.live)
|
||||
collectionMap[agg.collection] = data
|
||||
}
|
||||
|
||||
// Fold EC per-volume counts into the collection totals. fileCount is
|
||||
// deduped via max across every node reporting shards for the volume;
|
||||
// deleteCount is summed across the same nodes.
|
||||
@@ -2031,17 +2004,6 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col
|
||||
return collectionMap
|
||||
}
|
||||
|
||||
// totalCollectionFileCount is the cluster-wide live chunk count: the sum of
|
||||
// every collection's deduped count. Volumes and EC volumes are reported by
|
||||
// each replica or shard holder, so only this aggregation counts a chunk once.
|
||||
func totalCollectionFileCount(topologyInfo *master_pb.TopologyInfo) int64 {
|
||||
var total int64
|
||||
for _, stats := range collectCollectionStats(topologyInfo) {
|
||||
total += stats.FileCount
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// getCollectionStats returns current collection statistics with caching
|
||||
func (s *AdminServer) getCollectionStats() (map[string]collectionStats, error) {
|
||||
now := time.Now()
|
||||
|
||||
@@ -310,10 +310,7 @@ func (s *AdminServer) SetBucketOwner(bucketName string, owner string) error {
|
||||
bucketEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
|
||||
// Set or remove the owner. The account id recorded by the S3 API goes with
|
||||
// it: the S3 API derives it from the owning identity when it is absent, so
|
||||
// leaving the old one behind would keep the previous owner of the objects.
|
||||
delete(bucketEntry.Extended, s3_constants.ExtAmzOwnerKey)
|
||||
// Set or remove the owner
|
||||
if owner == "" {
|
||||
delete(bucketEntry.Extended, s3_constants.AmzIdentityId)
|
||||
} else {
|
||||
|
||||
@@ -124,6 +124,13 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
|
||||
}
|
||||
|
||||
if resp.TopologyInfo != nil {
|
||||
// Dedupe EC volume file counts across the nodes that report
|
||||
// shards for the same volume: every shard holder reports the
|
||||
// same .ecx-derived file_count, so we keep the max and sum
|
||||
// node-local tombstones.
|
||||
ecFile := make(map[uint32]uint64)
|
||||
ecDel := make(map[uint32]uint64)
|
||||
|
||||
// Process gRPC response
|
||||
for _, dc := range resp.TopologyInfo.DataCenterInfos {
|
||||
dataCenter := DataCenter{
|
||||
@@ -142,6 +149,7 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
|
||||
var totalVolumes int64
|
||||
var totalMaxVolumes int64
|
||||
var totalSize int64
|
||||
var totalFiles int64
|
||||
// Prefer the real physical disk capacity the volume server
|
||||
// reports per disk; the slot-based estimate overstates capacity
|
||||
// when maxVolumeCount is configured higher than the disk holds.
|
||||
@@ -159,14 +167,23 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
|
||||
// Sum up individual volume information
|
||||
for _, volInfo := range diskInfo.VolumeInfos {
|
||||
totalSize += int64(volInfo.Size)
|
||||
totalFiles += int64(volInfo.FileCount)
|
||||
}
|
||||
|
||||
// ShardSizes is local to this node, so summing
|
||||
// across nodes gives the physical footprint.
|
||||
// Sum up EC shard sizes on this node and collect
|
||||
// volume-wide file/delete counts for later folding
|
||||
// into topology.TotalFiles. ShardSizes is local to
|
||||
// this node, so summing across nodes is correct;
|
||||
// FileCount/DeleteCount are per-volume and must be
|
||||
// deduped per volume id.
|
||||
for _, ecShardInfo := range diskInfo.EcShardInfos {
|
||||
for _, shardSize := range ecShardInfo.ShardSizes {
|
||||
totalSize += shardSize
|
||||
}
|
||||
if ecShardInfo.FileCount > ecFile[ecShardInfo.Id] {
|
||||
ecFile[ecShardInfo.Id] = ecShardInfo.FileCount
|
||||
}
|
||||
ecDel[ecShardInfo.Id] += ecShardInfo.DeleteCount
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,6 +214,7 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
|
||||
rackObj.Nodes = append(rackObj.Nodes, vs)
|
||||
topology.VolumeServers = append(topology.VolumeServers, vs)
|
||||
topology.TotalVolumes += vs.Volumes
|
||||
topology.TotalFiles += totalFiles
|
||||
topology.TotalSize += totalSize
|
||||
}
|
||||
|
||||
@@ -206,10 +224,17 @@ func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
|
||||
topology.DataCenters = append(topology.DataCenters, dataCenter)
|
||||
}
|
||||
|
||||
// Chunk counts come from the shared collection aggregation, which
|
||||
// nets out tombstones and counts a chunk once no matter how many
|
||||
// volume replicas or EC shard holders report it.
|
||||
topology.TotalChunks = totalCollectionFileCount(resp.TopologyInfo)
|
||||
// Fold deduped EC file counts into the cluster total so the
|
||||
// dashboard header does not drop after volumes are converted
|
||||
// to erasure coding.
|
||||
for vid, fc := range ecFile {
|
||||
dc := ecDel[vid]
|
||||
if fc >= dc {
|
||||
topology.TotalFiles += int64(fc - dc)
|
||||
} else {
|
||||
glog.Warningf("ec volume %d: summed delete_count=%d exceeds file_count=%d; skipping from TotalFiles", vid, dc, fc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -235,96 +235,3 @@ func TestCollectCollectionStatsECFileCountMaxDedupe(t *testing.T) {
|
||||
t.Errorf("FileCount: got %d, want 6 (max across reporters)", got.FileCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTotalCollectionFileCount verifies the cluster-wide chunk count counts a
|
||||
// chunk once per replicated volume and per EC volume, and nets out deletes.
|
||||
func TestTotalCollectionFileCount(t *testing.T) {
|
||||
// Volume 1 has replication 001 (two copies), so both nodes report the same
|
||||
// 101 chunks with 10 deleted: 91 live chunks in total, not 182 and not the
|
||||
// 90 that halving each report would give.
|
||||
replica := func() *master_pb.DiskInfo {
|
||||
return &master_pb.DiskInfo{
|
||||
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
||||
{
|
||||
Id: 1,
|
||||
Collection: "bucket-a",
|
||||
ReplicaPlacement: 1,
|
||||
FileCount: 101,
|
||||
DeleteCount: 10,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
// EC volume 2 has its 20 chunks reported by every shard holder.
|
||||
ecShards := func(bits uint32, sizes []int64) *master_pb.DiskInfo {
|
||||
return &master_pb.DiskInfo{
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{
|
||||
Id: 2,
|
||||
Collection: "bucket-b",
|
||||
EcIndexBits: bits,
|
||||
ShardSizes: sizes,
|
||||
FileCount: 20,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
topo := &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{
|
||||
{
|
||||
RackInfos: []*master_pb.RackInfo{
|
||||
{
|
||||
DataNodeInfos: []*master_pb.DataNodeInfo{
|
||||
{DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"disk1": replica(),
|
||||
"disk2": ecShards((1<<0)|(1<<1), []int64{1, 1}),
|
||||
}},
|
||||
{DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"disk1": replica(),
|
||||
"disk2": ecShards((1<<2)|(1<<3), []int64{1, 1}),
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if got := totalCollectionFileCount(topo); got != 111 {
|
||||
t.Errorf("totalCollectionFileCount: got %d, want 111 (91 replicated + 20 EC)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTotalCollectionFileCountUnderReplicated verifies a volume whose second
|
||||
// replica has not reported yet still contributes its full live count.
|
||||
func TestTotalCollectionFileCountUnderReplicated(t *testing.T) {
|
||||
topo := &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{
|
||||
{
|
||||
RackInfos: []*master_pb.RackInfo{
|
||||
{
|
||||
DataNodeInfos: []*master_pb.DataNodeInfo{
|
||||
{DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"disk1": {
|
||||
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
||||
{
|
||||
Id: 1,
|
||||
Collection: "bucket-a",
|
||||
ReplicaPlacement: 1,
|
||||
FileCount: 50,
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if got := totalCollectionFileCount(topo); got != 50 {
|
||||
t.Errorf("totalCollectionFileCount: got %d, want 50 (one replica reporting, not halved)", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
var collections []CollectionInfo
|
||||
var totalVolumes int
|
||||
var totalEcVolumes int
|
||||
var totalChunks int64
|
||||
var totalFiles int64
|
||||
var totalSize int64
|
||||
collectionMap := make(map[string]*CollectionInfo)
|
||||
|
||||
@@ -46,6 +46,7 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
// Get or create collection info
|
||||
if collection, exists := collectionMap[collectionName]; exists {
|
||||
collection.VolumeCount++
|
||||
collection.FileCount += int64(volInfo.FileCount)
|
||||
collection.TotalSize += int64(volInfo.Size)
|
||||
|
||||
// Update data center if this collection spans multiple DCs
|
||||
@@ -66,6 +67,7 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
}
|
||||
|
||||
totalVolumes++
|
||||
totalFiles += int64(volInfo.FileCount)
|
||||
totalSize += int64(volInfo.Size)
|
||||
} else {
|
||||
newCollection := CollectionInfo{
|
||||
@@ -73,11 +75,13 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
DataCenter: dc.Id,
|
||||
VolumeCount: 1,
|
||||
EcVolumeCount: 0,
|
||||
FileCount: int64(volInfo.FileCount),
|
||||
TotalSize: int64(volInfo.Size),
|
||||
DiskTypes: []string{diskType},
|
||||
}
|
||||
collectionMap[collectionName] = &newCollection
|
||||
totalVolumes++
|
||||
totalFiles += int64(volInfo.FileCount)
|
||||
totalSize += int64(volInfo.Size)
|
||||
}
|
||||
}
|
||||
@@ -129,6 +133,7 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
DataCenter: dc.Id,
|
||||
VolumeCount: 0,
|
||||
EcVolumeCount: 1,
|
||||
FileCount: 0,
|
||||
TotalSize: 0,
|
||||
DiskTypes: []string{diskType},
|
||||
}
|
||||
@@ -141,16 +146,6 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Chunk counts come from the shared collection aggregation, which
|
||||
// nets out tombstones and counts a chunk once no matter how many
|
||||
// volume replicas or EC shard holders report it.
|
||||
for collectionName, stats := range collectCollectionStats(resp.TopologyInfo) {
|
||||
if collection, exists := collectionMap[collectionName]; exists {
|
||||
collection.ChunkCount = stats.FileCount
|
||||
totalChunks += stats.FileCount
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -178,7 +173,7 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
TotalCollections: 0,
|
||||
TotalVolumes: 0,
|
||||
TotalEcVolumes: 0,
|
||||
TotalChunks: 0,
|
||||
TotalFiles: 0,
|
||||
TotalSize: 0,
|
||||
LastUpdated: time.Now(),
|
||||
}, nil
|
||||
@@ -189,7 +184,7 @@ func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
|
||||
TotalCollections: len(collections),
|
||||
TotalVolumes: totalVolumes,
|
||||
TotalEcVolumes: totalEcVolumes,
|
||||
TotalChunks: totalChunks,
|
||||
TotalFiles: totalFiles,
|
||||
TotalSize: totalSize,
|
||||
LastUpdated: time.Now(),
|
||||
}, nil
|
||||
@@ -213,6 +208,7 @@ func (s *AdminServer) GetCollectionDetails(collectionName string, page int, page
|
||||
|
||||
var regularVolumes []VolumeWithTopology
|
||||
var ecVolumes []EcVolumeWithShards
|
||||
var totalFiles int64
|
||||
var totalSize int64
|
||||
dataCenters := make(map[string]bool)
|
||||
diskTypes := make(map[string]bool)
|
||||
@@ -226,15 +222,10 @@ func (s *AdminServer) GetCollectionDetails(collectionName string, page int, page
|
||||
regularVolumes = regularVolumeData.Volumes
|
||||
totalSize = regularVolumeData.TotalSize
|
||||
|
||||
// Chunk counts come from the shared collection aggregation, which nets out
|
||||
// tombstones and counts a chunk once no matter how many volume replicas or
|
||||
// EC shard holders report it. Fail rather than render a zero that reads
|
||||
// like an empty collection.
|
||||
stats, err := s.getCollectionStats()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// Calculate total files from regular volumes
|
||||
for _, vol := range regularVolumes {
|
||||
totalFiles += int64(vol.FileCount)
|
||||
}
|
||||
totalChunks := stats[collectionName].FileCount
|
||||
|
||||
// Collect data centers and disk types from regular volumes
|
||||
for _, vol := range regularVolumes {
|
||||
@@ -380,7 +371,7 @@ func (s *AdminServer) GetCollectionDetails(collectionName string, page int, page
|
||||
EcVolumes: paginatedEcVolumes,
|
||||
TotalVolumes: len(regularVolumes),
|
||||
TotalEcVolumes: len(ecVolumes),
|
||||
TotalChunks: totalChunks,
|
||||
TotalFiles: totalFiles,
|
||||
TotalSize: totalSize,
|
||||
DataCenters: dcList,
|
||||
DiskTypes: diskTypeList,
|
||||
|
||||
@@ -18,7 +18,7 @@ type dashSample struct {
|
||||
volumes float64
|
||||
ecShards float64
|
||||
diskUsed float64
|
||||
chunks float64
|
||||
files float64
|
||||
tasks float64 // pending/assigned/in-progress maintenance tasks
|
||||
workers float64
|
||||
}
|
||||
@@ -33,7 +33,7 @@ type DashboardTrends struct {
|
||||
|
||||
// Sparklines (raw <svg>) for the existing summary cards.
|
||||
Volumes string `json:"-"`
|
||||
Chunks string `json:"-"`
|
||||
Files string `json:"-"`
|
||||
DiskUsed string `json:"-"`
|
||||
EcShards string `json:"-"`
|
||||
|
||||
@@ -61,7 +61,7 @@ func (s *AdminServer) recordDashboardSample() {
|
||||
volumes: float64(topology.TotalVolumes),
|
||||
ecShards: float64(ecShards),
|
||||
diskUsed: float64(topology.TotalSize),
|
||||
chunks: float64(topology.TotalChunks),
|
||||
files: float64(topology.TotalFiles),
|
||||
}
|
||||
if s.maintenanceManager != nil {
|
||||
if stats := s.maintenanceManager.GetStats(); stats != nil {
|
||||
@@ -106,7 +106,7 @@ func (s *AdminServer) GetDashboardTrends() DashboardTrends {
|
||||
return DashboardTrends{
|
||||
Samples: len(samples),
|
||||
Volumes: sparklineSVG(series(func(s dashSample) float64 { return s.volumes }), "#1cc88a"), // success
|
||||
Chunks: sparklineSVG(series(func(s dashSample) float64 { return s.chunks }), "#36b9cc"), // info
|
||||
Files: sparklineSVG(series(func(s dashSample) float64 { return s.files }), "#36b9cc"), // info
|
||||
DiskUsed: sparklineSVG(series(func(s dashSample) float64 { return s.diskUsed }), "#f6c23e"), // warning
|
||||
EcShards: sparklineSVG(series(func(s dashSample) float64 { return s.ecShards }), "#5a5c69"), // dark
|
||||
Tasks: sparklineSVG(tasks, "#36b9cc"),
|
||||
|
||||
@@ -14,11 +14,9 @@ type ClusterTopology struct {
|
||||
DataCenters []DataCenter `json:"datacenters"`
|
||||
VolumeServers []VolumeServer `json:"volume_servers"`
|
||||
TotalVolumes int `json:"total_volumes"`
|
||||
// TotalChunks counts chunks stored in volumes, not filer entries: a file
|
||||
// is split into one or more chunks.
|
||||
TotalChunks int64 `json:"total_chunks"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
TotalFiles int64 `json:"total_files"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type MasterNode struct {
|
||||
@@ -276,7 +274,7 @@ type CollectionInfo struct {
|
||||
DataCenter string `json:"datacenter"`
|
||||
VolumeCount int `json:"volume_count"`
|
||||
EcVolumeCount int `json:"ec_volume_count"`
|
||||
ChunkCount int64 `json:"chunk_count"`
|
||||
FileCount int64 `json:"file_count"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
DiskTypes []string `json:"disk_types"`
|
||||
}
|
||||
@@ -287,7 +285,7 @@ type ClusterCollectionsData struct {
|
||||
TotalCollections int `json:"total_collections"`
|
||||
TotalVolumes int `json:"total_volumes"`
|
||||
TotalEcVolumes int `json:"total_ec_volumes"`
|
||||
TotalChunks int64 `json:"total_chunks"`
|
||||
TotalFiles int64 `json:"total_files"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
@@ -590,7 +588,7 @@ type CollectionDetailsData struct {
|
||||
EcVolumes []EcVolumeWithShards `json:"ec_volumes"`
|
||||
TotalVolumes int `json:"total_volumes"`
|
||||
TotalEcVolumes int `json:"total_ec_volumes"`
|
||||
TotalChunks int64 `json:"total_chunks"`
|
||||
TotalFiles int64 `json:"total_files"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
DataCenters []string `json:"data_centers"`
|
||||
DiskTypes []string `json:"disk_types"`
|
||||
|
||||
@@ -577,7 +577,7 @@ func (h *AdminHandlers) getAdminData(r *http.Request) dash.AdminData {
|
||||
return dash.AdminData{
|
||||
Username: username,
|
||||
TotalVolumes: 0,
|
||||
TotalChunks: 0,
|
||||
TotalFiles: 0,
|
||||
TotalSize: 0,
|
||||
MasterNodes: masterNodes,
|
||||
VolumeServers: []dash.VolumeServer{},
|
||||
|
||||
@@ -50,18 +50,18 @@ templ Admin(data dash.AdminData) {
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-info text-uppercase mb-1" title="Chunks stored in volumes. A file is split into one or more chunks, so this is not the number of files.">
|
||||
Total Chunks
|
||||
<div class="text-xs font-weight-bold text-info text-uppercase mb-1">
|
||||
Total Files
|
||||
</div>
|
||||
<div class="h5 mb-0 font-weight-bold text-gray-800">
|
||||
{formatNumber(data.TotalChunks)}
|
||||
{formatNumber(data.TotalFiles)}
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-cubes fa-2x text-gray-300"></i>
|
||||
<i class="fas fa-file fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
@templ.Raw(data.Trends.Chunks)
|
||||
@templ.Raw(data.Trends.Files)
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Code generated by templ - DO NOT EDIT.
|
||||
|
||||
// templ: version: v0.3.1020
|
||||
// templ: version: v0.3.1001
|
||||
package app
|
||||
|
||||
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||
@@ -68,24 +68,24 @@ func Admin(data dash.AdminData) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</div></div></div><div class=\"col-xl-3 col-md-6 mb-4\"><div class=\"card border-left-info shadow h-100 py-2\"><div class=\"card-body\"><div class=\"row no-gutters align-items-center\"><div class=\"col mr-2\"><div class=\"text-xs font-weight-bold text-info text-uppercase mb-1\" title=\"Chunks stored in volumes. A file is split into one or more chunks, so this is not the number of files.\">Total Chunks</div><div class=\"h5 mb-0 font-weight-bold text-gray-800\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</div></div></div><div class=\"col-xl-3 col-md-6 mb-4\"><div class=\"card border-left-info shadow h-100 py-2\"><div class=\"card-body\"><div class=\"row no-gutters align-items-center\"><div class=\"col mr-2\"><div class=\"text-xs font-weight-bold text-info text-uppercase mb-1\">Total Files</div><div class=\"h5 mb-0 font-weight-bold text-gray-800\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var4 string
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(formatNumber(data.TotalChunks))
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(formatNumber(data.TotalFiles))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 57, Col: 67}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/admin.templ`, Line: 57, Col: 66}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</div></div><div class=\"col-auto\"><i class=\"fas fa-cubes fa-2x text-gray-300\"></i></div></div>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</div></div><div class=\"col-auto\"><i class=\"fas fa-file fa-2x text-gray-300\"></i></div></div>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templ.Raw(data.Trends.Chunks).Render(ctx, templ_7745c5c3_Buffer)
|
||||
templ_7745c5c3_Err = templ.Raw(data.Trends.Files).Render(ctx, templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
@@ -87,15 +87,15 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-warning text-uppercase mb-1" title="Chunks stored in volumes. A file is split into one or more chunks, so this is not the number of files.">
|
||||
Total Chunks
|
||||
<div class="text-xs font-weight-bold text-warning text-uppercase mb-1">
|
||||
Total Files
|
||||
</div>
|
||||
<div class="h5 mb-0 font-weight-bold text-gray-800">
|
||||
{fmt.Sprintf("%d", data.TotalChunks)}
|
||||
{fmt.Sprintf("%d", data.TotalFiles)}
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-cubes fa-2x text-gray-300"></i>
|
||||
<i class="fas fa-file fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -139,7 +139,7 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
|
||||
<th>Collection Name</th>
|
||||
<th>Regular Volumes</th>
|
||||
<th>EC Volumes</th>
|
||||
<th>Chunks</th>
|
||||
<th>Files</th>
|
||||
<th>Size (Logical)</th>
|
||||
<th>Disk Types</th>
|
||||
<th>Actions</th>
|
||||
@@ -179,8 +179,8 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
|
||||
</td>
|
||||
<td>
|
||||
<div class="d-flex align-items-center">
|
||||
<i class="fas fa-cubes me-2 text-muted"></i>
|
||||
{fmt.Sprintf("%d", collection.ChunkCount)}
|
||||
<i class="fas fa-file me-2 text-muted"></i>
|
||||
{fmt.Sprintf("%d", collection.FileCount)}
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
@@ -209,7 +209,7 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
|
||||
data-datacenter={collection.DataCenter}
|
||||
data-volume-count={fmt.Sprintf("%d", collection.VolumeCount)}
|
||||
data-ec-volume-count={fmt.Sprintf("%d", collection.EcVolumeCount)}
|
||||
data-chunk-count={fmt.Sprintf("%d", collection.ChunkCount)}
|
||||
data-file-count={fmt.Sprintf("%d", collection.FileCount)}
|
||||
data-total-size={fmt.Sprintf("%d", collection.TotalSize)}
|
||||
data-disk-types={formatDiskTypes(collection.DiskTypes)}>
|
||||
<i class="fas fa-eye"></i>
|
||||
@@ -262,7 +262,7 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
|
||||
datacenter: button.getAttribute('data-datacenter'),
|
||||
volumeCount: parseInt(button.getAttribute('data-volume-count')),
|
||||
ecVolumeCount: parseInt(button.getAttribute('data-ec-volume-count')),
|
||||
chunkCount: parseInt(button.getAttribute('data-chunk-count')),
|
||||
fileCount: parseInt(button.getAttribute('data-file-count')),
|
||||
totalSize: parseInt(button.getAttribute('data-total-size')),
|
||||
diskTypes: button.getAttribute('data-disk-types')
|
||||
};
|
||||
@@ -311,10 +311,10 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
|
||||
'<span>' + collection.ecVolumeCount.toLocaleString() + '</span>' +
|
||||
'</div>' +
|
||||
'</td></tr>' +
|
||||
'<tr><td><strong>Total Chunks:</strong></td><td>' +
|
||||
'<tr><td><strong>Total Files:</strong></td><td>' +
|
||||
'<div class="d-flex align-items-center">' +
|
||||
'<i class="fas fa-cubes me-2 text-muted"></i>' +
|
||||
'<span>' + collection.chunkCount.toLocaleString() + '</span>' +
|
||||
'<i class="fas fa-file me-2 text-muted"></i>' +
|
||||
'<span>' + collection.fileCount.toLocaleString() + '</span>' +
|
||||
'</div>' +
|
||||
'</td></tr>' +
|
||||
'<tr><td><strong>Total Size (Logical):</strong></td><td>' +
|
||||
@@ -396,18 +396,17 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
|
||||
return {
|
||||
name: cells[0].textContent.trim(),
|
||||
volumes: cells[1].textContent.trim(),
|
||||
ecVolumes: cells[2].textContent.trim(),
|
||||
chunks: cells[3].textContent.trim(),
|
||||
size: cells[4].textContent.trim(),
|
||||
diskTypes: cells[5].textContent.trim()
|
||||
files: cells[2].textContent.trim(),
|
||||
size: cells[3].textContent.trim(),
|
||||
diskTypes: cells[4].textContent.trim()
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}).filter(row => row !== null);
|
||||
|
||||
const csvContent = "data:text/csv;charset=utf-8," +
|
||||
"Collection Name,Volumes,EC Volumes,Chunks,Size,Disk Types\n" +
|
||||
rows.map(r => '"' + r.name + '","' + r.volumes + '","' + r.ecVolumes + '","' + r.chunks + '","' + r.size + '","' + r.diskTypes + '"').join("\n");
|
||||
|
||||
const csvContent = "data:text/csv;charset=utf-8," +
|
||||
"Collection Name,Volumes,Files,Size,Disk Types\n" +
|
||||
rows.map(r => '"' + r.name + '","' + r.volumes + '","' + r.files + '","' + r.size + '","' + r.diskTypes + '"').join("\n");
|
||||
|
||||
const encodedUri = encodeURI(csvContent);
|
||||
const link = document.createElement("a");
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -71,12 +71,12 @@ templ CollectionDetails(data dash.CollectionDetailsData) {
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Total Chunks</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.TotalChunks)}</h4>
|
||||
<small>A file is split into one or more chunks</small>
|
||||
<h6 class="card-title">Total Files</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.TotalFiles)}</h4>
|
||||
<small>Files stored</small>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-cubes fa-2x"></i>
|
||||
<i class="fas fa-file fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -169,7 +169,7 @@ templ CollectionDetails(data dash.CollectionDetailsData) {
|
||||
</a>
|
||||
</th>
|
||||
<th class="text-dark">Logical Size / Shard Count</th>
|
||||
<th class="text-dark">Chunks</th>
|
||||
<th class="text-dark">Files</th>
|
||||
<th class="text-dark">Status</th>
|
||||
<th class="text-dark">Actions</th>
|
||||
</tr>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Code generated by templ - DO NOT EDIT.
|
||||
|
||||
// templ: version: v0.3.1020
|
||||
// templ: version: v0.3.1001
|
||||
package app
|
||||
|
||||
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||
@@ -113,20 +113,20 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</h4><small>Erasure coded volumes</small></div><div class=\"align-self-center\"><i class=\"fas fa-th-large fa-2x\"></i></div></div></div></div></div><div class=\"col-md-3\"><div class=\"card text-bg-success\"><div class=\"card-body\"><div class=\"d-flex justify-content-between\"><div><h6 class=\"card-title\">Total Chunks</h6><h4 class=\"mb-0\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</h4><small>Erasure coded volumes</small></div><div class=\"align-self-center\"><i class=\"fas fa-th-large fa-2x\"></i></div></div></div></div></div><div class=\"col-md-3\"><div class=\"card text-bg-success\"><div class=\"card-body\"><div class=\"d-flex justify-content-between\"><div><h6 class=\"card-title\">Total Files</h6><h4 class=\"mb-0\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var8 string
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalChunks))
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalFiles))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 75, Col: 60}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 75, Col: 59}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</h4><small>A file is split into one or more chunks</small></div><div class=\"align-self-center\"><i class=\"fas fa-cubes fa-2x\"></i></div></div></div></div></div><div class=\"col-md-3\"><div class=\"card text-bg-warning\"><div class=\"card-body\"><div class=\"d-flex justify-content-between\"><div><h6 class=\"card-title\">Total Size (Logical)</h6><h4 class=\"mb-0\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</h4><small>Files stored</small></div><div class=\"align-self-center\"><i class=\"fas fa-file fa-2x\"></i></div></div></div></div></div><div class=\"col-md-3\"><div class=\"card text-bg-warning\"><div class=\"card-body\"><div class=\"d-flex justify-content-between\"><div><h6 class=\"card-title\">Total Size (Logical)</h6><h4 class=\"mb-0\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -269,7 +269,7 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</a></th><th class=\"text-dark\">Logical Size / Shard Count</th><th class=\"text-dark\">Chunks</th><th class=\"text-dark\">Status</th><th class=\"text-dark\">Actions</th></tr></thead> <tbody>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</a></th><th class=\"text-dark\">Logical Size / Shard Count</th><th class=\"text-dark\">Files</th><th class=\"text-dark\">Status</th><th class=\"text-dark\">Actions</th></tr></thead> <tbody>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -333,11 +333,11 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var16 string
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.ResolveAttributeValue(fmt.Sprintf("%d", volume.Id))
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", volume.Id))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 206, Col: 55}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var16)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -346,11 +346,11 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var17 string
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.ResolveAttributeValue(volume.Server)
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(volume.Server)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 207, Col: 37}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var17)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -419,11 +419,11 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var21 string
|
||||
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.ResolveAttributeValue(fmt.Sprintf("%d", ecVolume.VolumeID))
|
||||
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", ecVolume.VolumeID))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 249, Col: 63}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var21)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -466,11 +466,11 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var23 string
|
||||
templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.ResolveAttributeValue(fmt.Sprintf("%d", data.Page-1))
|
||||
templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.Page-1))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 280, Col: 104}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var23)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -504,11 +504,11 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var25 string
|
||||
templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.ResolveAttributeValue(fmt.Sprintf("%d", i))
|
||||
templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", i))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 291, Col: 95}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var25)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -547,11 +547,11 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var27 string
|
||||
templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.ResolveAttributeValue(fmt.Sprintf("%d", data.Page+1))
|
||||
templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.Page+1))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 306, Col: 104}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var27)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -560,11 +560,11 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var28 string
|
||||
templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.ResolveAttributeValue(fmt.Sprintf("%d", data.TotalPages))
|
||||
templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalPages))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/collection_details.templ`, Line: 309, Col: 108}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ_7745c5c3_Var28)
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
|
||||
@@ -234,7 +234,6 @@ catalog = load_catalog(
|
||||
**{
|
||||
"type": "rest",
|
||||
"uri": "http://localhost:` + fmt.Sprintf("%d", data.IcebergPort) + `",
|
||||
"warehouse": "s3://my-table-bucket/",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -302,14 +302,13 @@ catalog = load_catalog(
|
||||
**{
|
||||
"type": "rest",
|
||||
"uri": "http://localhost:` + fmt.Sprintf("%d", data.IcebergPort) + `",
|
||||
"warehouse": "s3://my-table-bucket/",
|
||||
}
|
||||
)
|
||||
|
||||
# List namespaces
|
||||
namespaces = catalog.list_namespaces()`)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/iceberg_catalog.templ`, Line: 242, Col: 39}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/iceberg_catalog.templ`, Line: 241, Col: 39}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
|
||||
@@ -235,7 +235,6 @@ catalog = load_catalog(
|
||||
**{
|
||||
"type": "rest",
|
||||
"uri": "http://localhost:` + fmt.Sprintf("%d", data.IcebergPort) + `",
|
||||
"warehouse": "s3://my-table-bucket/",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -361,13 +361,12 @@ catalog = load_catalog(
|
||||
**{
|
||||
"type": "rest",
|
||||
"uri": "http://localhost:` + fmt.Sprintf("%d", data.IcebergPort) + `",
|
||||
"warehouse": "s3://my-table-bucket/",
|
||||
}
|
||||
)
|
||||
|
||||
namespaces = catalog.list_namespaces()`)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 242, Col: 39}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_buckets.templ`, Line: 241, Col: 39}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
|
||||
@@ -257,7 +257,6 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
startDelay := time.Duration(2)
|
||||
if *filerStartS3 {
|
||||
filerS3Options.filer = &filerAddress
|
||||
filerS3Options.ip = f.ip
|
||||
if *filerS3Options.bindIp == "" {
|
||||
filerS3Options.bindIp = f.bindIp
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ func init() {
|
||||
m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft")
|
||||
m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster")
|
||||
m.telemetryUrl = cmdMaster.Flag.String("telemetry.url", "https://telemetry.seaweedfs.com/api/collect", "telemetry server URL to send usage statistics")
|
||||
m.telemetryEnabled = cmdMaster.Flag.Bool("telemetry", true, "report anonymous cluster statistics to telemetry.url, use -telemetry=false to opt out")
|
||||
m.telemetryEnabled = cmdMaster.Flag.Bool("telemetry", false, "enable telemetry reporting")
|
||||
m.debug = cmdMaster.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
|
||||
m.debugPort = cmdMaster.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
}
|
||||
|
||||
@@ -433,7 +433,7 @@ func initMiniMasterFlags() {
|
||||
miniMasterOptions.raftHashicorp = cmdMini.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
miniMasterOptions.raftBootstrap = cmdMini.Flag.Bool("master.raftBootstrap", false, "whether to bootstrap the Raft cluster")
|
||||
miniMasterOptions.telemetryUrl = cmdMini.Flag.String("master.telemetry.url", "https://telemetry.seaweedfs.com/api/collect", "telemetry server URL")
|
||||
miniMasterOptions.telemetryEnabled = cmdMini.Flag.Bool("master.telemetry", true, "report anonymous cluster statistics to master.telemetry.url, use -master.telemetry=false to opt out")
|
||||
miniMasterOptions.telemetryEnabled = cmdMini.Flag.Bool("master.telemetry", false, "enable telemetry reporting")
|
||||
}
|
||||
|
||||
// initMiniFilerFlags initializes Filer server flag options
|
||||
@@ -1223,7 +1223,6 @@ func runMini(cmd *Command, args []string) bool {
|
||||
miniFilerOptions.masters = pb.ServerAddresses(actualPeersForComponents).ToServiceDiscovery()
|
||||
miniFilerOptions.ip = miniIp
|
||||
miniFilerOptions.bindIp = miniBindIp
|
||||
miniS3Options.ip = miniIp
|
||||
miniS3Options.bindIp = miniBindIp
|
||||
miniWebDavOptions.ipBind = miniBindIp
|
||||
miniOptions.v.ip = miniIp
|
||||
|
||||
@@ -39,8 +39,6 @@ type MountOptions struct {
|
||||
debugFuse *bool
|
||||
localSocket *string
|
||||
disableXAttr *bool
|
||||
windowsUid *int
|
||||
windowsGid *int
|
||||
extraOptions []string
|
||||
fuseCommandPid int
|
||||
|
||||
@@ -56,11 +54,11 @@ type MountOptions struct {
|
||||
rdmaTimeoutMs *int
|
||||
|
||||
// Peer chunk sharing options (design-weed-mount-peer-chunk-sharing.md).
|
||||
peerEnabled *bool
|
||||
peerListen *string
|
||||
peerAdvertise *string
|
||||
peerDataCenter *string
|
||||
peerRack *string
|
||||
peerEnabled *bool
|
||||
peerListen *string
|
||||
peerAdvertise *string
|
||||
peerDataCenter *string
|
||||
peerRack *string
|
||||
|
||||
dirIdleEvictSec *int
|
||||
|
||||
@@ -129,8 +127,6 @@ func init() {
|
||||
mountOptions.debugFuse = cmdMount.Flag.Bool("debug.fuse", false, "log raw FUSE protocol requests and responses")
|
||||
mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")
|
||||
mountOptions.disableXAttr = cmdMount.Flag.Bool("disableXAttr", false, "disable xattr")
|
||||
mountOptions.windowsUid = cmdMount.Flag.Int("windows.uid", 0, "windows only: uid recorded on entries this mount creates, which other clients read")
|
||||
mountOptions.windowsGid = cmdMount.Flag.Int("windows.gid", 0, "windows only: gid recorded on entries this mount creates, which other clients read")
|
||||
mountOptions.hasAutofs = cmdMount.Flag.Bool("autofs", false, "ignore autofs mounted on the same mountpoint (useful when systemd.automount and autofs is used)")
|
||||
mountOptions.fuseCommandPid = 0
|
||||
|
||||
|
||||
@@ -1,277 +0,0 @@
|
||||
//go:build linux || darwin || freebsd || windows
|
||||
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
)
|
||||
|
||||
func runMount(cmd *Command, args []string) bool {
|
||||
|
||||
if *mountOptions.debug {
|
||||
go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil)
|
||||
}
|
||||
|
||||
*mountCpuProfile = util.ResolvePath(*mountCpuProfile)
|
||||
*mountMemProfile = util.ResolvePath(*mountMemProfile)
|
||||
grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
|
||||
if *mountReadRetryTime < time.Second {
|
||||
*mountReadRetryTime = time.Second
|
||||
}
|
||||
util.RetryWaitTime = *mountReadRetryTime
|
||||
|
||||
// 32 bits, not 64: os.FileMode is uint32, so a wider parse would let a
|
||||
// nonsense umask truncate silently instead of being rejected here.
|
||||
umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 32)
|
||||
if umaskErr != nil {
|
||||
fmt.Printf("can not parse umask %s", *mountOptions.umaskString)
|
||||
return false
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
return RunMount(&mountOptions, os.FileMode(umask))
|
||||
}
|
||||
|
||||
func ensureBucketAllowEmptyFolders(ctx context.Context, filerClient filer_pb.FilerClient, mountRoot, bucketRootPath string) error {
|
||||
bucketPath, isBucketRootMount := bucketPathForMountRoot(mountRoot, bucketRootPath)
|
||||
if !isBucketRootMount {
|
||||
return nil
|
||||
}
|
||||
|
||||
entry, _, _, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry == nil {
|
||||
return fmt.Errorf("bucket %s not found", bucketPath)
|
||||
}
|
||||
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(string(entry.Extended[s3_constants.ExtAllowEmptyFolders])), "true") {
|
||||
return nil
|
||||
}
|
||||
|
||||
entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte("true")
|
||||
|
||||
bucketFullPath := util.FullPath(bucketPath)
|
||||
parent, _ := bucketFullPath.DirAndName()
|
||||
if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
|
||||
Directory: parent,
|
||||
Entry: entry,
|
||||
})
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
glog.V(3).Infof("RunMount: set bucket %s %s=true", bucketPath, s3_constants.ExtAllowEmptyFolders)
|
||||
return nil
|
||||
}
|
||||
|
||||
func bucketPathForMountRoot(mountRoot, bucketRootPath string) (string, bool) {
|
||||
cleanPath := path.Clean("/" + strings.TrimPrefix(mountRoot, "/"))
|
||||
cleanBucketRoot := path.Clean("/" + strings.TrimPrefix(bucketRootPath, "/"))
|
||||
if cleanBucketRoot == "/" {
|
||||
return "", false
|
||||
}
|
||||
prefix := cleanBucketRoot + "/"
|
||||
if !strings.HasPrefix(cleanPath, prefix) {
|
||||
return "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(cleanPath, prefix)
|
||||
|
||||
bucketParts := strings.Split(rest, "/")
|
||||
if len(bucketParts) != 1 || bucketParts[0] == "" {
|
||||
return "", false
|
||||
}
|
||||
return cleanBucketRoot + "/" + bucketParts[0], true
|
||||
}
|
||||
|
||||
func peerStringOrEmpty(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
// connectToFiler retries the filer handshake, returning the cluster's cipher
|
||||
// setting and bucket root.
|
||||
func connectToFiler(option *MountOptions) (filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialOption, cipher bool, bucketRootPath string, ok bool) {
|
||||
// try to connect to filer
|
||||
filerAddresses = pb.ServerAddresses(*option.filer).ToAddresses()
|
||||
util.LoadSecurityConfiguration()
|
||||
grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
|
||||
var err error
|
||||
for i := 0; i < 10; i++ {
|
||||
err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer grpc address %v configuration: %w", filerAddresses, err)
|
||||
}
|
||||
cipher = resp.Cipher
|
||||
bucketRootPath = resp.DirBuckets
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
|
||||
glog.V(0).Infof("wait for %d seconds ...", i+1)
|
||||
time.Sleep(time.Duration(i+1) * time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
|
||||
return nil, nil, false, "", false
|
||||
}
|
||||
if bucketRootPath == "" {
|
||||
bucketRootPath = "/buckets"
|
||||
}
|
||||
return filerAddresses, grpcDialOption, cipher, bucketRootPath, true
|
||||
}
|
||||
|
||||
// fileSystemParams are the pieces of a mount that each platform works out for
|
||||
// itself: where it is attached, and whose identity the entries carry.
|
||||
type fileSystemParams struct {
|
||||
dir string
|
||||
mountRoot string
|
||||
filerAddresses []pb.ServerAddress
|
||||
grpcDialOption grpc.DialOption
|
||||
cipher bool
|
||||
uidGidMapper *meta_cache.UidGidMapper
|
||||
uid uint32
|
||||
gid uint32
|
||||
mountMode os.FileMode
|
||||
mountCtime time.Time
|
||||
umask os.FileMode
|
||||
chunkSizeLimitMB int
|
||||
cacheDirForRead string
|
||||
cacheDirForWrite string
|
||||
}
|
||||
|
||||
func buildSeaweedFileSystem(option *MountOptions, p fileSystemParams) *mount.WFS {
|
||||
return mount.NewSeaweedFileSystem(&mount.Option{
|
||||
MountDirectory: p.dir,
|
||||
FilerAddresses: p.filerAddresses,
|
||||
GrpcDialOption: p.grpcDialOption,
|
||||
FilerSigningKey: security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key")),
|
||||
FilerSigningExpiresAfterSec: util.GetViper().GetInt("jwt.filer_signing.expires_after_seconds"),
|
||||
FilerMountRootPath: p.mountRoot,
|
||||
Collection: *option.collection,
|
||||
Replication: *option.replication,
|
||||
TtlSec: int32(*option.ttlSec),
|
||||
DiskType: types.ToDiskType(*option.diskType),
|
||||
ChunkSizeLimit: int64(p.chunkSizeLimitMB) * 1024 * 1024,
|
||||
ConcurrentWriters: *option.concurrentWriters,
|
||||
ConcurrentReaders: *option.concurrentReaders,
|
||||
CacheDirForRead: p.cacheDirForRead,
|
||||
CacheSizeMBForRead: *option.cacheSizeMBForRead,
|
||||
CacheDirForWrite: p.cacheDirForWrite,
|
||||
WriteBufferSizeMB: *option.writeBufferSizeMB,
|
||||
CacheMetaTTlSec: *option.cacheMetaTtlSec,
|
||||
DataCenter: *option.dataCenter,
|
||||
Quota: int64(*option.collectionQuota) * 1024 * 1024,
|
||||
LogicalDiskUsage: *option.logicalDiskUsage,
|
||||
MountUid: p.uid,
|
||||
MountGid: p.gid,
|
||||
MountMode: p.mountMode,
|
||||
MountCtime: p.mountCtime,
|
||||
MountMtime: time.Now(),
|
||||
Umask: p.umask,
|
||||
VolumeServerAccess: *mountOptions.volumeServerAccess,
|
||||
Cipher: p.cipher,
|
||||
UidGidMapper: p.uidGidMapper,
|
||||
IncludeSystemEntries: *option.includeSystemEntries,
|
||||
DefaultPermissions: *option.defaultPermissions,
|
||||
DisableXAttr: *option.disableXAttr,
|
||||
IsMacOs: runtime.GOOS == "darwin",
|
||||
MetadataFlushSeconds: *option.metadataFlushSeconds,
|
||||
// RDMA acceleration options
|
||||
RdmaEnabled: *option.rdmaEnabled,
|
||||
RdmaSidecarAddr: *option.rdmaSidecarAddr,
|
||||
RdmaFallback: *option.rdmaFallback,
|
||||
RdmaReadOnly: *option.rdmaReadOnly,
|
||||
RdmaMaxConcurrent: *option.rdmaMaxConcurrent,
|
||||
RdmaTimeoutMs: *option.rdmaTimeoutMs,
|
||||
DirIdleEvictSec: *option.dirIdleEvictSec,
|
||||
EnableDistributedLock: option.distributedLock != nil && *option.distributedLock,
|
||||
WritebackCache: option.writebackCache != nil && *option.writebackCache,
|
||||
PosixDirNlink: option.posixDirNlink != nil && *option.posixDirNlink,
|
||||
// Peer chunk sharing
|
||||
PeerEnabled: option.peerEnabled != nil && *option.peerEnabled,
|
||||
PeerListen: peerStringOrEmpty(option.peerListen),
|
||||
PeerAdvertise: peerStringOrEmpty(option.peerAdvertise),
|
||||
PeerDataCenter: peerStringOrEmpty(option.peerDataCenter),
|
||||
PeerRack: peerStringOrEmpty(option.peerRack),
|
||||
})
|
||||
}
|
||||
|
||||
// createMountRoot makes the filer-side directory the mount is rooted at.
|
||||
func createMountRoot(wfs *mount.WFS, mountRoot, bucketRootPath string, filerAddresses []pb.ServerAddress) bool {
|
||||
mountRootPath := util.FullPath(mountRoot)
|
||||
mountRootParent, mountDir := mountRootPath.DirAndName()
|
||||
if err := filer_pb.Mkdir(context.Background(), wfs, mountRootParent, mountDir, nil); err != nil {
|
||||
fmt.Printf("failed to create dir %s on filer %s: %v\n", mountRoot, filerAddresses, err)
|
||||
return false
|
||||
}
|
||||
if err := ensureBucketAllowEmptyFolders(context.Background(), wfs, mountRoot, bucketRootPath); err != nil {
|
||||
fmt.Printf("failed to set bucket auto-remove-empty-folders policy for %s: %v\n", mountRoot, err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// serveMountGrpc exposes the local control socket used by "weed mount.stats".
|
||||
func serveMountGrpc(wfs *mount.WFS, listener net.Listener) {
|
||||
grpcS := pb.NewGrpcServer()
|
||||
mount_pb.RegisterSeaweedMountServer(grpcS, wfs)
|
||||
reflection.Register(grpcS)
|
||||
go grpcS.Serve(listener)
|
||||
}
|
||||
|
||||
// resolveMountRoot trims the trailing slash the filer path must not carry.
|
||||
func resolveMountRoot(filerMountRootPath string) string {
|
||||
mountRoot := filerMountRootPath
|
||||
if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
|
||||
mountRoot = mountRoot[0 : len(mountRoot)-1]
|
||||
}
|
||||
return mountRoot
|
||||
}
|
||||
|
||||
// resolveCacheDirs falls back to the read cache when no write cache is set.
|
||||
func resolveCacheDirs(option *MountOptions) (string, string) {
|
||||
cacheDirForRead := util.ResolvePath(*option.cacheDirForRead)
|
||||
cacheDirForWrite := util.ResolvePath(*option.cacheDirForWrite)
|
||||
if cacheDirForWrite == "" {
|
||||
cacheDirForWrite = cacheDirForRead
|
||||
}
|
||||
return cacheDirForRead, cacheDirForWrite
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build !linux && !darwin && !freebsd && !windows
|
||||
//go:build !linux && !darwin && !freebsd
|
||||
|
||||
package command
|
||||
|
||||
|
||||
+209
-24
@@ -3,26 +3,122 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
||||
|
||||
"github.com/seaweedfs/go-fuse/v2/fuse"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount/unmount"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
)
|
||||
|
||||
func runMount(cmd *Command, args []string) bool {
|
||||
|
||||
if *mountOptions.debug {
|
||||
go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil)
|
||||
}
|
||||
|
||||
*mountCpuProfile = util.ResolvePath(*mountCpuProfile)
|
||||
*mountMemProfile = util.ResolvePath(*mountMemProfile)
|
||||
grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
|
||||
if *mountReadRetryTime < time.Second {
|
||||
*mountReadRetryTime = time.Second
|
||||
}
|
||||
util.RetryWaitTime = *mountReadRetryTime
|
||||
|
||||
umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 64)
|
||||
if umaskErr != nil {
|
||||
fmt.Printf("can not parse umask %s", *mountOptions.umaskString)
|
||||
return false
|
||||
}
|
||||
|
||||
if len(args) > 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
return RunMount(&mountOptions, os.FileMode(umask))
|
||||
}
|
||||
|
||||
func ensureBucketAllowEmptyFolders(ctx context.Context, filerClient filer_pb.FilerClient, mountRoot, bucketRootPath string) error {
|
||||
bucketPath, isBucketRootMount := bucketPathForMountRoot(mountRoot, bucketRootPath)
|
||||
if !isBucketRootMount {
|
||||
return nil
|
||||
}
|
||||
|
||||
entry, _, _, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry == nil {
|
||||
return fmt.Errorf("bucket %s not found", bucketPath)
|
||||
}
|
||||
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(string(entry.Extended[s3_constants.ExtAllowEmptyFolders])), "true") {
|
||||
return nil
|
||||
}
|
||||
|
||||
entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte("true")
|
||||
|
||||
bucketFullPath := util.FullPath(bucketPath)
|
||||
parent, _ := bucketFullPath.DirAndName()
|
||||
if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
|
||||
Directory: parent,
|
||||
Entry: entry,
|
||||
})
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
glog.V(3).Infof("RunMount: set bucket %s %s=true", bucketPath, s3_constants.ExtAllowEmptyFolders)
|
||||
return nil
|
||||
}
|
||||
|
||||
func bucketPathForMountRoot(mountRoot, bucketRootPath string) (string, bool) {
|
||||
cleanPath := path.Clean("/" + strings.TrimPrefix(mountRoot, "/"))
|
||||
cleanBucketRoot := path.Clean("/" + strings.TrimPrefix(bucketRootPath, "/"))
|
||||
if cleanBucketRoot == "/" {
|
||||
return "", false
|
||||
}
|
||||
prefix := cleanBucketRoot + "/"
|
||||
if !strings.HasPrefix(cleanPath, prefix) {
|
||||
return "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(cleanPath, prefix)
|
||||
|
||||
bucketParts := strings.Split(rest, "/")
|
||||
if len(bucketParts) != 1 || bucketParts[0] == "" {
|
||||
return "", false
|
||||
}
|
||||
return cleanBucketRoot + "/" + bucketParts[0], true
|
||||
}
|
||||
|
||||
func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
|
||||
// basic checks
|
||||
@@ -32,10 +128,36 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
filerAddresses, grpcDialOption, cipher, bucketRootPath, ok := connectToFiler(option)
|
||||
if !ok {
|
||||
// try to connect to filer
|
||||
filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses()
|
||||
util.LoadSecurityConfiguration()
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
||||
var cipher bool
|
||||
var bucketRootPath string
|
||||
var err error
|
||||
for i := 0; i < 10; i++ {
|
||||
err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer grpc address %v configuration: %w", filerAddresses, err)
|
||||
}
|
||||
cipher = resp.Cipher
|
||||
bucketRootPath = resp.DirBuckets
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
|
||||
glog.V(0).Infof("wait for %d seconds ...", i+1)
|
||||
time.Sleep(time.Duration(i+1) * time.Second)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
|
||||
return true
|
||||
}
|
||||
if bucketRootPath == "" {
|
||||
bucketRootPath = "/buckets"
|
||||
}
|
||||
|
||||
filerMountRootPath := *option.filerMountRootPath
|
||||
|
||||
@@ -194,35 +316,88 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
fuseMountOptions.EnableSymlinkCaching = true
|
||||
}
|
||||
|
||||
mountRoot := resolveMountRoot(filerMountRootPath)
|
||||
cacheDirForRead, cacheDirForWrite := resolveCacheDirs(option)
|
||||
// find mount point
|
||||
mountRoot := filerMountRootPath
|
||||
if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
|
||||
mountRoot = mountRoot[0 : len(mountRoot)-1]
|
||||
}
|
||||
|
||||
seaweedFileSystem := buildSeaweedFileSystem(option, fileSystemParams{
|
||||
dir: dir,
|
||||
mountRoot: mountRoot,
|
||||
filerAddresses: filerAddresses,
|
||||
grpcDialOption: grpcDialOption,
|
||||
cipher: cipher,
|
||||
uidGidMapper: uidGidMapper,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
mountMode: mountMode,
|
||||
mountCtime: fileInfo.ModTime(),
|
||||
umask: umask,
|
||||
chunkSizeLimitMB: chunkSizeLimitMB,
|
||||
cacheDirForRead: cacheDirForRead,
|
||||
cacheDirForWrite: cacheDirForWrite,
|
||||
cacheDirForRead := util.ResolvePath(*option.cacheDirForRead)
|
||||
cacheDirForWrite := util.ResolvePath(*option.cacheDirForWrite)
|
||||
if cacheDirForWrite == "" {
|
||||
cacheDirForWrite = cacheDirForRead
|
||||
}
|
||||
|
||||
seaweedFileSystem := mount.NewSeaweedFileSystem(&mount.Option{
|
||||
MountDirectory: dir,
|
||||
FilerAddresses: filerAddresses,
|
||||
GrpcDialOption: grpcDialOption,
|
||||
FilerSigningKey: security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key")),
|
||||
FilerSigningExpiresAfterSec: util.GetViper().GetInt("jwt.filer_signing.expires_after_seconds"),
|
||||
FilerMountRootPath: mountRoot,
|
||||
Collection: *option.collection,
|
||||
Replication: *option.replication,
|
||||
TtlSec: int32(*option.ttlSec),
|
||||
DiskType: types.ToDiskType(*option.diskType),
|
||||
ChunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024,
|
||||
ConcurrentWriters: *option.concurrentWriters,
|
||||
ConcurrentReaders: *option.concurrentReaders,
|
||||
CacheDirForRead: cacheDirForRead,
|
||||
CacheSizeMBForRead: *option.cacheSizeMBForRead,
|
||||
CacheDirForWrite: cacheDirForWrite,
|
||||
WriteBufferSizeMB: *option.writeBufferSizeMB,
|
||||
CacheMetaTTlSec: *option.cacheMetaTtlSec,
|
||||
DataCenter: *option.dataCenter,
|
||||
Quota: int64(*option.collectionQuota) * 1024 * 1024,
|
||||
LogicalDiskUsage: *option.logicalDiskUsage,
|
||||
MountUid: uid,
|
||||
MountGid: gid,
|
||||
MountMode: mountMode,
|
||||
MountCtime: fileInfo.ModTime(),
|
||||
MountMtime: time.Now(),
|
||||
Umask: umask,
|
||||
VolumeServerAccess: *mountOptions.volumeServerAccess,
|
||||
Cipher: cipher,
|
||||
UidGidMapper: uidGidMapper,
|
||||
IncludeSystemEntries: *option.includeSystemEntries,
|
||||
DefaultPermissions: *option.defaultPermissions,
|
||||
DisableXAttr: *option.disableXAttr,
|
||||
IsMacOs: runtime.GOOS == "darwin",
|
||||
MetadataFlushSeconds: *option.metadataFlushSeconds,
|
||||
// RDMA acceleration options
|
||||
RdmaEnabled: *option.rdmaEnabled,
|
||||
RdmaSidecarAddr: *option.rdmaSidecarAddr,
|
||||
RdmaFallback: *option.rdmaFallback,
|
||||
RdmaReadOnly: *option.rdmaReadOnly,
|
||||
RdmaMaxConcurrent: *option.rdmaMaxConcurrent,
|
||||
RdmaTimeoutMs: *option.rdmaTimeoutMs,
|
||||
DirIdleEvictSec: *option.dirIdleEvictSec,
|
||||
EnableDistributedLock: option.distributedLock != nil && *option.distributedLock,
|
||||
WritebackCache: option.writebackCache != nil && *option.writebackCache,
|
||||
PosixDirNlink: option.posixDirNlink != nil && *option.posixDirNlink,
|
||||
// Peer chunk sharing
|
||||
PeerEnabled: option.peerEnabled != nil && *option.peerEnabled,
|
||||
PeerListen: peerStringOrEmpty(option.peerListen),
|
||||
PeerAdvertise: peerStringOrEmpty(option.peerAdvertise),
|
||||
PeerDataCenter: peerStringOrEmpty(option.peerDataCenter),
|
||||
PeerRack: peerStringOrEmpty(option.peerRack),
|
||||
})
|
||||
|
||||
if !createMountRoot(seaweedFileSystem, mountRoot, bucketRootPath, filerAddresses) {
|
||||
// create mount root
|
||||
mountRootPath := util.FullPath(mountRoot)
|
||||
mountRootParent, mountDir := mountRootPath.DirAndName()
|
||||
if err = filer_pb.Mkdir(context.Background(), seaweedFileSystem, mountRootParent, mountDir, nil); err != nil {
|
||||
fmt.Printf("failed to create dir %s on filer %s: %v\n", mountRoot, filerAddresses, err)
|
||||
return false
|
||||
}
|
||||
if err := ensureBucketAllowEmptyFolders(context.Background(), seaweedFileSystem, mountRoot, bucketRootPath); err != nil {
|
||||
fmt.Printf("failed to set bucket auto-remove-empty-folders policy for %s: %v\n", mountRoot, err)
|
||||
return false
|
||||
}
|
||||
|
||||
server, err := fuse.NewServer(seaweedFileSystem, dir, fuseMountOptions)
|
||||
if err != nil {
|
||||
// A failed mount is an environment problem (no /dev/fuse, fusermount not
|
||||
// setuid, stale mount point); the goroutine dump Fatalf adds buries it.
|
||||
glog.Exitf("Mount fail: %v", err)
|
||||
glog.Fatalf("Mount fail: %v", err)
|
||||
}
|
||||
grace.OnInterrupt(func() {
|
||||
if err := unmount.Unmount(dir); err != nil {
|
||||
@@ -239,7 +414,10 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
}
|
||||
}
|
||||
|
||||
serveMountGrpc(seaweedFileSystem, montSocketListener)
|
||||
grpcS := pb.NewGrpcServer()
|
||||
mount_pb.RegisterSeaweedMountServer(grpcS, seaweedFileSystem)
|
||||
reflection.Register(grpcS)
|
||||
go grpcS.Serve(montSocketListener)
|
||||
|
||||
err = seaweedFileSystem.StartBackgroundTasks()
|
||||
if err != nil {
|
||||
@@ -260,3 +438,10 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func peerStringOrEmpty(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mount/winfsp"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
||||
)
|
||||
|
||||
// ownedByMounter reports the mount root as belonging to whoever started it,
|
||||
// matching the uid=-1 option handed to WinFsp. It is a display value only:
|
||||
// WinFsp substitutes the calling user, and it must never be persisted, since
|
||||
// every other client would read the entry as owned by uid 4294967295.
|
||||
const ownedByMounter = ^uint32(0)
|
||||
|
||||
// windowsAttrTimeoutSec bounds how long WinFsp may serve cached attributes.
|
||||
// Nothing invalidates that cache from this side, so it stays short rather than
|
||||
// borrowing an unrelated flag's value.
|
||||
const windowsAttrTimeoutSec = 1.0
|
||||
|
||||
func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
chunkSizeLimitMB := *mountOptions.chunkSizeLimitMB
|
||||
if chunkSizeLimitMB <= 0 {
|
||||
fmt.Printf("Please specify a reasonable buffer size.\n")
|
||||
return false
|
||||
}
|
||||
|
||||
dir := *option.dir
|
||||
if dir == "" {
|
||||
fmt.Printf("Please specify the mount point via \"-dir\", for example -dir=S:\n")
|
||||
return false
|
||||
}
|
||||
// A drive letter or a not-yet-existing directory is what WinFsp wants, so
|
||||
// the mount point deliberately goes through none of the unix preparation:
|
||||
// no ResolvePath, no auto-create, no stat.
|
||||
if err := checkWindowsMountPoint(dir); err != nil {
|
||||
fmt.Printf("%v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
filerAddresses, grpcDialOption, cipher, bucketRootPath, ok := connectToFiler(option)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
|
||||
if *option.localSocket == "" {
|
||||
mountDirHash := util.HashToInt32([]byte(dir))
|
||||
if mountDirHash < 0 {
|
||||
mountDirHash = -mountDirHash
|
||||
}
|
||||
*option.localSocket = filepath.Join(os.TempDir(), fmt.Sprintf("seaweedfs-mount-%d.sock", mountDirHash))
|
||||
}
|
||||
if err := os.Remove(*option.localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", *option.localSocket, err.Error())
|
||||
}
|
||||
mountSocketListener, err := net.Listen("unix", *option.localSocket)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to listen on %s: %v", *option.localSocket, err)
|
||||
}
|
||||
|
||||
uidGidMapper, err := meta_cache.NewUidGidMapper(*option.uidMap, *option.gidMap)
|
||||
if err != nil {
|
||||
fmt.Printf("failed to parse %s %s: %v\n", *option.uidMap, *option.gidMap, err)
|
||||
return false
|
||||
}
|
||||
|
||||
mountRoot := resolveMountRoot(*option.filerMountRootPath)
|
||||
cacheDirForRead, cacheDirForWrite := resolveCacheDirs(option)
|
||||
|
||||
seaweedFileSystem := buildSeaweedFileSystem(option, fileSystemParams{
|
||||
dir: dir,
|
||||
mountRoot: mountRoot,
|
||||
filerAddresses: filerAddresses,
|
||||
grpcDialOption: grpcDialOption,
|
||||
cipher: cipher,
|
||||
uidGidMapper: uidGidMapper,
|
||||
uid: uint32(*option.windowsUid),
|
||||
gid: uint32(*option.windowsGid),
|
||||
mountMode: os.ModeDir | 0777,
|
||||
mountCtime: time.Now(),
|
||||
umask: umask,
|
||||
chunkSizeLimitMB: chunkSizeLimitMB,
|
||||
cacheDirForRead: cacheDirForRead,
|
||||
cacheDirForWrite: cacheDirForWrite,
|
||||
})
|
||||
|
||||
if !createMountRoot(seaweedFileSystem, mountRoot, bucketRootPath, filerAddresses) {
|
||||
return false
|
||||
}
|
||||
|
||||
host := winfsp.New(seaweedFileSystem, winfsp.Options{
|
||||
VolumeName: strings.ReplaceAll(*option.filer, ",", "+"),
|
||||
Uid: ownedByMounter,
|
||||
Gid: ownedByMounter,
|
||||
AttrTimeout: windowsAttrTimeoutSec,
|
||||
ReadOnly: *option.readOnly,
|
||||
Debug: *option.debugFuse,
|
||||
ExtraOptions: option.extraOptions,
|
||||
})
|
||||
|
||||
// Windows caches entries on its own side and the mount cannot invalidate
|
||||
// that cache directly, so changes made elsewhere have to be pushed out.
|
||||
host.Notify(seaweedFileSystem)
|
||||
|
||||
grace.OnInterrupt(func() {
|
||||
// The signal handler exits the process as soon as the hooks return, so
|
||||
// anything still queued has to be flushed here rather than after Serve.
|
||||
// WaitForAsyncFlush is idempotent, so the post-Serve call below is
|
||||
// harmless if both run.
|
||||
host.Unmount()
|
||||
seaweedFileSystem.WaitForAsyncFlush()
|
||||
})
|
||||
|
||||
serveMountGrpc(seaweedFileSystem, mountSocketListener)
|
||||
|
||||
if err := seaweedFileSystem.StartBackgroundTasks(); err != nil {
|
||||
fmt.Printf("failed to start background tasks: %v\n", err)
|
||||
return false
|
||||
}
|
||||
|
||||
glog.V(0).Infof("mounting %s%s to %v", *option.filer, mountRoot, dir)
|
||||
glog.V(0).Infof("This is SeaweedFS version %s %s %s", version.Version(), runtime.GOOS, runtime.GOARCH)
|
||||
glog.V(0).Infof("Windows mount is beta: hard links are unavailable and byte-range locks are not shared across mounts")
|
||||
|
||||
if err := host.Serve(windowsMountPoint(dir)); err != nil {
|
||||
glog.Errorf("%v", err)
|
||||
return false
|
||||
}
|
||||
|
||||
seaweedFileSystem.WaitForAsyncFlush()
|
||||
seaweedFileSystem.ClearCacheDir()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// checkWindowsMountPoint rejects the mount points WinFsp cannot take, which is
|
||||
// worth doing up front because its own failure is a bare false.
|
||||
func checkWindowsMountPoint(dir string) error {
|
||||
if isDriveLetter(dir) {
|
||||
if _, err := os.Stat(strings.TrimRight(dir, `\/`) + `\`); err == nil {
|
||||
return fmt.Errorf("drive %s is already in use", dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(dir, `\\`) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// WinFsp creates the directory itself, with FILE_CREATE, and deletes it
|
||||
// again when the filesystem goes away. An existing one — empty or not —
|
||||
// fails with "mount point in use".
|
||||
if _, err := os.Stat(dir); err == nil {
|
||||
return fmt.Errorf("mount point %s already exists; WinFsp creates the directory itself, so give it a path that does not exist yet", dir)
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Dir(dir)); err != nil {
|
||||
return fmt.Errorf("parent of mount point %s does not exist", dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// windowsMountPoint drops a trailing separator from a drive letter. WinFsp
|
||||
// recognises a drive only as exactly two characters; "S:\\" falls through to
|
||||
// its directory handling and the mount fails.
|
||||
func windowsMountPoint(dir string) string {
|
||||
if isDriveLetter(dir) {
|
||||
return strings.TrimRight(dir, `\/`)
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// isDriveLetter accepts both "S:" and "S:\\"; the trailing separator is how
|
||||
// the drive is usually written, and windowsMountPoint normalises it.
|
||||
func isDriveLetter(dir string) bool {
|
||||
trimmed := strings.TrimRight(dir, `\/`)
|
||||
if len(trimmed) != 2 || trimmed[1] != ':' {
|
||||
return false
|
||||
}
|
||||
c := trimmed[0]
|
||||
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
|
||||
}
|
||||
@@ -42,7 +42,6 @@ var (
|
||||
// When adding a new field, update all four flag registration sites.
|
||||
type S3Options struct {
|
||||
filer *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
port *int
|
||||
portHttps *int
|
||||
@@ -84,7 +83,6 @@ type S3Options struct {
|
||||
func init() {
|
||||
cmdS3.Run = runS3 // break init cycle
|
||||
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "comma-separated filer server addresses for high availability")
|
||||
s3StandaloneOptions.ip = cmdS3.Flag.String("ip", "", "ip address advertised to the cluster. If empty, default to -ip.bind, or the auto-detected address.")
|
||||
s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to 0.0.0.0.")
|
||||
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
|
||||
s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port")
|
||||
@@ -360,7 +358,6 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
EnableIam: *s3opt.enableIam, // Embedded IAM API (enabled by default)
|
||||
IamReadOnly: *s3opt.iamReadOnly,
|
||||
Cipher: *s3opt.cipher, // encrypt data on volume servers
|
||||
Ip: *s3opt.ip,
|
||||
BindIp: *s3opt.bindIp,
|
||||
GrpcPort: *s3opt.portGrpc,
|
||||
ExternalUrl: s3opt.resolveExternalUrl(),
|
||||
|
||||
@@ -108,7 +108,7 @@ func init() {
|
||||
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
masterOptions.telemetryUrl = cmdServer.Flag.String("master.telemetry.url", "https://telemetry.seaweedfs.com/api/collect", "telemetry server URL to send usage statistics")
|
||||
masterOptions.telemetryEnabled = cmdServer.Flag.Bool("master.telemetry", true, "report anonymous cluster statistics to master.telemetry.url, use -master.telemetry=false to opt out")
|
||||
masterOptions.telemetryEnabled = cmdServer.Flag.Bool("master.telemetry", false, "enable telemetry reporting")
|
||||
|
||||
filerOptions.filerGroup = cmdServer.Flag.String("filer.filerGroup", "", "share metadata with other filers in the same filerGroup")
|
||||
filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
|
||||
@@ -289,7 +289,6 @@ func runServer(cmd *Command, args []string) bool {
|
||||
filerOptions.masters = pb.ServerAddresses(actualPeersForComponents).ToServiceDiscovery()
|
||||
filerOptions.ip = serverIp
|
||||
filerOptions.bindIp = serverBindIp
|
||||
s3Options.ip = serverIp
|
||||
if *s3Options.bindIp == "" {
|
||||
s3Options.bindIp = serverBindIp
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package filer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -160,51 +159,6 @@ func (entry *Entry) IsExpireS3Enabled() (exist bool) {
|
||||
return exist
|
||||
}
|
||||
|
||||
func (entry *Entry) isS3Entry() bool {
|
||||
if entry.Extended == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if _, found := entry.Extended[s3_constants.ExtAmzOwnerKey]; found {
|
||||
return true
|
||||
}
|
||||
|
||||
if _, found := entry.Extended[s3_constants.ExtETagKey]; found {
|
||||
return true
|
||||
}
|
||||
|
||||
for key := range entry.Extended {
|
||||
if strings.HasPrefix(key, s3_constants.ExtAmzPrefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (entry *Entry) ApplyS3ExpiryMetadata() {
|
||||
if entry.TtlSec == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if _, found := entry.Extended[s3_constants.SeaweedFSExpiresS3]; found {
|
||||
return
|
||||
}
|
||||
|
||||
// The s3 expiry path skips versioned entries, so stamping one would drop its
|
||||
// expiry entirely instead of moving it onto mtime.
|
||||
if entry.IsS3Versioning() {
|
||||
return
|
||||
}
|
||||
|
||||
if !entry.isS3Entry() {
|
||||
return
|
||||
}
|
||||
|
||||
entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (entry *Entry) IsS3Versioning() (exist bool) {
|
||||
if entry.Extended != nil {
|
||||
_, exist = entry.Extended[s3_constants.ExtVersionIdKey]
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
func TestApplyS3ExpiryMetadata(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
ttlSec int32
|
||||
extended map[string][]byte
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "s3 entry with ttl",
|
||||
ttlSec: 3600,
|
||||
extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "no ttl",
|
||||
extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")},
|
||||
},
|
||||
{
|
||||
name: "not an s3 entry",
|
||||
ttlSec: 3600,
|
||||
},
|
||||
{
|
||||
name: "versioned entry keeps crtime expiry",
|
||||
ttlSec: 3600,
|
||||
extended: map[string][]byte{
|
||||
s3_constants.ExtETagKey: []byte("abc123"),
|
||||
s3_constants.ExtVersionIdKey: []byte("v1"),
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
entry := &Entry{Attr: Attr{TtlSec: tc.ttlSec}, Extended: tc.extended}
|
||||
entry.ApplyS3ExpiryMetadata()
|
||||
if got := entry.IsExpireS3Enabled(); got != tc.want {
|
||||
t.Fatalf("IsExpireS3Enabled() = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/viant/ptrie"
|
||||
jsonpb "google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -28,12 +27,6 @@ const (
|
||||
IamPoliciesFile = "policies.json"
|
||||
)
|
||||
|
||||
// FilerConfVersion is stamped into every configuration this build writes.
|
||||
// Version 0 predates worm presence: it was written with EmitUnpopulated, so every
|
||||
// rule carries an explicit "worm": false that meant nothing. Reading one back as an
|
||||
// override would silently lift worm off nested paths, so it is dropped to unset.
|
||||
const FilerConfVersion = 1
|
||||
|
||||
type FilerConf struct {
|
||||
rules ptrie.Trie[*filer_pb.FilerConf_PathConf]
|
||||
}
|
||||
@@ -122,9 +115,6 @@ func (fc *FilerConf) LoadFromBytes(data []byte) (err error) {
|
||||
|
||||
func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) {
|
||||
for _, location := range conf.Locations {
|
||||
if conf.Version < FilerConfVersion && location.Worm != nil && !*location.Worm {
|
||||
location.Worm = nil
|
||||
}
|
||||
err = fc.SetLocationConf(location)
|
||||
if err != nil {
|
||||
// this is not recoverable
|
||||
@@ -223,10 +213,6 @@ func ClonePathConf(src *filer_pb.FilerConf_PathConf) *filer_pb.FilerConf_PathCon
|
||||
if src == nil {
|
||||
return &filer_pb.FilerConf_PathConf{}
|
||||
}
|
||||
var worm *bool
|
||||
if src.Worm != nil {
|
||||
worm = proto.Bool(*src.Worm)
|
||||
}
|
||||
return &filer_pb.FilerConf_PathConf{
|
||||
LocationPrefix: src.LocationPrefix,
|
||||
Collection: src.Collection,
|
||||
@@ -241,7 +227,7 @@ func ClonePathConf(src *filer_pb.FilerConf_PathConf) *filer_pb.FilerConf_PathCon
|
||||
Rack: src.Rack,
|
||||
DataNode: src.DataNode,
|
||||
DisableChunkDeletion: src.DisableChunkDeletion,
|
||||
Worm: worm,
|
||||
Worm: src.Worm,
|
||||
WormGracePeriodSeconds: src.WormGracePeriodSeconds,
|
||||
WormRetentionTimeSeconds: src.WormRetentionTimeSeconds,
|
||||
}
|
||||
@@ -348,13 +334,7 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
|
||||
a.Rack = util.Nvl(b.Rack, a.Rack)
|
||||
a.DataNode = util.Nvl(b.DataNode, a.DataNode)
|
||||
a.DisableChunkDeletion = b.DisableChunkDeletion || a.DisableChunkDeletion
|
||||
// worm merges on presence, so a nested rule can turn it off. readOnly, fsync and
|
||||
// disableChunkDeletion stay OR'ed on purpose: a nested rule must not be able to
|
||||
// lift a lock the bucket set.
|
||||
if b.Worm != nil {
|
||||
// copy the value: a is often a scratch conf while b is a live trie entry
|
||||
a.Worm = proto.Bool(*b.Worm)
|
||||
}
|
||||
a.Worm = b.Worm || a.Worm
|
||||
if b.WormRetentionTimeSeconds > 0 {
|
||||
a.WormRetentionTimeSeconds = b.WormRetentionTimeSeconds
|
||||
}
|
||||
@@ -364,7 +344,7 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
|
||||
}
|
||||
|
||||
func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
|
||||
m := &filer_pb.FilerConf{Version: FilerConfVersion}
|
||||
m := &filer_pb.FilerConf{}
|
||||
fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
|
||||
m.Locations = append(m.Locations, value)
|
||||
return true
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestFilerConf(t *testing.T) {
|
||||
@@ -51,68 +49,6 @@ func TestFilerConf(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestWormInheritance(t *testing.T) {
|
||||
fc := NewFilerConf()
|
||||
fc.doLoadConf(&filer_pb.FilerConf{
|
||||
Version: FilerConfVersion,
|
||||
Locations: []*filer_pb.FilerConf_PathConf{
|
||||
{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true), Ttl: "7d"},
|
||||
{LocationPrefix: "/buckets/b/quiet/", Collection: "quiet"},
|
||||
{LocationPrefix: "/buckets/b/scratch/", Worm: proto.Bool(false)},
|
||||
{LocationPrefix: "/buckets/b/scratch/keep/", Worm: proto.Bool(true)},
|
||||
},
|
||||
})
|
||||
|
||||
// a rule that says nothing about worm keeps inheriting it, along with the ttl
|
||||
rule := fc.MatchStorageRule("/buckets/b/quiet/x")
|
||||
assert.True(t, rule.GetWorm())
|
||||
assert.Equal(t, "7d", rule.Ttl)
|
||||
|
||||
// an explicit false turns it off, and a deeper rule turns it back on
|
||||
assert.False(t, fc.MatchStorageRule("/buckets/b/scratch/x").GetWorm())
|
||||
assert.True(t, fc.MatchStorageRule("/buckets/b/scratch/keep/x").GetWorm())
|
||||
|
||||
// paths with no rule at all are unaffected
|
||||
assert.False(t, fc.MatchStorageRule("/buckets/other/x").GetWorm())
|
||||
}
|
||||
|
||||
// TestWormLegacyFalseIsNotAnOverride pins that the explicit "worm": false every
|
||||
// version 0 configuration carries does not read as a per-path opt-out.
|
||||
func TestWormLegacyFalseIsNotAnOverride(t *testing.T) {
|
||||
const conf = `{
|
||||
"locations": [
|
||||
{"locationPrefix": "/buckets/b/", "worm": true},
|
||||
{"locationPrefix": "/buckets/b/sub/", "collection": "sub", "worm": false}
|
||||
]
|
||||
}`
|
||||
|
||||
fc := NewFilerConf()
|
||||
assert.NoError(t, fc.LoadFromBytes([]byte(conf)))
|
||||
assert.True(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
||||
|
||||
// the same file at the current version means what it says
|
||||
fc = NewFilerConf()
|
||||
assert.NoError(t, fc.LoadFromBytes([]byte(`{"version": 1,`+conf[1:])))
|
||||
assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
||||
}
|
||||
|
||||
// TestWormSurvivesRoundTrip guards the write side: an explicit false has to be
|
||||
// stamped along with a version that says to honor it.
|
||||
func TestWormSurvivesRoundTrip(t *testing.T) {
|
||||
fc := NewFilerConf()
|
||||
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true)})
|
||||
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/sub/", Worm: proto.Bool(false)})
|
||||
fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/other/", Ttl: "7d"})
|
||||
|
||||
var buf bytes.Buffer
|
||||
assert.NoError(t, fc.ToText(&buf))
|
||||
|
||||
reloaded := NewFilerConf()
|
||||
assert.NoError(t, reloaded.LoadFromBytes(buf.Bytes()))
|
||||
assert.False(t, reloaded.MatchStorageRule("/buckets/b/sub/x").GetWorm())
|
||||
assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetWorm())
|
||||
}
|
||||
|
||||
// TestClonePathConf verifies that ClonePathConf copies all exported fields.
|
||||
// Uses reflection to automatically detect new fields added to the protobuf,
|
||||
// ensuring the test fails if ClonePathConf is not updated for new fields.
|
||||
@@ -132,7 +68,7 @@ func TestClonePathConf(t *testing.T) {
|
||||
Rack: "rack1",
|
||||
DataNode: "node1",
|
||||
DisableChunkDeletion: true,
|
||||
Worm: proto.Bool(true),
|
||||
Worm: true,
|
||||
WormGracePeriodSeconds: 3600,
|
||||
WormRetentionTimeSeconds: 86400,
|
||||
}
|
||||
@@ -183,10 +119,8 @@ func TestClonePathConf(t *testing.T) {
|
||||
// Verify mutation of clone doesn't affect source
|
||||
clone.Collection = "modified"
|
||||
clone.ReadOnly = false
|
||||
*clone.Worm = false
|
||||
assert.Equal(t, "test_collection", src.Collection, "Modifying clone should not affect source Collection")
|
||||
assert.Equal(t, true, src.ReadOnly, "Modifying clone should not affect source ReadOnly")
|
||||
assert.Equal(t, true, src.GetWorm(), "Modifying clone should not affect source Worm")
|
||||
}
|
||||
|
||||
func TestClonePathConfNil(t *testing.T) {
|
||||
|
||||
+10
-149
@@ -23,138 +23,13 @@ type LogFileEntry struct {
|
||||
FileEntry *Entry
|
||||
}
|
||||
|
||||
// logFileMayContainAfter reports whether the log file named for fileTsNs can
|
||||
// hold any entry past startTsNs. The following file's name is not the bound: a
|
||||
// file is named for the start of the window it holds, minute-truncated, and the
|
||||
// window runs up to a flush interval longer, so "12-30" can hold 12:31:20 while
|
||||
// "12-31" exists alongside it. Comparing against the next name dropped exactly
|
||||
// the spanning file a mid-window cursor needs.
|
||||
func logFileMayContainAfter(fileTsNs, startTsNs int64) bool {
|
||||
return fileTsNs+int64(time.Minute)+int64(LogFlushInterval) > startTsNs
|
||||
}
|
||||
|
||||
// persistedLogScanStart backs a read position off by one flush interval before
|
||||
// choosing which log files to open. A log file is named for the start of the
|
||||
// window it holds and a window spans up to flushInterval, so a file whose name
|
||||
// sorts before the cursor's own minute can still hold entries after it -- a
|
||||
// window sealed at 12:30:59 and ending 12:31:58 lives in "12-30". Entries are
|
||||
// filtered against the exact cursor afterwards, so widening the file scan only
|
||||
// costs a little extra reading and never re-delivers.
|
||||
func persistedLogScanStart(t time.Time) time.Time {
|
||||
return t.Add(-LogFlushInterval)
|
||||
}
|
||||
|
||||
// LastShippedLogEntryTsNsForFiler mirrors the client's reader across one
|
||||
// filer's shipped files, in ship order: a file that fails mid-read still
|
||||
// delivers its readable prefix and later files still deliver after it, so the
|
||||
// newest file with readable content answers. answeredFileTsNs names that file
|
||||
// so the caller can roll back the sent state of everything newer - refs the
|
||||
// cursor did not reach must re-ship, or a transient probe failure leaves the
|
||||
// cursor behind them for the life of the connection.
|
||||
// complete reports whether the answering file was read through to its end: a
|
||||
// prefix-limited answer means the ref's unread suffix must re-ship too, or a
|
||||
// transient mid-file failure abandons it for the life of the connection.
|
||||
func (f *Filer) LastShippedLogEntryTsNsForFiler(refs []*filer_pb.LogFileChunkRef) (tsNs int64, answeredFileTsNs int64, ok bool, complete bool) {
|
||||
for i := len(refs) - 1; i >= 0; i-- {
|
||||
if tsNs, ok, complete = f.lastShippedLogEntryTsNs(refs[i].Chunks); ok {
|
||||
return tsNs, refs[i].FileTsNs, ok, complete
|
||||
}
|
||||
}
|
||||
return 0, 0, false, false
|
||||
}
|
||||
|
||||
// lookupLogChunkFn reports whether a chunk still resolves to a live volume.
|
||||
// Swapped in tests.
|
||||
var lookupLogChunkFn = func(f *Filer, fileId string) error {
|
||||
_, err := f.MasterClient.GetLookupFileIdFunction()(context.Background(), fileId)
|
||||
return err
|
||||
}
|
||||
|
||||
// lastShippedLogEntryTsNs mirrors the client's read of one shipped file: a
|
||||
// sequential scan of exactly these chunks that ends at the first unreadable
|
||||
// one, so a marker built from it never claims content the client will not
|
||||
// apply. Unreadable includes transient failures - understating the marker
|
||||
// only re-ships, overstating loses events - and no error escapes: a probe
|
||||
// failure must never block the transition the client is waiting on.
|
||||
//
|
||||
// Readability is judged where the client reads - the volumes - not by the
|
||||
// decoded-chunk cache: a chunk this server decoded an hour ago may sit on a
|
||||
// volume that has since died, and the direct-reading client stops there no
|
||||
// matter how well the cache still remembers the bytes.
|
||||
func (f *Filer) lastShippedLogEntryTsNs(chunks []*filer_pb.FileChunk) (tsNs int64, ok bool, complete bool) {
|
||||
for _, chunk := range chunks {
|
||||
if lookupErr := lookupLogChunkFn(f, chunk.GetFileIdString()); lookupErr != nil {
|
||||
return tsNs, ok, false
|
||||
}
|
||||
entries, loadErr := f.persistedLogCache.getOrLoad(chunk.GetFileIdString(), int64(chunk.Size), func() ([]*filer_pb.LogEntry, bool, error) {
|
||||
return loadLogFileEntriesFn(f.MasterClient, chunk)
|
||||
})
|
||||
if errors.Is(loadErr, errLogChunkIncomplete) {
|
||||
// Records span chunks; the client streams such a file whole.
|
||||
if streamTsNs, streamOk, streamComplete := f.lastStreamedLogEntryTsNs(chunks); streamOk {
|
||||
return streamTsNs, true, streamComplete
|
||||
}
|
||||
return tsNs, ok, false
|
||||
}
|
||||
if loadErr != nil {
|
||||
return tsNs, ok, false // prefix ends here, like the client's reader
|
||||
}
|
||||
if len(entries) > 0 {
|
||||
tsNs, ok = entries[len(entries)-1].TsNs, true
|
||||
}
|
||||
}
|
||||
return tsNs, ok, true
|
||||
}
|
||||
|
||||
// lastStreamedLogEntryTsNs scans the chunk list as one byte stream, ending
|
||||
// where the client's reader ends: a clean or torn tail, a missing chunk, or
|
||||
// undecodable bytes all terminate the scan with the progress made.
|
||||
func (f *Filer) lastStreamedLogEntryTsNs(chunks []*filer_pb.FileChunk) (tsNs int64, ok bool, complete bool) {
|
||||
r := newLogFileStreamReader(f.MasterClient, chunks)
|
||||
if closer, isCloser := r.(io.Closer); isCloser {
|
||||
defer closer.Close()
|
||||
}
|
||||
sizeBuf := make([]byte, 4)
|
||||
for {
|
||||
if _, readErr := io.ReadFull(r, sizeBuf); readErr != nil {
|
||||
// A clean or torn tail is where the client's read ends too; only a
|
||||
// mid-stream failure leaves an unread remainder worth re-shipping.
|
||||
ended := readErr == io.EOF || readErr == io.ErrUnexpectedEOF
|
||||
return tsNs, ok, ended
|
||||
}
|
||||
size := util.BytesToUint32(sizeBuf)
|
||||
if size > maxLogEntrySize {
|
||||
return tsNs, ok, false
|
||||
}
|
||||
data := make([]byte, size)
|
||||
if _, readErr := io.ReadFull(r, data); readErr != nil {
|
||||
return tsNs, ok, false
|
||||
}
|
||||
logEntry := &filer_pb.LogEntry{}
|
||||
if unmarshalErr := logEntry.UnmarshalVT(data); unmarshalErr != nil {
|
||||
return tsNs, ok, false
|
||||
}
|
||||
tsNs, ok = logEntry.TsNs, true
|
||||
}
|
||||
}
|
||||
|
||||
// PersistedLogScanStartTsNs is the oldest file-name timestamp a scan from t
|
||||
// can still list. File names are minute-truncated, and the collector compares
|
||||
// names at minute granularity, so the bound must truncate too: an exact-ns
|
||||
// bound sits inside the boundary file's minute and disowns a file the next
|
||||
// collection will still return.
|
||||
func PersistedLogScanStartTsNs(t time.Time) int64 {
|
||||
return persistedLogScanStart(t).Truncate(time.Minute).UnixNano()
|
||||
}
|
||||
|
||||
func (f *Filer) collectPersistedLogBuffer(startPosition log_buffer.MessagePosition, stopTsNs int64) (v *OrderedLogVisitor, err error) {
|
||||
|
||||
if stopTsNs != 0 && startPosition.Time.UnixNano() > stopTsNs {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
scanFrom := persistedLogScanStart(startPosition.Time)
|
||||
startDate := fmt.Sprintf("%04d-%02d-%02d", scanFrom.Year(), scanFrom.Month(), scanFrom.Day())
|
||||
startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Time.Year(), startPosition.Time.Month(), startPosition.Time.Day())
|
||||
|
||||
dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, math.MaxInt32, "", "", "")
|
||||
if listDayErr != nil {
|
||||
@@ -173,9 +48,8 @@ func (f *Filer) CollectLogFileRefs(ctx context.Context, startPosition log_buffer
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
scanFrom := persistedLogScanStart(startPosition.Time)
|
||||
startDate := fmt.Sprintf("%04d-%02d-%02d", scanFrom.Year(), scanFrom.Month(), scanFrom.Day())
|
||||
startHourMinute := fmt.Sprintf("%02d-%02d", scanFrom.Hour(), scanFrom.Minute())
|
||||
startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Time.Year(), startPosition.Time.Month(), startPosition.Time.Day())
|
||||
startHourMinute := fmt.Sprintf("%02d-%02d", startPosition.Time.Hour(), startPosition.Time.Minute())
|
||||
var stopDate, stopHourMinute string
|
||||
if stopTsNs != 0 {
|
||||
stopTime := time.Unix(0, stopTsNs).UTC()
|
||||
@@ -231,24 +105,9 @@ func (f *Filer) CollectLogFileRefs(ctx context.Context, startPosition log_buffer
|
||||
lastTsNs = t.UnixNano()
|
||||
}
|
||||
}
|
||||
lastTsNs = clampLogRefsCursor(lastTsNs, startPosition.Time.UnixNano())
|
||||
return
|
||||
}
|
||||
|
||||
// clampLogRefsCursor keeps a chunk-ref read from moving the subscriber's cursor
|
||||
// backwards. Refs are named for the minute their window starts in, so the last
|
||||
// one routinely sorts before the position that was asked for -- always, now
|
||||
// that the scan reaches back a flush interval to pick up a spanning file. The
|
||||
// caller makes this value the new read position, and rewinding it would replay
|
||||
// memory from before the client's own SinceNs and re-send what the chunk reader
|
||||
// has already been handed.
|
||||
func clampLogRefsCursor(lastTsNs, startTsNs int64) int64 {
|
||||
if lastTsNs < startTsNs {
|
||||
return startTsNs
|
||||
}
|
||||
return lastTsNs
|
||||
}
|
||||
|
||||
func (f *Filer) HasPersistedLogFiles(startPosition log_buffer.MessagePosition) (bool, error) {
|
||||
startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Time.Year(), startPosition.Time.Month(), startPosition.Time.Day())
|
||||
dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 1, "", "", "")
|
||||
@@ -375,9 +234,8 @@ func NewLogFileEntryCollector(f *Filer, startPosition log_buffer.MessagePosition
|
||||
// println("enqueue day entry", dayEntry.Name())
|
||||
}
|
||||
|
||||
scanFrom := persistedLogScanStart(startPosition.Time)
|
||||
startDate := fmt.Sprintf("%04d-%02d-%02d", scanFrom.Year(), scanFrom.Month(), scanFrom.Day())
|
||||
startHourMinute := fmt.Sprintf("%02d-%02d", scanFrom.Hour(), scanFrom.Minute())
|
||||
startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Time.Year(), startPosition.Time.Month(), startPosition.Time.Day())
|
||||
startHourMinute := fmt.Sprintf("%02d-%02d", startPosition.Time.Hour(), startPosition.Time.Minute())
|
||||
var stopDate, stopHourMinute string
|
||||
if stopTsNs != 0 {
|
||||
stopTime := time.Unix(0, stopTsNs+24*60*60*int64(time.Second)).UTC()
|
||||
@@ -552,12 +410,15 @@ func (iter *LogFileQueueIterator) getNext(v *OrderedLogVisitor) (logEntry *filer
|
||||
if iter.stopTsNs != 0 && t.TsNs > iter.stopTsNs {
|
||||
return nil, io.EOF
|
||||
}
|
||||
if iter.q.Peek() == nil {
|
||||
next := iter.q.Peek()
|
||||
if next == nil {
|
||||
if collectErr := v.logFileEntryCollector.collectMore(v); collectErr != nil && collectErr != io.EOF {
|
||||
return nil, collectErr
|
||||
}
|
||||
next = iter.q.Peek() // Re-peek after collectMore
|
||||
}
|
||||
if !logFileMayContainAfter(t.TsNs, iter.startTsNs) {
|
||||
// skip the file if the next entry is before the startTsNs
|
||||
if next != nil && next.TsNs <= iter.startTsNs {
|
||||
continue
|
||||
}
|
||||
iter.currentFileIterator = newLogFileIterator(iter.masterClient, iter.cache, t.FileEntry, iter.startTsNs, iter.stopTsNs)
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
// TestPersistedLogScanStartCoversSpanningWindow pins the file-selection window.
|
||||
// A log file is named for the start of the window it holds, and a window spans
|
||||
// up to LogFlushInterval, so a cursor inside the window's later minutes must
|
||||
// still open the file named for its earlier one -- otherwise the read reports
|
||||
// "nothing on disk" for entries that are sitting in it, and the gap resolvers
|
||||
// take that miss as proof the range is empty.
|
||||
func TestPersistedLogScanStartCoversSpanningWindow(t *testing.T) {
|
||||
// A window sealed at 12:30:59 ending 12:31:58 is written to "12-30".
|
||||
sealed := time.Date(2026, 6, 29, 12, 30, 59, 0, time.UTC)
|
||||
fileMinute := sealed.Format("15-04")
|
||||
|
||||
// A subscriber resuming mid-window must not sort past that file.
|
||||
for _, cursor := range []time.Time{
|
||||
sealed.Add(11 * time.Second), // 12:31:10, the reported case
|
||||
sealed.Add(59 * time.Second), // 12:31:58, the window's last entry
|
||||
sealed, // exactly the window start
|
||||
} {
|
||||
scanMinute := persistedLogScanStart(cursor).Format("15-04")
|
||||
if scanMinute > fileMinute {
|
||||
t.Fatalf("cursor %v scans from %q, which sorts past the file %q holding it",
|
||||
cursor.Format("15:04:05"), scanMinute, fileMinute)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A cursor just after midnight has to reach back into the previous day.
|
||||
func TestPersistedLogScanStartCrossesMidnight(t *testing.T) {
|
||||
cursor := time.Date(2026, 6, 29, 0, 0, 30, 0, time.UTC)
|
||||
scanFrom := persistedLogScanStart(cursor)
|
||||
if got, want := scanFrom.Format("2006-01-02"), "2026-06-28"; got != want {
|
||||
t.Fatalf("scan date = %q, want %q so the previous day's last file is listed", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSpanningLogFileIsNotSkipped pins the other half of the minute-boundary
|
||||
// fix, against the predicate the iterator actually calls. Widening which files
|
||||
// get listed is useless if the iterator then drops the spanning file, which it
|
||||
// did by treating the following file's name as an upper bound on this file's
|
||||
// contents -- it is not.
|
||||
func TestSpanningLogFileIsNotSkipped(t *testing.T) {
|
||||
// Window sealed at 12:30:59, ending 12:31:58, written to "12-30".
|
||||
spanning := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC).UnixNano()
|
||||
// The next window starts at 12:31:59 and is written to "12-31".
|
||||
following := time.Date(2026, 6, 29, 12, 31, 0, 0, time.UTC).UnixNano()
|
||||
|
||||
cursor := time.Date(2026, 6, 29, 12, 31, 10, 0, time.UTC).UnixNano()
|
||||
if following > cursor {
|
||||
t.Fatal("precondition: the following file's name sorts at or before the cursor")
|
||||
}
|
||||
if !logFileMayContainAfter(spanning, cursor) {
|
||||
t.Fatal("the spanning file holds entries past the cursor and must be read")
|
||||
}
|
||||
|
||||
// A file that genuinely cannot reach the cursor is still skipped, so the
|
||||
// widening does not turn into reading the whole day.
|
||||
old := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC).UnixNano()
|
||||
if logFileMayContainAfter(old, cursor) {
|
||||
t.Fatal("a file a full interval behind the cursor should still be skipped")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClampLogRefsCursorNeverRewinds pins that a chunk-ref read cannot move the
|
||||
// subscriber backwards. The refs carry minute-level names, so the last one
|
||||
// normally sorts before the requested position, and the caller assigns that
|
||||
// value straight to the read cursor.
|
||||
func TestClampLogRefsCursorNeverRewinds(t *testing.T) {
|
||||
cursor := time.Date(2026, 6, 29, 12, 31, 10, 0, time.UTC).UnixNano()
|
||||
spanningFile := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC).UnixNano()
|
||||
|
||||
if got := clampLogRefsCursor(spanningFile, cursor); got != cursor {
|
||||
t.Fatalf("cursor moved to %v, want it held at %v", time.Unix(0, got), time.Unix(0, cursor))
|
||||
}
|
||||
// A ref genuinely ahead of the cursor still advances it.
|
||||
ahead := time.Date(2026, 6, 29, 12, 32, 0, 0, time.UTC).UnixNano()
|
||||
if got := clampLogRefsCursor(ahead, cursor); got != ahead {
|
||||
t.Fatalf("cursor = %v, want it to advance to %v", time.Unix(0, got), time.Unix(0, ahead))
|
||||
}
|
||||
}
|
||||
|
||||
// TestPersistedLogScanStartTsNsMinuteAligned pins the prune bound against the
|
||||
// collector's minute-granular file comparison: a cursor at 12:31:20 still
|
||||
// collects the 12-30 file, so the bound must not sit past 12:30:00 - an
|
||||
// exact-ns bound disowns the file's sent state and the next pass reships it
|
||||
// whole, re-creating the duplicate-refs class the state exists to prevent.
|
||||
func TestPersistedLogScanStartTsNsMinuteAligned(t *testing.T) {
|
||||
cursor := time.Date(2026, 6, 29, 12, 31, 20, 0, time.UTC)
|
||||
fileTsNs := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC).UnixNano()
|
||||
|
||||
bound := PersistedLogScanStartTsNs(cursor)
|
||||
if fileTsNs < bound {
|
||||
t.Fatalf("bound %v disowns the 12-30 file the collector still lists", time.Unix(0, bound).UTC())
|
||||
}
|
||||
// A file a full interval plus a minute behind is genuinely out of scan range.
|
||||
old := time.Date(2026, 6, 29, 12, 28, 0, 0, time.UTC).UnixNano()
|
||||
if old >= bound {
|
||||
t.Fatalf("bound %v keeps state for files the scan can no longer list", time.Unix(0, bound).UTC())
|
||||
}
|
||||
}
|
||||
|
||||
// TestLastShippedLogEntryTsNs pins the chunk-mode cursor source against the
|
||||
// client's reader, which it must mirror: per file the readable prefix - the
|
||||
// client stops at the first unreadable chunk and never resumes within the file
|
||||
// - and per filer the newest file with content, because the client keeps
|
||||
// reading later files after skipping a bad one. A marker past what the client
|
||||
// applies loses events; one short only re-ships.
|
||||
func TestLastShippedLogEntryTsNs(t *testing.T) {
|
||||
f := &Filer{persistedLogCache: newPersistedLogCache(1 << 20)}
|
||||
|
||||
entry := func(ts int64) *filer_pb.LogEntry { return &filer_pb.LogEntry{TsNs: ts} }
|
||||
chunk := func(id string) *filer_pb.FileChunk { return &filer_pb.FileChunk{FileId: id, Size: 10} }
|
||||
ref := func(fileTsNs int64, chunks ...*filer_pb.FileChunk) *filer_pb.LogFileChunkRef {
|
||||
return &filer_pb.LogFileChunkRef{FilerId: "a", FileTsNs: fileTsNs, Chunks: chunks}
|
||||
}
|
||||
|
||||
byId := map[string]struct {
|
||||
entries []*filer_pb.LogEntry
|
||||
err error
|
||||
}{
|
||||
"good-early": {entries: []*filer_pb.LogEntry{entry(100), entry(200)}},
|
||||
"good-late": {entries: []*filer_pb.LogEntry{entry(300), entry(400)}},
|
||||
"missing": {err: fmt.Errorf("volume 7 not found")},
|
||||
"missing2": {err: fmt.Errorf("volume 8 not found")},
|
||||
"empty": {},
|
||||
}
|
||||
origLoad := loadLogFileEntriesFn
|
||||
loadLogFileEntriesFn = func(masterClient *wdclient.MasterClient, c *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
|
||||
r := byId[c.FileId]
|
||||
return r.entries, true, r.err
|
||||
}
|
||||
origLookup := lookupLogChunkFn
|
||||
deadVolumes := map[string]bool{}
|
||||
lookupLogChunkFn = func(f *Filer, fileId string) error {
|
||||
if deadVolumes[fileId] {
|
||||
return fmt.Errorf("volume 9 not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() { loadLogFileEntriesFn = origLoad; lookupLogChunkFn = origLookup }()
|
||||
|
||||
t.Run("a fully readable file answers with its tail, complete", func(t *testing.T) {
|
||||
ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")})
|
||||
if !ok || ts != 400 || !complete {
|
||||
t.Fatalf("ts=%d ok=%v complete=%v, want 400 and complete", ts, ok, complete)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the prefix ends at the first missing chunk", func(t *testing.T) {
|
||||
// The client's reader stops there and never resumes within the file:
|
||||
// answering from the readable suffix would put the marker past entries
|
||||
// the client never applied, losing them permanently.
|
||||
ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("missing"), chunk("good-late")})
|
||||
if !ok || ts != 200 {
|
||||
t.Fatalf("ts=%d ok=%v, want the prefix end 200, never the suffix 400", ts, ok)
|
||||
}
|
||||
if complete {
|
||||
t.Fatal("a prefix-limited read must report incomplete, or the unread suffix is never re-shipped")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nothing readable is a clean no-answer", func(t *testing.T) {
|
||||
if _, ok, _ := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("missing"), chunk("empty")}); ok {
|
||||
t.Fatal("want no answer")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an entirely missing last file falls back to earlier files", func(t *testing.T) {
|
||||
// The client skips the bad file but has applied the earlier ones;
|
||||
// discarding their progress would rewind the marker to the start
|
||||
// cursor and stall or replay.
|
||||
ts, answeredFileTsNs, ok, _ := f.LastShippedLogEntryTsNsForFiler([]*filer_pb.LogFileChunkRef{
|
||||
ref(1000, chunk("good-late")),
|
||||
ref(2000, chunk("missing"), chunk("missing2")),
|
||||
})
|
||||
if !ok || ts != 400 {
|
||||
t.Fatalf("ts=%d ok=%v, want the previous file's 400", ts, ok)
|
||||
}
|
||||
if answeredFileTsNs != 1000 {
|
||||
t.Fatalf("answered file %d, want 1000: the caller rolls back sent state above it", answeredFileTsNs)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a cached chunk on a dead volume ends the prefix", func(t *testing.T) {
|
||||
// Warm the cache, then kill the middle chunk's volume. The client
|
||||
// reads directly and stops there; a probe trusting its own decoded
|
||||
// cache would sail past to the tail and put the marker beyond entries
|
||||
// the client never applied.
|
||||
if ts, ok, _ := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")}); !ok || ts != 400 {
|
||||
t.Fatalf("warmup: ts=%d ok=%v, want 400", ts, ok)
|
||||
}
|
||||
deadVolumes["good-early"] = true
|
||||
defer delete(deadVolumes, "good-early")
|
||||
ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")})
|
||||
if ok || ts != 0 {
|
||||
t.Fatalf("ts=%d ok=%v, want no answer: the first chunk's volume is gone however warm the cache", ts, ok)
|
||||
}
|
||||
if complete {
|
||||
t.Fatal("a lookup-limited read must report incomplete so the ref re-ships")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a dead needle behind a warm cache is the accepted residual", func(t *testing.T) {
|
||||
// Production liveness is a volume lookup: it cannot see a needle 404
|
||||
// or a stale location inside a resolvable volume, so a warm cache can
|
||||
// answer past a chunk the client fails to read. Accepted because
|
||||
// metadata log chunks die volume-at-a-time (TTL'd log volumes) and the
|
||||
// alternative is a real read per probe, which is what the probe exists
|
||||
// to avoid; the client logs the skip at V(0). This test pins the
|
||||
// boundary so a change to it is a decision, not an accident.
|
||||
if ts, ok, _ := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")}); !ok || ts != 400 {
|
||||
t.Fatalf("warmup: ts=%d ok=%v, want 400", ts, ok)
|
||||
}
|
||||
// The needle dies: loads fail, but the volume still resolves.
|
||||
prev := byId["good-early"]
|
||||
byId["good-early"] = struct {
|
||||
entries []*filer_pb.LogEntry
|
||||
err error
|
||||
}{err: fmt.Errorf("read: 404 Not Found: not found")}
|
||||
defer func() { byId["good-early"] = prev }()
|
||||
ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")})
|
||||
if !ok || ts != 400 || !complete {
|
||||
t.Fatalf("ts=%d ok=%v complete=%v: the warm cache masks a dead needle - if this now fails, the residual was closed and this test should assert the new behavior", ts, ok, complete)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("spanning records stream the shipped list", func(t *testing.T) {
|
||||
origLoad2 := loadLogFileEntriesFn
|
||||
loadLogFileEntriesFn = func(masterClient *wdclient.MasterClient, c *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
|
||||
return nil, false, errLogChunkIncomplete
|
||||
}
|
||||
origStream := newLogFileStreamReader
|
||||
newLogFileStreamReader = func(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) io.Reader {
|
||||
var buf bytes.Buffer
|
||||
for _, ts := range []int64{500, 600} {
|
||||
data, _ := (&filer_pb.LogEntry{TsNs: ts}).MarshalVT()
|
||||
var sizeBuf [4]byte
|
||||
util.Uint32toBytes(sizeBuf[:], uint32(len(data)))
|
||||
buf.Write(sizeBuf[:])
|
||||
buf.Write(data)
|
||||
}
|
||||
// A torn trailing size prefix, as a crashed writer leaves it: the
|
||||
// client reads it as a clean end, so the probe must too - failing
|
||||
// here would block the marker forever on data the client accepts.
|
||||
buf.Write([]byte{0x01, 0x02})
|
||||
return &buf
|
||||
}
|
||||
defer func() { loadLogFileEntriesFn = origLoad2; newLogFileStreamReader = origStream }()
|
||||
|
||||
ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("incomplete")})
|
||||
if !ok || ts != 600 {
|
||||
t.Fatalf("ts=%d ok=%v, want the streamed tail 600 past the torn prefix", ts, ok)
|
||||
}
|
||||
if !complete {
|
||||
t.Fatal("a torn tail is where the client's read ends too: complete, nothing to re-ship")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
)
|
||||
|
||||
// SetLogReadHooksForTesting swaps the volume-touching pieces of persisted log
|
||||
// reading - chunk decode, byte streaming, volume liveness - for fakes, so loop
|
||||
// tests can run the real subscribe machinery against an in-memory volume
|
||||
// layer. Returns a restore func. Test support only.
|
||||
func SetLogReadHooksForTesting(
|
||||
load func(chunk *filer_pb.FileChunk) ([]*filer_pb.LogEntry, error),
|
||||
stream func(chunks []*filer_pb.FileChunk) io.Reader,
|
||||
lookup func(fileId string) error,
|
||||
) (restore func()) {
|
||||
prevLoad, prevStream, prevLookup := loadLogFileEntriesFn, newLogFileStreamReader, lookupLogChunkFn
|
||||
loadLogFileEntriesFn = func(masterClient *wdclient.MasterClient, chunk *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
|
||||
entries, err := load(chunk)
|
||||
return entries, true, err
|
||||
}
|
||||
newLogFileStreamReader = func(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) io.Reader {
|
||||
return stream(chunks)
|
||||
}
|
||||
lookupLogChunkFn = func(f *Filer, fileId string) error {
|
||||
return lookup(fileId)
|
||||
}
|
||||
return func() {
|
||||
loadLogFileEntriesFn, newLogFileStreamReader, lookupLogChunkFn = prevLoad, prevStream, prevLookup
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user