mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-06 14:45:51 +00:00
Compare commits
6
Commits
4.24
..
iam-phase1
| Author | SHA1 | Date | |
|---|---|---|---|
 Chris Lu
|
18ae84e1ec | ||
|
Chris Lu
|
641bea825d | ||
|
Chris Lu
|
91fe0a5162 | ||
|
Chris Lu
|
64a60607c6 | ||
|
Chris Lu
|
d4365e2f37 | ||
|
Chris Lu
|
f9dfc0ea37 |
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294
|
||||
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
|
||||
|
||||
@@ -57,12 +57,12 @@ jobs:
|
||||
echo "FUSE device: $(ls -la /dev/fuse 2>&1 || echo '/dev/fuse not found')"
|
||||
|
||||
- name: Start SeaweedFS
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
# Enable Docker buildkit for better caching
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
|
||||
# Build with retry logic
|
||||
for i in {1..3}; do
|
||||
echo "Build attempt $i/3"
|
||||
@@ -77,18 +77,10 @@ jobs:
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# Start services with wait
|
||||
docker compose -f ./compose/e2e-mount.yml up --wait
|
||||
|
||||
- name: Rotate buildx cache
|
||||
if: always()
|
||||
run: |
|
||||
# Without this, --cache-to writes to .buildx-cache-new but actions/cache only
|
||||
# uploads .buildx-cache, so layers (notably the slow apt RUN) never persist.
|
||||
rm -rf /tmp/.buildx-cache
|
||||
if [ -d /tmp/.buildx-cache-new ]; then mv /tmp/.buildx-cache-new /tmp/.buildx-cache; fi
|
||||
|
||||
- name: Run FIO 4k
|
||||
timeout-minutes: 15
|
||||
run: |
|
||||
|
||||
@@ -245,51 +245,6 @@ jobs:
|
||||
path: test/s3/retention/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-lifecycle-tests:
|
||||
name: S3 Lifecycle Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Lifecycle Tests
|
||||
timeout-minutes: 12
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
set -x
|
||||
make test-with-server
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/lifecycle
|
||||
run: |
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Last 200 lines of server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080|8888)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-lifecycle-test-logs
|
||||
path: test/s3/lifecycle/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-checksum-tests:
|
||||
name: S3 Checksum Tests
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -662,77 +662,6 @@ jobs:
|
||||
path: test/s3tables/lakekeeper/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
unity-catalog-integration-tests:
|
||||
name: Unity Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Unity Catalog image
|
||||
run: docker pull unitycatalog/unitycatalog:v0.4.0
|
||||
|
||||
- name: Pre-pull Python image for delta-rs writer
|
||||
run: docker pull python:3.11-slim
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run Unity Catalog Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
docker info
|
||||
echo "=== Starting Unity Catalog Tests ==="
|
||||
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "Unity Catalog integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/unity_catalog
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker|unitycatalog)" || true
|
||||
echo "=== Unity Catalog containers ==="
|
||||
docker ps -a --filter "name=seaweed-unity-catalog" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: unity-catalog-integration-test-logs
|
||||
path: test/s3tables/unity_catalog/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
@@ -132,49 +132,7 @@ jobs:
|
||||
done
|
||||
|
||||
echo "✅ S3 server is responding, starting tests..."
|
||||
|
||||
# Spawn the lifecycle worker so test_lifecycle_expiration etc. have
|
||||
# something driving deletions. The s3tests build tag rescales one
|
||||
# day to LifeCycleInterval=10s, so a 1d rule fires within ~10s of
|
||||
# the upload's mtime; -dispatch / -checkpoint defaults are already
|
||||
# tightened under the same build tag.
|
||||
LC_LOG=/tmp/lifecycle-worker.log
|
||||
# -debug routes glog to stderr so the bootstrap walker's progress
|
||||
# shows up in $LC_LOG; without it weed shell silences glog.
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18000 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9333 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
# Aliveness check: a bad shell command exits in <1s and the suite
|
||||
# would otherwise just timeout the expiration tests with no signal.
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
# bash -e exits the step on the first tox failure, so move teardown
|
||||
# into a trap to guarantee the worker log + data dir reach the runner.
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -354,8 +312,11 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
@@ -997,39 +958,6 @@ jobs:
|
||||
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Spawn the lifecycle worker (see basic-tests block for context).
|
||||
LC_LOG=/tmp/lifecycle-worker-sql.log
|
||||
(echo "s3.lifecycle.run-shard -shards 0-15 -s3 localhost:18004 -events 0 -runtime 1800s -refresh 2s" && echo exit) \
|
||||
| weed shell -debug -master=localhost:9337 \
|
||||
> "$LC_LOG" 2>&1 &
|
||||
lc_pid=$!
|
||||
sleep 2
|
||||
if ! kill -0 "$lc_pid" 2>/dev/null; then
|
||||
echo "lifecycle worker died on startup"
|
||||
tail -50 "$LC_LOG" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "lifecycle worker pid=$lc_pid"
|
||||
|
||||
cleanup() {
|
||||
status=$?
|
||||
# SIGTERM first so the worker's stdout flushes; SIGKILL is the
|
||||
# bash fallback if it ignores TERM. Reading the log AFTER the
|
||||
# graceful-stop window catches the bootstrap walker's progress.
|
||||
kill -TERM "$lc_pid" 2>/dev/null || true
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
if [ "$status" -ne 0 ]; then
|
||||
echo "=== lifecycle worker log (tail) ==="
|
||||
tail -200 "$LC_LOG" 2>/dev/null || true
|
||||
fi
|
||||
kill -9 "$lc_pid" 2>/dev/null || true
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
rm -rf "$WEED_DATA_DIR" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
tox -- \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -1209,7 +1137,10 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_lifecycle_get \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_set_filter \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration
|
||||
# cleanup() trap handles worker/server kill + data dir wipe.
|
||||
s3tests/functional/test_s3.py::test_lifecyclev2_expiration \
|
||||
s3tests/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,15 +2,13 @@ FROM ubuntu:22.04
|
||||
|
||||
LABEL author="Chris Lu"
|
||||
|
||||
# Use Azure's Ubuntu mirror — much faster than archive.ubuntu.com from GitHub-hosted runners,
|
||||
# which have been hanging long enough on Ign:/retry to trip the 10-min step timeout.
|
||||
# Use faster mirrors and optimize package installation
|
||||
# Note: This e2e test image intentionally runs as root for simplicity and compatibility.
|
||||
# Production images (Dockerfile.go_build) use proper user isolation with su-exec.
|
||||
# For testing purposes, running as root avoids permission complexities and dependency
|
||||
# on Alpine-specific tools like su-exec (not available in Ubuntu repos).
|
||||
RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
|
||||
apt-get -o Acquire::http::Timeout=15 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=15 install -y \
|
||||
RUN apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y \
|
||||
--no-install-recommends \
|
||||
--no-install-suggests \
|
||||
curl \
|
||||
|
||||
@@ -34,7 +34,7 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.21.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
@@ -46,7 +46,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.6
|
||||
github.com/klauspost/compress v1.18.5
|
||||
github.com/klauspost/reedsolomon v1.14.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.7
|
||||
@@ -61,14 +61,14 @@ require (
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/procfs v0.20.1
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.8
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/viper v1.21.0
|
||||
@@ -93,15 +93,15 @@ require (
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.45.0
|
||||
golang.org/x/crypto v0.50.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/image v0.39.0
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
|
||||
golang.org/x/image v0.38.0
|
||||
golang.org/x/net v0.53.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/sys v0.43.0
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/tools v0.44.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.278.0
|
||||
google.golang.org/api v0.274.0
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
@@ -113,7 +113,7 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/kms v1.31.0
|
||||
cloud.google.com/go/kms v1.26.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
@@ -130,7 +130,7 @@ require (
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.44.1
|
||||
github.com/go-git/go-billy/v5 v5.9.0
|
||||
github.com/go-git/go-billy/v5 v5.8.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.13
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
@@ -143,17 +143,17 @@ require (
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.28.0
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.11.0
|
||||
github.com/rclone/rclone v1.74.1
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.73.5
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.3
|
||||
github.com/shirou/gopsutil/v4 v4.26.2
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.2
|
||||
github.com/testcontainers/testcontainers-go v0.40.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/willscott/go-nfs v0.0.4
|
||||
github.com/willscott/go-nfs v0.0.3
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
@@ -176,13 +176,12 @@ require (
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
|
||||
github.com/apache/thrift v0.23.0 // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
@@ -191,7 +190,8 @@ require (
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
@@ -224,7 +224,7 @@ require (
|
||||
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
|
||||
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
|
||||
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -257,10 +257,10 @@ require (
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.82 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
@@ -285,14 +285,14 @@ require (
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.35.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth v0.19.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.7.0 // indirect
|
||||
@@ -302,23 +302,23 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.10.3 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
@@ -326,7 +326,7 @@ require (
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
@@ -348,13 +348,13 @@ require (
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
@@ -368,7 +368,7 @@ require (
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
@@ -376,19 +376,19 @@ require (
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/errors v0.22.4 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.28.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0
|
||||
@@ -405,7 +405,7 @@ require (
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
|
||||
github.com/jonboulle/clockwork v0.5.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
@@ -419,9 +419,9 @@ require (
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.22 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
@@ -436,8 +436,8 @@ require (
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.5 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
@@ -453,7 +453,7 @@ require (
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/rfjakob/eme v1.2.0 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
@@ -466,11 +466,11 @@ require (
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.6.3 // indirect
|
||||
github.com/tinylib/msgp v1.5.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twmb/murmur3 v1.1.8 // indirect
|
||||
@@ -490,8 +490,8 @@ require (
|
||||
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
@@ -500,9 +500,9 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.42.0
|
||||
golang.org/x/time v0.15.0
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
@@ -510,17 +510,12 @@ require (
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
storj.io/picobuf v0.0.4 // indirect
|
||||
storj.io/uplink v1.14.0 // indirect
|
||||
storj.io/uplink v1.13.1 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
|
||||
// apache/thrift v0.23.0 uses math.MaxUint32 as an untyped int constant in
|
||||
// lib/go/thrift/framed_transport.go, which overflows int on 32-bit GOARCHes
|
||||
// (e.g. openbsd/arm, linux/arm). Pin to v0.22.0 until upstream fixes it.
|
||||
replace github.com/apache/thrift => github.com/apache/thrift v0.22.0
|
||||
|
||||
@@ -94,8 +94,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
|
||||
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
|
||||
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
|
||||
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
|
||||
cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA=
|
||||
cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q=
|
||||
cloud.google.com/go/auth v0.19.0 h1:DGYwtbcsGsT1ywuxsIoWi1u/vlks0moIblQHgSDgQkQ=
|
||||
cloud.google.com/go/auth v0.19.0/go.mod h1:2Aph7BT2KnaSFOM0JDPyiYgNh6PL9vGMiP8CUIXZ+IY=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
|
||||
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
|
||||
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
|
||||
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
|
||||
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
|
||||
cloud.google.com/go/kms v1.31.0 h1:LS8N92OxFDgOLg5NCo3OmbvjtQAIVT5gUHVLKIDHaFE=
|
||||
cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U=
|
||||
cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
|
||||
cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
|
||||
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
|
||||
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
|
||||
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
|
||||
@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiIS75iTVAKIUxrST5X2dVHMTptL4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
|
||||
@@ -593,10 +593,10 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
|
||||
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0=
|
||||
github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 h1:2RfP0d2QgkFH64BjZSWd59aMsc28IyWsUuHqU0txBtY=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82/go.mod h1:IPk80dOmc7VFC0DJ85xMTPmre+8xoXX6kGHAkf5jRRw=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38 h1:YCyHs3wUXEe2BEWn40vcoyaQ2ruHNmNwkasfo3Th16A=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 h1:lMHTplAYI9FtmCo/QOcpRxmPA5REVAct1r2riQmDQKw=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264/go.mod h1:wGqkOzRu/ClJibvDgcfuJNAqI2nLhe8g91tPlDKRCdE=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8=
|
||||
@@ -605,8 +605,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk=
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 h1:mJ5QbLPOm4g5qhZiVB6wbSllfpeUExftGoyPek2hk4M=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2/go.mod h1:ngpMgwkjur1VNUjqn11LPk3o5eCyOCRbcfg/0YAY7Hc=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 h1:DUnYhvC4SoC8T84rx5omnhY3+xcQg/Whyoa3mDPIMkk=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0/go.mod h1:Q3BYO6iDA2zweQPDGbNTtqft5tDcEpm6RTuqMlPcvbw=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1 h1:gN3sjapb+fVbiebS7AfQQgbV2ecTOI7ur7NPPC7Mhoc=
|
||||
github.com/Jille/raft-grpc-transport v1.6.1/go.mod h1:HbOjEdu/yzCJ/mjTF6wEOJNbAUpHfU2UOA2hVD4CNFg=
|
||||
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
|
||||
@@ -633,16 +633,16 @@ github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk=
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
|
||||
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
|
||||
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
|
||||
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
|
||||
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
|
||||
github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI=
|
||||
github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s=
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
|
||||
github.com/Sereal/Sereal/Go/sereal v0.0.0-20231009093132-b9187f1a92c6/go.mod h1:JwrycNnC8+sZPDyzM3MQ86LvaGzSpfxg885KOOwFRW4=
|
||||
github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=
|
||||
github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=
|
||||
@@ -656,14 +656,12 @@ github.com/a-h/templ v0.3.1001 h1:yHDTgexACdJttyiyamcTHXr2QkIeVF1MukLy44EAhMY=
|
||||
github.com/a-h/templ v0.3.1001/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 h1:iLDOF0rdGTrol/q8OfPIIs5kLD8XvA2q75o6Uq/tgak=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0/go.mod h1:DrEWcQJjz7t5iF2duaiyhg4jyoF0kxOD6LtECNGkZ/Q=
|
||||
github.com/aalpar/deheap v1.1.2 h1:MABHLcnjqsffb8GLkUFDigqpBBxOMz0DoKM9QfELeTw=
|
||||
github.com/aalpar/deheap v1.1.2/go.mod h1:A+nfkD4JbS05sewV0he/MYgR/90vfqyMoNNROgs+rmA=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 h1:hhdWprfSpFbN7lz3W1gM40vOgvSh1WCSMxYD6gGB4Hs=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3/go.mod h1:XaUnRxSCYgL3kkgX0QHIV0D+znljPIDImxlv2kbGv0Y=
|
||||
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
|
||||
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
|
||||
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
|
||||
github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
|
||||
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
|
||||
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
|
||||
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
|
||||
@@ -698,6 +696,7 @@ github.com/apache/cassandra-gocql-driver/v2 v2.1.0 h1:VEbbeJ2ift4deKMZ6Fs55Vs3fq
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.0/go.mod h1:QH/asJjB3mHvY6Dot6ZKMMpTcOrWJ8i9GhsvG1g0PK4=
|
||||
github.com/apache/iceberg-go v0.5.0 h1:wQj4CK5YiXZcB+tj19gWG+Jf1I6MiORQ/StSL/E5gGQ=
|
||||
github.com/apache/iceberg-go v0.5.0/go.mod h1:F/rdP1yZmnO4mQ0Qew2HTGdc+ZV57cRfxbbq/uJm1eM=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
|
||||
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
|
||||
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
|
||||
@@ -725,8 +724,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1 h1:IbWiN670htmBioc+Zj32vSpJgQ2+OYSlvTvfQ1nCORQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1/go.mod h1:tw/B596EUhBWDFGdDGuLC21fVU4A3s4/5Efy8S39W18=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
@@ -828,13 +827,15 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 h1:iLoIwb7BJECHTbNcmIhYDsQhoZiACWGNvEpyqQy97Dk=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 h1:ugiQwb7DwpWQnete2AZkTh94MonZKmxD7hDGy1qTzDs=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=
|
||||
@@ -904,9 +905,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
|
||||
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
|
||||
github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
|
||||
github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
|
||||
github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=
|
||||
github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 h1:tLWBZgPg6TV67oe76W4p+aUQEWIa52wbcuiz8GFd3vo=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
|
||||
github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
|
||||
@@ -1047,8 +1047,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ
|
||||
github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
|
||||
@@ -1071,8 +1071,8 @@ github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3
|
||||
github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY=
|
||||
github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY=
|
||||
github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA=
|
||||
github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw=
|
||||
github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0=
|
||||
github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@@ -1096,8 +1096,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
|
||||
github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM=
|
||||
github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
|
||||
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
|
||||
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
|
||||
@@ -1116,8 +1116,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
|
||||
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
|
||||
github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
|
||||
github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
|
||||
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
|
||||
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI=
|
||||
@@ -1126,8 +1126,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
|
||||
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
|
||||
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
|
||||
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
|
||||
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
@@ -1136,8 +1136,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
|
||||
github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
|
||||
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
@@ -1284,8 +1284,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
|
||||
@@ -1296,8 +1296,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
|
||||
github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
|
||||
github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
|
||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY=
|
||||
github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI=
|
||||
github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4=
|
||||
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
|
||||
@@ -1397,8 +1397,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 h1:PV6ipOJN0JIek4dqPqsTtenQmjI1SZntCUgPzhfk9gM=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160/go.mod h1:yRuPDnHgGmsbvopF0amMqXxr4n32GynzX5GTwFYDHaw=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd h1:dSIuz2mpJAPQfhHYtG57D0qwSkgC/vQ69gHfeyQ4kxA=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -1429,8 +1429,8 @@ github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
|
||||
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb h1:6vkM8gO+zFV2m21QzGYyUSq5TP0VQgP2Xz3UQyCN2kI=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb/go.mod h1:H1+whwD0Qe3YOunlXIWhh3rlvzW5cZfkMDYGQPg+KAM=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
@@ -1475,8 +1475,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
@@ -1523,8 +1523,8 @@ github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJV
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts=
|
||||
github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
|
||||
github.com/lpar/date v1.0.0/go.mod h1:KjYe0dDyMQTgpqcUz4LEIeM5VZwhggjVx/V2dtc8NSo=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
|
||||
github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
|
||||
@@ -1544,8 +1544,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk=
|
||||
github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
|
||||
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
@@ -1657,12 +1657,12 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg=
|
||||
github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 h1:eDkWg6ZN0uKwWzSekoFcQJhR+C+F/aVdTwr+lGHU9Qk=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0/go.mod h1:8ZzvzuEG/cFLFZhxg/Mg1w19KqyXBKO3c17QIc5PkGs=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 h1:l9awEvzWvxmYhy/97A0hZ87pa7BncYXmcO/S8+rvgK0=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0/go.mod h1:oB8jFGVc/7/zJ+DbleE8MzGHjhs2ioCz5stRTdZdIcY=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
||||
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
|
||||
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
|
||||
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
|
||||
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
|
||||
github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
|
||||
@@ -1755,8 +1755,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8=
|
||||
github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
@@ -1779,18 +1779,18 @@ github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8=
|
||||
github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg=
|
||||
github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
|
||||
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
|
||||
github.com/rclone/go-proton-api v1.0.2/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.74.1 h1:QQI//tl4O5uooTowdKwNKcv5qCQvJ3LIBG00+TKeXw0=
|
||||
github.com/rclone/rclone v1.74.1/go.mod h1:44epaKf5w/1ZQounefr76wx4+y9NZmgfMKle9T3pzj0=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.73.5 h1:r8a9JHYIWUqk7hNRJuMJ3cROkKfB2zmfkADg8ZLYh6I=
|
||||
github.com/rclone/rclone v1.73.5/go.mod h1:WVv8gvA/lEl/Y37e8I8yosm7ZY+Szq7ujXbJS8Ol63o=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
@@ -1809,8 +1809,8 @@ github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=
|
||||
github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
@@ -1856,15 +1856,15 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
|
||||
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
|
||||
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
|
||||
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
|
||||
@@ -1887,8 +1887,8 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
@@ -1930,8 +1930,8 @@ github.com/substrait-io/substrait-protobuf/go v0.81.0 h1:/qC1XYKuO4oPdTwLYySuVZ6
|
||||
github.com/substrait-io/substrait-protobuf/go v0.81.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c h1:dtcOwRimeiBFrlutmF6K94l0rxYFARNFMA+lSQ41C+M=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 h1:rrGZv6xYk37hx0tW2sYfgbO0PqStbHqz6Bq6oc9Hurg=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/tarantool/go-iproto v1.1.0 h1:HULVOIHsiehI+FnHfM7wMDntuzUddO09DKqu2WnFQ5A=
|
||||
github.com/tarantool/go-iproto v1.1.0/go.mod h1:LNCtdyZxojUed8SbOiYHoc3v9NvaZTB7p96hUySMlIo=
|
||||
@@ -1961,8 +1961,8 @@ github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1/go.mod h1:3cTcfo8GR
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8QU9nHY3xJSZR2kX9bgpZekRKGkLTmEXA=
|
||||
github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g=
|
||||
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
|
||||
github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s=
|
||||
github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/tinylib/msgp v1.5.0 h1:GWnqAE54wmnlFazjq2+vgr736Akg58iiHImh+kPY2pc=
|
||||
github.com/tinylib/msgp v1.5.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
@@ -2028,8 +2028,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/willscott/go-nfs v0.0.4 h1:1vpOPAdECmoT2KmZ8u+ukO/jfvDjMEUNYhA2F1jGJtI=
|
||||
github.com/willscott/go-nfs v0.0.4/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
|
||||
github.com/willscott/go-nfs v0.0.3 h1:Z5fHVxMsppgEucdkKBN26Vou19MtEM875NmRwj156RE=
|
||||
github.com/willscott/go-nfs v0.0.3/go.mod h1:VhNccO67Oug787VNXcyx9JDI3ZoSpqoKMT/lWMhUIDg=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886 h1:DtrBtkgTJk2XGt4T7eKdKVkd9A5NCevN2e4inLXtsqA=
|
||||
github.com/willscott/go-nfs-client v0.0.0-20251022144359-801f10d98886/go.mod h1:Tq++Lr/FgiS3X48q5FETemXiSLGuYMQT2sPjYNPJSwA=
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
|
||||
@@ -2098,8 +2098,6 @@ github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
|
||||
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
|
||||
github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
|
||||
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/mwc v0.0.7 h1:0NerGhCww6ZQx+/xCx5iwznftveokvto1KILpYfENZk=
|
||||
github.com/zeebo/mwc v0.0.7/go.mod h1:0B32or6moOig1YGuqMoimBpU9QK9uYaGG2bBOuddqtE=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
@@ -2130,12 +2128,12 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE=
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0/go.mod h1:fvPi2qXDqFs8M4B4fmJhE92TyQs9Ydjlg3RvfUp+NbQ=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 h1:0tY123n7CdWMem7MOVdKOt0YfshufLCwfE5Bob+hQuM=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0/go.mod h1:CosX/aS4eHnG9D7nESYpV753l4j9q5j3SL/PUYd2lR8=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
@@ -2233,8 +2231,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
@@ -2248,8 +2246,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
|
||||
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -2283,8 +2281,8 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
|
||||
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2514,8 +2512,8 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4=
|
||||
golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c h1:6a8FdnNk6bTXBjR4AGKFgUKuo+7GnR3FX5L7CbveeZc=
|
||||
golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2632,8 +2630,8 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
|
||||
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -2711,8 +2709,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
|
||||
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
|
||||
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
|
||||
google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E=
|
||||
google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -2850,8 +2848,8 @@ google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgn
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -3054,8 +3052,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aN
|
||||
sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
|
||||
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
|
||||
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a h1:7gSBQY3vhQMIqi3vfaEXR7mreRjcBLfVsYY0rHIN7P0=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a/go.mod h1:kyxwKwlfH4paBZCZt/szvRB770ieAIJF73iDy2DpEHw=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c h1:UDXSrdeLJe3QFouavSW10fYdpclK0YNu3KvQHzqq2+k=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c/go.mod h1:XNX7uykja6aco92y2y8RuqaXIDRPpt1YA2OQDKlKEUk=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 h1:8OE12DvUnB9lfZcHe7IDGsuhjrY9GBAr964PVHmhsro=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 h1:5MZ0CyMbG6Pi0rRzUWVG6dvpXjbBYEX2oyXuj+tT+sk=
|
||||
@@ -3064,7 +3062,7 @@ storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q=
|
||||
storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs=
|
||||
storj.io/picobuf v0.0.4 h1:qswHDla+YZ2TovGtMnU4astjvrADSIz84FXRn0qgP6o=
|
||||
storj.io/picobuf v0.0.4/go.mod h1:hSMxmZc58MS/2qSLy1I0idovlO7+6K47wIGUyRZa6mg=
|
||||
storj.io/uplink v1.14.0 h1:J1yXlt0aRr6kgLTHWXOWosNCFVfbamlcyd+CSxyIczo=
|
||||
storj.io/uplink v1.14.0/go.mod h1:2ysmjzd/1Xtz4VKoErNcSqBQz3UC9WKTVuLMV1cNu6E=
|
||||
storj.io/uplink v1.13.1 h1:C8RdW/upALoCyuF16Lod9XGCXEdbJAS+ABQy9JO/0pA=
|
||||
storj.io/uplink v1.13.1/go.mod h1:x0MQr4UfFsQBwgVWZAtEsLpuwAn6dg7G0Mpne1r516E=
|
||||
tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc=
|
||||
tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.24"
|
||||
appVersion: "4.23"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.24.0
|
||||
version: 4.23.0
|
||||
|
||||
@@ -247,9 +247,6 @@ message VolumeUnmountResponse {
|
||||
message VolumeDeleteRequest {
|
||||
uint32 volume_id = 1;
|
||||
bool only_empty = 2;
|
||||
// when true, do not remove the cloud-tier object backing the volume.
|
||||
// used for moves where another server is taking over the same .vif.
|
||||
bool keep_remote_data = 3;
|
||||
}
|
||||
message VolumeDeleteResponse {
|
||||
}
|
||||
@@ -458,7 +455,6 @@ message VolumeEcShardsMountRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
repeated uint32 shard_ids = 3;
|
||||
string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
|
||||
}
|
||||
message VolumeEcShardsMountResponse {
|
||||
}
|
||||
@@ -641,7 +637,6 @@ message FetchAndWriteNeedleRequest {
|
||||
}
|
||||
repeated Replica replicas = 6;
|
||||
string auth = 7;
|
||||
int32 download_concurrency = 8; // multipart download concurrency if supported by the remote storage client; for S3, 0 = default (5)
|
||||
// remote conf
|
||||
remote_pb.RemoteConf remote_conf = 15;
|
||||
remote_pb.RemoteStorageLocation remote_location = 16;
|
||||
|
||||
@@ -288,10 +288,6 @@ async fn run(
|
||||
config.jwt_read_signing_expires_seconds,
|
||||
);
|
||||
let master_url = config.masters.first().cloned().unwrap_or_default();
|
||||
// Defensive-copy the configured seed masters before freezing the lookup
|
||||
// set, so any later mutation of config.masters cannot desync them.
|
||||
let master_urls: Vec<String> = config.masters.clone();
|
||||
let seed_master_set = VolumeServerState::build_seed_master_set(&master_urls);
|
||||
let self_url = format!("{}:{}", config.ip, config.port);
|
||||
let (http_client, outgoing_http_scheme) = build_outgoing_http_client(&config)?;
|
||||
let outgoing_grpc_tls = load_outgoing_grpc_tls(&config)?;
|
||||
@@ -328,9 +324,7 @@ async fn run(
|
||||
),
|
||||
read_mode: config.read_mode,
|
||||
master_url,
|
||||
master_urls,
|
||||
seed_master_set,
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
master_urls: config.masters.clone(),
|
||||
self_url,
|
||||
http_client,
|
||||
outgoing_http_scheme,
|
||||
|
||||
@@ -145,37 +145,6 @@ pub struct VolumeGrpcService {
|
||||
}
|
||||
|
||||
impl VolumeGrpcService {
|
||||
/// Verifies the gRPC caller is allowed to invoke a destructive admin
|
||||
/// operation. Mirrors the Go side's checkGrpcAdminAuth: an empty
|
||||
/// whitelist accepts everyone (insecure-by-default for tests and
|
||||
/// upgrades), a populated whitelist accepts only matching peer IPs.
|
||||
///
|
||||
/// `remote_addr()` on a real gRPC connection always yields the peer's
|
||||
/// SocketAddr; if it is somehow None we deny, matching "if we don't
|
||||
/// know who the caller is, refuse."
|
||||
fn check_grpc_admin_auth<T>(&self, request: &Request<T>) -> Result<(), Status> {
|
||||
let remote = match request.remote_addr() {
|
||||
Some(addr) => addr,
|
||||
None => {
|
||||
tracing::warn!("gRPC admin auth failed: no peer info");
|
||||
return Err(Status::permission_denied("no peer info"));
|
||||
}
|
||||
};
|
||||
let host = remote.ip().to_string();
|
||||
let guard = self.state.guard.read().unwrap();
|
||||
if !guard.check_whitelist(&host) {
|
||||
tracing::warn!(
|
||||
"gRPC admin auth failed: {} is not whitelisted (remote: {})",
|
||||
host,
|
||||
remote,
|
||||
);
|
||||
return Err(Status::permission_denied(format!(
|
||||
"not authorized: {host}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn notify_master_volume_readonly(
|
||||
&self,
|
||||
info: &MasterVolumeInfo,
|
||||
@@ -490,7 +459,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCompactRequest>,
|
||||
) -> Result<Response<Self::VacuumVolumeCompactStream>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -550,7 +518,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCommitRequest>,
|
||||
) -> Result<Response<volume_server_pb::VacuumVolumeCommitResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
|
||||
@@ -586,7 +553,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VacuumVolumeCleanupRequest>,
|
||||
) -> Result<Response<volume_server_pb::VacuumVolumeCleanupResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
@@ -602,7 +568,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::DeleteCollectionRequest>,
|
||||
) -> Result<Response<volume_server_pb::DeleteCollectionResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let collection = &request.into_inner().collection;
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
store
|
||||
@@ -615,7 +580,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::AllocateVolumeRequest>,
|
||||
) -> Result<Response<volume_server_pb::AllocateVolumeResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -764,7 +728,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMountRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMountResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
|
||||
@@ -781,7 +744,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeUnmountRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeUnmountResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let vid = VolumeId(request.into_inner().volume_id);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
// Go returns nil when volume is not found (idempotent unmount)
|
||||
@@ -795,7 +757,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeDeleteRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeDeleteResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -809,7 +770,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
}
|
||||
store
|
||||
.delete_volume(vid, req.only_empty, req.keep_remote_data)
|
||||
.delete_volume(vid, req.only_empty)
|
||||
.map_err(|e| Status::internal(e.to_string()))?;
|
||||
self.state.volume_state_notify.notify_one();
|
||||
Ok(Response::new(volume_server_pb::VolumeDeleteResponse {}))
|
||||
@@ -819,7 +780,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMarkReadonlyRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMarkReadonlyResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
// Go: volume lookup (L239-241) happens before maintenance check (L166 in makeVolumeReadonly)
|
||||
@@ -839,7 +799,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeMarkWritableRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeMarkWritableResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
let info = {
|
||||
@@ -889,7 +848,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeConfigureRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeConfigureResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -1043,10 +1001,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
if store.find_volume(vid).is_some() {
|
||||
drop(store);
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
// keep remote data: the inbound copy carries a .vif that may
|
||||
// point at the same cloud-tier object the existing volume
|
||||
// references.
|
||||
store.delete_volume(vid, false, true).map_err(|e| {
|
||||
store.delete_volume(vid, false).map_err(|e| {
|
||||
Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
|
||||
})?;
|
||||
self.state.volume_state_notify.notify_one();
|
||||
@@ -2587,7 +2542,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeEcShardsDeleteRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeEcShardsDeleteResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let vid = VolumeId(req.volume_id);
|
||||
@@ -2612,7 +2566,7 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let mut store = self.state.store.write().unwrap();
|
||||
for &shard_id in &req.shard_ids {
|
||||
store
|
||||
.mount_ec_shard(vid, &req.collection, shard_id, &req.source_disk_type)
|
||||
.mount_ec_shard(vid, &req.collection, shard_id)
|
||||
.map_err(|e| {
|
||||
Status::internal(format!("mount {}.{}: {}", req.volume_id, shard_id, e))
|
||||
})?;
|
||||
@@ -3286,9 +3240,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
async fn volume_server_leave(
|
||||
&self,
|
||||
request: Request<volume_server_pb::VolumeServerLeaveRequest>,
|
||||
_request: Request<volume_server_pb::VolumeServerLeaveRequest>,
|
||||
) -> Result<Response<volume_server_pb::VolumeServerLeaveResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
*self.state.is_stopping.write().unwrap() = true;
|
||||
self.state.is_heartbeating.store(false, Ordering::Relaxed);
|
||||
// Wake heartbeat loop to send deregistration.
|
||||
@@ -3438,7 +3391,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
&self,
|
||||
request: Request<volume_server_pb::ScrubVolumeRequest>,
|
||||
) -> Result<Response<volume_server_pb::ScrubVolumeResponse>, Status> {
|
||||
self.check_grpc_admin_auth(&request)?;
|
||||
let req = request.into_inner();
|
||||
|
||||
// Validate mode
|
||||
@@ -3898,24 +3850,6 @@ impl VolumeServer for VolumeGrpcService {
|
||||
|
||||
let start = now_ns();
|
||||
|
||||
// Empty target is a self-liveness probe and stays unauthenticated.
|
||||
// Otherwise gate the dial on cluster membership: volume servers only
|
||||
// know masters, so any other target type is refused. Mirrors Go's
|
||||
// volume_grpc_admin.go Ping admission check. tonic forbids returning
|
||||
// a body alongside an error, so we surface the InvalidArgument status
|
||||
// alone — behaviour-identical to Go's status.Errorf return.
|
||||
if !req.target.is_empty()
|
||||
&& !self
|
||||
.state
|
||||
.is_known_ping_target(&req.target, &req.target_type)
|
||||
.await
|
||||
{
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"unknown ping target {} of type {}",
|
||||
req.target, req.target_type
|
||||
)));
|
||||
}
|
||||
|
||||
// Route ping based on target type (matches Go's volume_grpc_admin.go Ping)
|
||||
let remote_time_ns = if req.target_type == "volumeServer" {
|
||||
match ping_volume_server_target(&req.target, self.state.outgoing_grpc_tls.as_ref())
|
||||
@@ -4578,8 +4512,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -4682,8 +4614,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -4732,206 +4662,6 @@ mod tests {
|
||||
.remove("s3.incr_copy_test");
|
||||
}
|
||||
|
||||
/// Build a bare-bones service with no on-disk store but a configurable
|
||||
/// seed master set, for Ping admission tests. Matches the structure of
|
||||
/// `make_local_service_with_volume` minus the volume bits.
|
||||
fn make_service_with_seed_masters(seeds: &[&str]) -> (VolumeGrpcService, TempDir) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let master_urls: Vec<String> = seeds.iter().map(|s| (*s).to_string()).collect();
|
||||
let seed_master_set =
|
||||
crate::server::volume_server::VolumeServerState::build_seed_master_set(&master_urls);
|
||||
|
||||
let state = Arc::new(VolumeServerState {
|
||||
store: RwLock::new(store),
|
||||
guard: RwLock::new(Guard::new(
|
||||
&[],
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
SigningKey(vec![]),
|
||||
0,
|
||||
)),
|
||||
is_stopping: RwLock::new(false),
|
||||
maintenance: std::sync::atomic::AtomicBool::new(false),
|
||||
state_version: std::sync::atomic::AtomicU32::new(0),
|
||||
concurrent_upload_limit: 0,
|
||||
concurrent_download_limit: 0,
|
||||
inflight_upload_data_timeout: std::time::Duration::from_secs(60),
|
||||
inflight_download_data_timeout: std::time::Duration::from_secs(60),
|
||||
inflight_upload_bytes: std::sync::atomic::AtomicI64::new(0),
|
||||
inflight_download_bytes: std::sync::atomic::AtomicI64::new(0),
|
||||
upload_notify: tokio::sync::Notify::new(),
|
||||
download_notify: tokio::sync::Notify::new(),
|
||||
data_center: String::new(),
|
||||
rack: String::new(),
|
||||
file_size_limit_bytes: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
|
||||
has_master: false,
|
||||
pre_stop_seconds: 0,
|
||||
volume_state_notify: tokio::sync::Notify::new(),
|
||||
write_queue: std::sync::OnceLock::new(),
|
||||
s3_tier_registry: std::sync::RwLock::new(
|
||||
crate::remote_storage::s3_tier::S3TierRegistry::new(),
|
||||
),
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: master_urls.first().cloned().unwrap_or_default(),
|
||||
master_urls,
|
||||
seed_master_set,
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
outgoing_grpc_tls: None,
|
||||
metrics_runtime: std::sync::RwLock::new(
|
||||
crate::server::volume_server::RuntimeMetricsConfig::default(),
|
||||
),
|
||||
metrics_notify: tokio::sync::Notify::new(),
|
||||
fix_jpg_orientation: false,
|
||||
has_slow_read: false,
|
||||
read_buffer_size_bytes: 1024 * 1024,
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
});
|
||||
|
||||
(VolumeGrpcService { state }, tmp)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_empty_target_is_self_probe() {
|
||||
// Empty target stays unauthenticated and returns Ok with timing fields
|
||||
// populated — it is the local liveness probe path.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&[]);
|
||||
let response = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: String::new(),
|
||||
target_type: String::new(),
|
||||
}))
|
||||
.await
|
||||
.expect("empty target ping must succeed");
|
||||
let inner = response.into_inner();
|
||||
assert!(inner.start_time_ns > 0);
|
||||
assert!(inner.stop_time_ns >= inner.start_time_ns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_seed_master_target_passes_admission() {
|
||||
// A target that matches a configured seed master clears admission.
|
||||
// The dial itself may or may not succeed depending on what's listening
|
||||
// on the loopback; either way, the response must not be the
|
||||
// InvalidArgument the gate would surface.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9333".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
assert!(
|
||||
!err.message().contains("unknown ping target"),
|
||||
"admission gate should have allowed this target: {}",
|
||||
err.message()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_unknown_master_target_rejected() {
|
||||
// A master-type target not in the seed list and not the current
|
||||
// master is refused with InvalidArgument.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let err = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9999".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await
|
||||
.expect_err("unknown master target must be rejected");
|
||||
assert_eq!(err.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
"unknown ping target localhost:9999 of type master"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_volume_server_target_always_rejected() {
|
||||
// Volume servers do not maintain a peer-volume list, so volumeServer
|
||||
// pings are refused regardless of address.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
let err = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:8080".to_string(),
|
||||
target_type: "volumeServer".to_string(),
|
||||
}))
|
||||
.await
|
||||
.expect_err("volumeServer target must be rejected");
|
||||
assert_eq!(err.code(), tonic::Code::InvalidArgument);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
"unknown ping target localhost:8080 of type volumeServer"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_current_master_target_passes_admission() {
|
||||
// A target that matches the current (post-leader-change) master also
|
||||
// clears admission, even if it is not in the seed list. Pick a port
|
||||
// that is extremely unlikely to be live so the test does not flake on
|
||||
// a developer machine that happens to be running a real master.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333"]);
|
||||
// Simulate the heartbeat goroutine having moved to a new leader.
|
||||
*service.state.current_master_url.write().await = "127.0.0.1:1".to_string();
|
||||
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "127.0.0.1:1".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
assert!(
|
||||
!err.message().contains("unknown ping target"),
|
||||
"leader-change master should be admitted: {}",
|
||||
err.message()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ping_target_with_grpc_port_suffix_is_normalised() {
|
||||
// Seed masters in pb.ServerAddress form (`host:port.grpcPort`) must
|
||||
// match a Ping target sent in plain `host:port` form, since the gate
|
||||
// normalises both sides through to_http_address.
|
||||
let (service, _tmp) = make_service_with_seed_masters(&["localhost:9333.19333"]);
|
||||
let result = service
|
||||
.ping(Request::new(volume_server_pb::PingRequest {
|
||||
target: "localhost:9333".to_string(),
|
||||
target_type: "master".to_string(),
|
||||
}))
|
||||
.await;
|
||||
if let Err(err) = result {
|
||||
assert_ne!(err.code(), tonic::Code::InvalidArgument, "got {err:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_volume_ec_shards_generate_persists_expire_at_sec() {
|
||||
let ttl = crate::storage::needle::ttl::TTL::read("3m").unwrap();
|
||||
|
||||
@@ -397,15 +397,6 @@ async fn do_heartbeat(
|
||||
let mut response_stream = client.send_heartbeat(stream).await?.into_inner();
|
||||
|
||||
info!("Heartbeat stream established with {}", grpc_addr);
|
||||
// Publish the master we're now talking to in canonical http host:port
|
||||
// form so Ping admission can recognise it once a leader change moves us
|
||||
// off the seed list. Mirrors Go's vs.setCurrentMaster(masterAddress).
|
||||
{
|
||||
let normalised =
|
||||
super::volume_server::to_http_address(current_master).into_owned();
|
||||
let mut guard = state.current_master_url.write().await;
|
||||
*guard = normalised;
|
||||
}
|
||||
if is_stopping(state) {
|
||||
state.is_heartbeating.store(false, Ordering::Relaxed);
|
||||
send_deregister_heartbeat(config, state, &tx).await;
|
||||
@@ -869,7 +860,7 @@ fn build_heartbeat_with_ec_status(
|
||||
}
|
||||
|
||||
for vid in delete_vids {
|
||||
let _ = loc.delete_volume(vid, false, false);
|
||||
let _ = loc.delete_volume(vid, false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1042,8 +1033,6 @@ mod tests {
|
||||
read_mode: ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
@@ -1252,7 +1241,7 @@ mod tests {
|
||||
let shard_path = format!("{}/ec_metrics_case_27.ec00", dir);
|
||||
std::fs::write(&shard_path, b"ec-shard").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(27), "ec_metrics_case", &[0])
|
||||
.unwrap();
|
||||
|
||||
let state = test_state_with_store(store);
|
||||
@@ -1294,7 +1283,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/expired_heartbeat_ec_31.ec00", dir), b"expired").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0], "")
|
||||
.mount_ec_shards(VolumeId(31), "expired_heartbeat_ec", &[0])
|
||||
.unwrap();
|
||||
store
|
||||
.find_ec_volume_mut(VolumeId(31))
|
||||
@@ -1597,7 +1586,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/ec_delta_case_81.ec00", dir), b"delta").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(81), "ec_delta_case", &[0])
|
||||
.unwrap();
|
||||
let current = collect_ec_shard_delta_messages(&store);
|
||||
let (new_ec_shards, deleted_ec_shards) =
|
||||
|
||||
@@ -81,16 +81,6 @@ pub struct VolumeServerState {
|
||||
pub master_url: String,
|
||||
/// Seed master addresses for UI rendering.
|
||||
pub master_urls: Vec<String>,
|
||||
/// Canonical http `host:port` form of every configured seed master.
|
||||
/// Built once at construction so Ping admission stays O(1). Mirrors
|
||||
/// Go's `seedMasterSet` on `VolumeServer`.
|
||||
pub seed_master_set: std::collections::HashSet<String>,
|
||||
/// Current master this server is heartbeating with, in canonical http
|
||||
/// `host:port` form. Empty when no heartbeat connection is active. The
|
||||
/// heartbeat goroutine writes; admission reads — the lock keeps them
|
||||
/// from racing on a leader change. Mirrors Go's `currentMaster` plus
|
||||
/// `currentMasterLock`.
|
||||
pub current_master_url: tokio::sync::RwLock<String>,
|
||||
/// This server's own address (ip:port) for filtering self from lookup results.
|
||||
pub self_url: String,
|
||||
/// HTTP client for proxy requests and master lookups.
|
||||
@@ -127,35 +117,6 @@ impl VolumeServerState {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build the seed master set from a list of raw `host:port[.grpcPort]`
|
||||
/// addresses, normalised the same way Go's `pb.ServerAddress.ToHttpAddress`
|
||||
/// does (drop the `.grpcPort` suffix, preserve everything else).
|
||||
pub fn build_seed_master_set(master_urls: &[String]) -> std::collections::HashSet<String> {
|
||||
master_urls
|
||||
.iter()
|
||||
.map(|m| to_http_address(m).into_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Returns true iff `target` (normalised to canonical http `host:port`)
|
||||
/// is a master this server already knows about. Volume servers do not
|
||||
/// keep a peer-volume or peer-filer list, so Ping is scoped to masters.
|
||||
/// Mirrors Go's `VolumeServer.isKnownPingTarget`.
|
||||
pub async fn is_known_ping_target(&self, target: &str, target_type: &str) -> bool {
|
||||
if target_type != "master" {
|
||||
return false;
|
||||
}
|
||||
let key = to_http_address(target).into_owned();
|
||||
if key.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let current = self.current_master_url.read().await.clone();
|
||||
if !current.is_empty() && current == key {
|
||||
return true;
|
||||
}
|
||||
self.seed_master_set.contains(&key)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn build_metrics_router() -> Router {
|
||||
|
||||
@@ -207,8 +207,6 @@ mod tests {
|
||||
read_mode: crate::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
|
||||
@@ -304,11 +304,7 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
/// Remove all EC-related files for a volume.
|
||||
/// `pub(crate)` so the Store-level cross-disk passes in
|
||||
/// `store_ec_reconcile.rs` can call it to scrub partial EC artefacts
|
||||
/// when a healthy `.dat` for the same vid lives on a sibling disk
|
||||
/// (seaweedfs/seaweedfs#9478).
|
||||
pub(crate) fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
|
||||
fn remove_ec_volume_files(&self, collection: &str, vid: VolumeId) {
|
||||
let base = volume_file_name(&self.directory, collection, vid);
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
const MAX_SHARD_COUNT: usize = 32;
|
||||
@@ -390,20 +386,13 @@ impl DiskLocation {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove, close, and delete all files for a volume. When keep_remote_data
|
||||
/// is true the cloud-tier object backing the volume is left intact — used
|
||||
/// by moves where another server is taking over the same .vif.
|
||||
pub fn delete_volume(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
only_empty: bool,
|
||||
keep_remote_data: bool,
|
||||
) -> Result<(), VolumeError> {
|
||||
/// Remove, close, and delete all files for a volume.
|
||||
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
|
||||
if let Some(mut v) = self.volumes.remove(&vid) {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
v.destroy(only_empty, keep_remote_data)?;
|
||||
v.destroy(only_empty)?;
|
||||
Ok(())
|
||||
} else {
|
||||
Err(VolumeError::NotFound)
|
||||
@@ -424,7 +413,7 @@ impl DiskLocation {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
if let Err(e) = v.destroy(false, false) {
|
||||
if let Err(e) = v.destroy(false) {
|
||||
warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
|
||||
}
|
||||
}
|
||||
@@ -604,22 +593,14 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
/// Mount EC shards for a volume on this location.
|
||||
///
|
||||
/// `source_disk_type` is the source volume's disk type carried on the
|
||||
/// `VolumeEcShardsMount` RPC. When non-empty it overrides the in-memory
|
||||
/// EC volume's reported disk type so heartbeats keep reporting under
|
||||
/// the source's disk type after EC encoding (#9423). Empty means "use
|
||||
/// this location's disk type" — used by disk-scan reload paths that
|
||||
/// have no orchestrator context.
|
||||
pub fn mount_ec_shards(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_ids: &[u32],
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
let idx_dir = self.idx_directory.clone();
|
||||
self.mount_ec_shards_with_idx_dir(vid, collection, shard_ids, &idx_dir, source_disk_type)
|
||||
self.mount_ec_shards_with_idx_dir(vid, collection, shard_ids, &idx_dir)
|
||||
}
|
||||
|
||||
/// Mount EC shards but explicitly specify the idx directory the
|
||||
@@ -639,7 +620,6 @@ impl DiskLocation {
|
||||
collection: &str,
|
||||
shard_ids: &[u32],
|
||||
idx_dir: &str,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
let dir = self.directory.clone();
|
||||
// Avoid the entry().or_insert_with() pattern here: that closure
|
||||
@@ -656,22 +636,10 @@ impl DiskLocation {
|
||||
.ec_volumes
|
||||
.get_mut(&vid)
|
||||
.expect("just inserted above");
|
||||
// When the orchestrator supplied a source disk type on the Mount
|
||||
// RPC, override the EC volume's disk type so heartbeats report
|
||||
// under the source volume's disk type (#9423). When the caller
|
||||
// passed "" (disk-scan reload, orphan-shard reconcile, restart)
|
||||
// we only default to this location's disk type for a fresh EC
|
||||
// volume; otherwise we leave whatever a prior RPC mount set in
|
||||
// place, so empty-source reloads don't reintroduce the drift.
|
||||
if !source_disk_type.is_empty() {
|
||||
ec_vol.set_disk_type(DiskType::from_string(source_disk_type));
|
||||
} else if ec_vol.shard_count() == 0 {
|
||||
ec_vol.set_disk_type(self.disk_type.clone());
|
||||
}
|
||||
ec_vol.disk_type = self.disk_type.clone();
|
||||
|
||||
for &shard_id in shard_ids {
|
||||
let mut shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
|
||||
shard.disk_type = ec_vol.disk_type.clone();
|
||||
let shard = EcVolumeShard::new(&dir, collection, vid, shard_id as u8);
|
||||
ec_vol.add_shard(shard).map_err(VolumeError::Io)?;
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[collection, "ec_shards"])
|
||||
@@ -829,7 +797,7 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
let shard_ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
|
||||
if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids, "") {
|
||||
if let Err(e) = self.mount_ec_shards(vid, collection, &shard_ids) {
|
||||
// mount_ec_shards adds shards one at a time and increments
|
||||
// the per-shard metric for each. If it fails halfway, plain
|
||||
// ec_volumes.remove(vid) would leak metric increments for
|
||||
@@ -1189,7 +1157,7 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(loc.volumes_len(), 2);
|
||||
|
||||
loc.delete_volume(VolumeId(1), false, false).unwrap();
|
||||
loc.delete_volume(VolumeId(1), false).unwrap();
|
||||
assert_eq!(loc.volumes_len(), 1);
|
||||
assert!(loc.find_volume(VolumeId(1)).is_none());
|
||||
}
|
||||
@@ -1262,7 +1230,7 @@ mod tests {
|
||||
let shard_path = format!("{}/pics_7.ec00", dir);
|
||||
std::fs::write(&shard_path, b"ec-shard").unwrap();
|
||||
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0]).unwrap();
|
||||
assert!(loc.has_ec_volume(VolumeId(7)));
|
||||
assert!(std::path::Path::new(&shard_path).exists());
|
||||
assert!(std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
|
||||
@@ -1274,56 +1242,6 @@ mod tests {
|
||||
assert!(!std::path::Path::new(&format!("{}/pics_7.ecj", dir)).exists());
|
||||
}
|
||||
|
||||
/// #9423: after an RPC `VolumeEcShardsMount` records the source
|
||||
/// volume's disk type on the EcVolume, a later empty-source mount
|
||||
/// call (disk-scan reload, orphan-shard reconcile, restart) must
|
||||
/// not clobber that override back to the physical location's disk
|
||||
/// type — otherwise heartbeat reporting drifts back to "hdd" on
|
||||
/// every reload.
|
||||
#[test]
|
||||
fn test_mount_ec_shards_empty_source_preserves_existing_disk_type() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut loc = DiskLocation::new(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Plant the first shard so the EC volume can mount, then call
|
||||
// mount_ec_shards with source_disk_type="ssd" — simulating the
|
||||
// VolumeEcShardsMount RPC path.
|
||||
std::fs::write(format!("{}/pics_7.ec00", dir), b"ec-shard").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[0], "ssd").unwrap();
|
||||
{
|
||||
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume mounted");
|
||||
assert_eq!(
|
||||
ec_vol.disk_type,
|
||||
DiskType::Ssd,
|
||||
"first RPC mount should have set disk type to source's value",
|
||||
);
|
||||
}
|
||||
|
||||
// Plant another shard and re-mount with an empty source —
|
||||
// matching what a disk-scan reload or orphan-shard reconcile
|
||||
// would pass. The previously-recorded "ssd" override must
|
||||
// survive.
|
||||
std::fs::write(format!("{}/pics_7.ec01", dir), b"ec-shard").unwrap();
|
||||
loc.mount_ec_shards(VolumeId(7), "pics", &[1], "").unwrap();
|
||||
{
|
||||
let ec_vol = loc.find_ec_volume(VolumeId(7)).expect("ec volume still mounted");
|
||||
assert_eq!(
|
||||
ec_vol.disk_type,
|
||||
DiskType::Ssd,
|
||||
"empty-source remount must not reset disk type to the physical location's",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disk_location_persists_directory_uuid_and_tags() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
|
||||
@@ -120,15 +120,10 @@ impl EcVolume {
|
||||
shards.push(None);
|
||||
}
|
||||
|
||||
// Read expire_at_sec, version, and dat_file_size from .vif if
|
||||
// present (matches Go's MaybeLoadVolumeInfo). Prefer the data
|
||||
// dir; fall back to the idx dir for the cross-disk reconcile
|
||||
// case (#9212 / #9244). `dat_file_size` is the source .dat
|
||||
// size at encode time, used both by `locate_ec_shard_needle`
|
||||
// for shard-size math and by the Store-level prune in
|
||||
// `store_ec_reconcile.rs` to verify a sibling-disk .dat is
|
||||
// plausibly the encoding source (#9478).
|
||||
let (expire_at_sec, vif_version, vif_dat_file_size) = {
|
||||
// Read expire_at_sec and version from .vif if present (matches Go's MaybeLoadVolumeInfo).
|
||||
// Prefer the data dir; fall back to the idx dir for the
|
||||
// cross-disk reconcile case (#9212 / #9244).
|
||||
let (expire_at_sec, vif_version) = {
|
||||
let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
|
||||
if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
|
||||
if let Ok(vif_info) =
|
||||
@@ -139,12 +134,12 @@ impl EcVolume {
|
||||
} else {
|
||||
Version::current()
|
||||
};
|
||||
(vif_info.expire_at_sec, ver, vif_info.dat_file_size)
|
||||
(vif_info.expire_at_sec, ver)
|
||||
} else {
|
||||
(0, Version::current(), 0)
|
||||
(0, Version::current())
|
||||
}
|
||||
} else {
|
||||
(0, Version::current(), 0)
|
||||
(0, Version::current())
|
||||
}
|
||||
};
|
||||
|
||||
@@ -155,7 +150,7 @@ impl EcVolume {
|
||||
dir_idx: dir_idx.to_string(),
|
||||
version: vif_version,
|
||||
shards,
|
||||
dat_file_size: vif_dat_file_size,
|
||||
dat_file_size: 0,
|
||||
data_shards,
|
||||
parity_shards,
|
||||
ecx_file: None,
|
||||
@@ -332,20 +327,6 @@ impl EcVolume {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Override the disk type the EC volume (and its already-mounted
|
||||
/// shards) reports under. Used by the `VolumeEcShardsMount` handler
|
||||
/// so the source volume's disk type is preserved across encoding
|
||||
/// (#9423). Not persisted across restarts — disk-scan reload paths
|
||||
/// default to the physical location's disk type.
|
||||
pub fn set_disk_type(&mut self, d: DiskType) {
|
||||
self.disk_type = d.clone();
|
||||
for slot in self.shards.iter_mut() {
|
||||
if let Some(shard) = slot {
|
||||
shard.disk_type = d.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove and close a shard.
|
||||
pub fn remove_shard(&mut self, shard_id: ShardId) {
|
||||
if let Some(ref mut shard) = self.shards[shard_id as usize] {
|
||||
|
||||
@@ -85,17 +85,6 @@ impl Store {
|
||||
|
||||
self.locations.push(loc);
|
||||
|
||||
// First scrub partial EC artefacts left on one disk by an
|
||||
// interrupted encode while the source .dat still lives on a
|
||||
// sibling disk of the same store. The per-disk loader cannot
|
||||
// see the sibling .dat and so loads the partial shards as if
|
||||
// they were a distributed-EC layout, which makes the volume
|
||||
// server heartbeat both a regular replica and an EC shard set
|
||||
// for the same vid (seaweedfs/seaweedfs#9478). Running before
|
||||
// the cross-disk reconcile keeps that pass from later
|
||||
// re-loading shards we just cleaned up.
|
||||
self.prune_incomplete_ec_with_sibling_dat();
|
||||
|
||||
// After every disk has finished its per-disk EC scan, sweep
|
||||
// the store for shards that live on a disk without local index
|
||||
// files and load them by reaching across to a sibling disk's
|
||||
@@ -117,7 +106,6 @@ impl Store {
|
||||
tracing::error!("load_new_volumes error in {}: {}", loc.directory, e);
|
||||
}
|
||||
}
|
||||
self.prune_incomplete_ec_with_sibling_dat();
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
}
|
||||
|
||||
@@ -319,18 +307,11 @@ impl Store {
|
||||
)
|
||||
}
|
||||
|
||||
/// Delete a volume from any location. When keep_remote_data is true the
|
||||
/// cloud-tier object backing the volume is left intact — used by moves
|
||||
/// where another server is taking over the same .vif.
|
||||
pub fn delete_volume(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
only_empty: bool,
|
||||
keep_remote_data: bool,
|
||||
) -> Result<(), VolumeError> {
|
||||
/// Delete a volume from any location.
|
||||
pub fn delete_volume(&mut self, vid: VolumeId, only_empty: bool) -> Result<(), VolumeError> {
|
||||
for loc in &mut self.locations {
|
||||
if loc.find_volume(vid).is_some() {
|
||||
return loc.delete_volume(vid, only_empty, keep_remote_data);
|
||||
return loc.delete_volume(vid, only_empty);
|
||||
}
|
||||
}
|
||||
Err(VolumeError::NotFound)
|
||||
@@ -648,25 +629,22 @@ impl Store {
|
||||
))
|
||||
})?;
|
||||
|
||||
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids, "")
|
||||
self.locations[loc_idx].mount_ec_shards(vid, collection, shard_ids)
|
||||
}
|
||||
|
||||
/// Mount a single EC shard, searching all locations for the shard file.
|
||||
/// Matches Go's Store.MountEcShards which mounts one shard at a time.
|
||||
/// `source_disk_type` is the source volume's disk type from the
|
||||
/// `VolumeEcShardsMount` RPC (#9423); pass `""` for non-RPC paths.
|
||||
pub fn mount_ec_shard(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
collection: &str,
|
||||
shard_id: u32,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
for loc in &mut self.locations {
|
||||
// Check if the shard file exists on this location
|
||||
let shard = EcVolumeShard::new(&loc.directory, collection, vid, shard_id as u8);
|
||||
if std::path::Path::new(&shard.file_name()).exists() {
|
||||
loc.mount_ec_shards(vid, collection, &[shard_id], source_disk_type)?;
|
||||
loc.mount_ec_shards(vid, collection, &[shard_id])?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -1496,7 +1474,7 @@ mod tests {
|
||||
|
||||
std::fs::write(format!("{}/expired_ec_case_9.ec00", dir), b"expired").unwrap();
|
||||
store.locations[0]
|
||||
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0], "")
|
||||
.mount_ec_shards(VolumeId(9), "expired_ec_case", &[0])
|
||||
.unwrap();
|
||||
store.find_ec_volume_mut(VolumeId(9)).unwrap().expire_at_sec = 1;
|
||||
|
||||
@@ -1591,7 +1569,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
store.locations[1]
|
||||
.mount_ec_shards(vid, collection, &[0], "")
|
||||
.mount_ec_shards(vid, collection, &[0])
|
||||
.unwrap();
|
||||
|
||||
// Stray .ecx on disk 2 must not win.
|
||||
@@ -1658,7 +1636,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
store.locations[1]
|
||||
.mount_ec_shards(vid, collection, &[0], "")
|
||||
.mount_ec_shards(vid, collection, &[0])
|
||||
.unwrap();
|
||||
|
||||
let got = store.find_ec_shard_target_location(collection, vid, 10);
|
||||
|
||||
@@ -21,22 +21,9 @@ use std::fs;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage::disk_location::{is_ec_shard_extension, parse_collection_volume_id_pub};
|
||||
use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
|
||||
use crate::storage::store::Store;
|
||||
use crate::storage::super_block::SUPER_BLOCK_SIZE;
|
||||
use crate::storage::types::VolumeId;
|
||||
|
||||
/// Sibling-disk `.dat` candidate for `prune_incomplete_ec_with_sibling_dat`.
|
||||
/// We record both the disk index and the file's size: the size is
|
||||
/// consulted before deleting any EC artefacts. A zero-byte or truncated
|
||||
/// `.dat` is not a credible fallback, and we'd rather leave the partial
|
||||
/// EC in place than wipe it based on garbage.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct DatOwnerInfo {
|
||||
location: usize,
|
||||
size: u64,
|
||||
}
|
||||
|
||||
/// Key for orphan-shard reconciliation: collection + volume id. Two
|
||||
/// collections can re-use the same volume id, and we must only pair
|
||||
/// shards with their own `.ecx`.
|
||||
@@ -128,7 +115,6 @@ impl Store {
|
||||
&key.collection,
|
||||
&shard_ids,
|
||||
&owner_idx_dir,
|
||||
"",
|
||||
) {
|
||||
// mount_ec_shards_with_idx_dir adds shards one at a
|
||||
// time and increments the `ec_shards` gauge per shard
|
||||
@@ -149,187 +135,6 @@ impl Store {
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove leftover EC artefacts on one disk when a healthy `.dat`
|
||||
/// for the same `(collection, vid)` lives on a sibling disk of the
|
||||
/// same store. Mirrors `Store.pruneIncompleteEcWithSiblingDat` in
|
||||
/// `weed/storage/store_ec_reconcile.go` (seaweedfs/seaweedfs#9478).
|
||||
///
|
||||
/// `DiskLocation::handle_found_ecx_file` only looks for `.dat` in
|
||||
/// the same disk as the EC shards. In a multi-disk volume server an
|
||||
/// interrupted EC encode can leave `.ec??` + `.ecx` on disk B while
|
||||
/// the source `.dat` still lives on disk A. The per-disk loader
|
||||
/// sees no local `.dat`, classifies the layout as distributed EC,
|
||||
/// and mounts the partial shards. The volume server then heartbeats
|
||||
/// both a regular replica and an EC shard for the same vid, and
|
||||
/// master ends up with a contradictory view.
|
||||
///
|
||||
/// Cleanup is gated on `shard_count < DATA_SHARDS_COUNT` so that a
|
||||
/// deliberate "full local EC, `.dat` retained" layout split across
|
||||
/// two disks (`.dat` on disk A, all 10+ shards on disk B) is left
|
||||
/// alone — the per-disk loader already keeps that configuration on
|
||||
/// a single disk, and pruning it here would be a behaviour
|
||||
/// regression. Distributed EC volumes (no `.dat` on any disk of
|
||||
/// this server) also fall through unchanged because the `.dat`
|
||||
/// index never matches.
|
||||
///
|
||||
/// Before deleting any EC files we also check that the sibling
|
||||
/// `.dat` is plausibly the encoding source: at least
|
||||
/// `SUPER_BLOCK_SIZE` bytes long, and — when the EC's `.vif`
|
||||
/// recorded a non-zero source size in `dat_file_size` — at least
|
||||
/// that many bytes. A zero-byte shell or a truncated `.dat` does
|
||||
/// not justify wiping the partial EC, because that EC shard may
|
||||
/// still combine usefully with shards on other servers in a
|
||||
/// recoverable distributed-EC layout.
|
||||
///
|
||||
/// We don't have to push anything to a deleted-shards channel
|
||||
/// here: the Rust heartbeat path in `server/heartbeat.rs` diffs
|
||||
/// the current `ec_volumes` snapshot against the previous one each
|
||||
/// tick, so the first heartbeat after the prune naturally emits a
|
||||
/// delete delta for whatever the per-disk pass had already
|
||||
/// reported.
|
||||
pub fn prune_incomplete_ec_with_sibling_dat(&mut self) {
|
||||
if self.locations.len() < 2 {
|
||||
return;
|
||||
}
|
||||
|
||||
let dat_owners = self.index_dat_owners();
|
||||
if dat_owners.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Snapshot under an immutable borrow so we can release it
|
||||
// before taking the mutable borrow needed for cleanup.
|
||||
struct Victim {
|
||||
loc_idx: usize,
|
||||
collection: String,
|
||||
vid: VolumeId,
|
||||
dat_dir: String,
|
||||
shard_count: usize,
|
||||
}
|
||||
let mut victims: Vec<Victim> = Vec::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
for (vid, ev) in loc.ec_volumes() {
|
||||
let shard_count = ev.shard_count();
|
||||
if shard_count >= DATA_SHARDS_COUNT {
|
||||
continue;
|
||||
}
|
||||
let key = EcKey {
|
||||
collection: ev.collection.clone(),
|
||||
vid: *vid,
|
||||
};
|
||||
let Some(owner) = dat_owners.get(&key) else {
|
||||
continue;
|
||||
};
|
||||
if owner.location == loc_idx {
|
||||
// Same-disk .dat is the job of
|
||||
// DiskLocation::validate_ec_volume during the
|
||||
// per-disk pass; don't second-guess it here.
|
||||
continue;
|
||||
}
|
||||
// Decide whether the sibling .dat is credible. Prefer
|
||||
// the size baked into .vif at encode time; fall back
|
||||
// to "at least a superblock" for old EC volumes whose
|
||||
// .vif predates the field.
|
||||
let required = if ev.dat_file_size > 0 {
|
||||
ev.dat_file_size as u64
|
||||
} else {
|
||||
SUPER_BLOCK_SIZE as u64
|
||||
};
|
||||
if owner.size < required {
|
||||
warn!(
|
||||
volume_id = vid.0,
|
||||
collection = %ev.collection,
|
||||
directory = %loc.directory,
|
||||
shard_count,
|
||||
sibling_dir = %self.locations[owner.location].directory,
|
||||
sibling_dat_size = owner.size,
|
||||
required,
|
||||
"sibling .dat is smaller than the EC source size; leaving partial EC in place so distributed reconstruction is still possible (issue 9478)",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
victims.push(Victim {
|
||||
loc_idx,
|
||||
collection: ev.collection.clone(),
|
||||
vid: *vid,
|
||||
dat_dir: self.locations[owner.location].directory.clone(),
|
||||
shard_count,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for v in victims {
|
||||
warn!(
|
||||
volume_id = v.vid.0,
|
||||
collection = %v.collection,
|
||||
directory = %self.locations[v.loc_idx].directory,
|
||||
shard_count = v.shard_count,
|
||||
required = DATA_SHARDS_COUNT,
|
||||
dat_dir = %v.dat_dir,
|
||||
"partial EC shards on this disk while a healthy .dat exists on a sibling disk; cleaning up leftover EC files (issue 9478)",
|
||||
);
|
||||
let loc = &mut self.locations[v.loc_idx];
|
||||
if let Some(mut ec_vol) = loc.remove_ec_volume(v.vid) {
|
||||
for _ in 0..ec_vol.shard_count() {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&ec_vol.collection, "ec_shards"])
|
||||
.dec();
|
||||
}
|
||||
// destroy() closes shard file handles before unlinking
|
||||
// (matters on Windows) and removes .ecx / .ecj / .vif.
|
||||
ec_vol.destroy();
|
||||
}
|
||||
// Also sweep any unmounted shard files (.ec00 .. .ec31)
|
||||
// that the per-disk loader skipped — destroy() only walks
|
||||
// the in-memory shards, but the disk may still hold others.
|
||||
loc.remove_ec_volume_files(&v.collection, v.vid);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns, for every `(collection, vid)`, the index of the first
|
||||
/// disk on this store that holds a `.dat` for it plus the file's
|
||||
/// size. Used by `prune_incomplete_ec_with_sibling_dat` so it can
|
||||
/// decide whether partial EC artefacts on another disk are
|
||||
/// leftovers of an interrupted encode AND whether the sibling
|
||||
/// `.dat` is large enough to be a credible fallback.
|
||||
///
|
||||
/// Any `.dat` `read_dir` can see is recorded — including zero-byte
|
||||
/// shells. Their mere presence means this volume was a regular
|
||||
/// volume on this server at some point, which rules out the
|
||||
/// "distributed EC, no .dat anywhere" reading. Whether the file is
|
||||
/// actually usable is the caller's call, made by comparing this
|
||||
/// size to the EC's recorded source size in `.vif`.
|
||||
fn index_dat_owners(&self) -> HashMap<EcKey, DatOwnerInfo> {
|
||||
let mut owners: HashMap<EcKey, DatOwnerInfo> = HashMap::new();
|
||||
for (loc_idx, loc) in self.locations.iter().enumerate() {
|
||||
let Ok(read) = fs::read_dir(&loc.directory) else {
|
||||
continue;
|
||||
};
|
||||
for ent in read.flatten() {
|
||||
if ent.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
let name = ent.file_name().to_string_lossy().into_owned();
|
||||
let Some(base) = name.strip_suffix(".dat") else {
|
||||
continue;
|
||||
};
|
||||
let Some((collection, vid)) = parse_collection_volume_id_pub(base) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(meta) = ent.metadata() else {
|
||||
continue;
|
||||
};
|
||||
owners
|
||||
.entry(EcKey { collection, vid })
|
||||
.or_insert(DatOwnerInfo {
|
||||
location: loc_idx,
|
||||
size: meta.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
owners
|
||||
}
|
||||
|
||||
/// Build a `(collection, vid) -> EcxOwnerInfo` map of which disk
|
||||
/// owns the `.ecx` file. `.ecx` normally lives in `IdxDirectory`
|
||||
/// but may have been written into the data directory before
|
||||
@@ -919,249 +724,6 @@ mod tests {
|
||||
assert!(ev.has_shard(0));
|
||||
assert!(ev.has_shard(1));
|
||||
}
|
||||
|
||||
/// Reproduces the issue 9478 layout for a single volume server:
|
||||
/// disk A holds a healthy `.dat` for vid 122; disk B holds a single
|
||||
/// `.ec01` plus `.ecx` / `.ecj` / `.vif` and no `.dat`. The per-disk
|
||||
/// loader mounts the partial shard on disk B as if it were a
|
||||
/// distributed-EC layout. The Store-level prune must unmount and
|
||||
/// delete the leftover EC files on disk B while leaving disk A's
|
||||
/// `.dat` untouched.
|
||||
#[test]
|
||||
fn test_prune_drops_partial_ec_when_sibling_disk_has_dat() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dat_dir = tmp.path().join("sdd");
|
||||
let ec_dir = tmp.path().join("sdf");
|
||||
std::fs::create_dir_all(&dat_dir).unwrap();
|
||||
std::fs::create_dir_all(&ec_dir).unwrap();
|
||||
|
||||
let collection = "";
|
||||
let vid = 122u32;
|
||||
|
||||
// Disk A (sdd): a non-trivial .dat plus the regular-volume
|
||||
// sidecars. The .dat content doesn't matter for the prune —
|
||||
// load_existing_volumes refuses to mount it because the
|
||||
// superblock is absent — but its file name has to be present
|
||||
// so index_dat_owners records this disk as the .dat owner.
|
||||
let dat_path = dat_dir.join(format!("{}_{}.dat", collection, vid).trim_start_matches('_'));
|
||||
std::fs::write(&dat_path, vec![0u8; 1024]).unwrap();
|
||||
|
||||
// Disk B (sdf): partial EC — one shard, plus .ecx / .ecj / .vif.
|
||||
write_shard(ec_dir.to_str().unwrap(), collection, vid, 1);
|
||||
write_index_files(ec_dir.to_str().unwrap(), collection, vid, 10, 4);
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dat_dir.to_str().unwrap(),
|
||||
dat_dir.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.add_location(
|
||||
ec_dir.to_str().unwrap(),
|
||||
ec_dir.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// After add_location for sdf, the per-disk loader has mounted
|
||||
// the partial shard AND the prune has run as part of
|
||||
// add_location's epilogue. Confirm the prune removed it.
|
||||
let ec_loc = &store.locations[1];
|
||||
assert!(
|
||||
ec_loc.find_ec_volume(VolumeId(vid)).is_none(),
|
||||
"partial EC volume should have been unmounted by the prune",
|
||||
);
|
||||
|
||||
let ec_base = ec_dir
|
||||
.join(format!("{}_{}", collection, vid).trim_start_matches('_'))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
assert!(
|
||||
!std::path::Path::new(&format!("{}.ec01", ec_base)).exists(),
|
||||
"partial shard file should have been removed",
|
||||
);
|
||||
assert!(
|
||||
!std::path::Path::new(&format!("{}.ecx", ec_base)).exists(),
|
||||
".ecx should have been removed",
|
||||
);
|
||||
assert!(
|
||||
!std::path::Path::new(&format!("{}.ecj", ec_base)).exists(),
|
||||
".ecj should have been removed",
|
||||
);
|
||||
|
||||
// The .dat on the sibling disk must survive — pruning the
|
||||
// partial EC must never touch the healthy replica.
|
||||
assert!(
|
||||
dat_path.exists(),
|
||||
"healthy .dat on sibling disk should be left alone",
|
||||
);
|
||||
}
|
||||
|
||||
/// Safety guard for `prune_incomplete_ec_with_sibling_dat`: when
|
||||
/// the sibling `.dat` is smaller than the source size recorded in
|
||||
/// the EC's `.vif`, the `.dat` is clearly truncated and is not a
|
||||
/// credible fallback. The partial shard may still combine with
|
||||
/// shards on other servers in a recoverable distributed-EC layout,
|
||||
/// so we leave the EC files in place.
|
||||
#[test]
|
||||
fn test_prune_keeps_partial_ec_when_sibling_dat_is_smaller_than_vif_source_size() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dat_dir = tmp.path().join("sdd");
|
||||
let ec_dir = tmp.path().join("sdf");
|
||||
std::fs::create_dir_all(&dat_dir).unwrap();
|
||||
std::fs::create_dir_all(&ec_dir).unwrap();
|
||||
|
||||
let collection = "logs";
|
||||
let vid = 122u32;
|
||||
|
||||
// Tiny but loadable .dat on sdd: a valid 8-byte version-3
|
||||
// superblock so Volume::new succeeds, padded out to a few
|
||||
// hundred bytes so we have something well below the .vif-
|
||||
// recorded source size below. Anything smaller would let
|
||||
// Volume::new's auto-write extend the file to SUPER_BLOCK_SIZE
|
||||
// and obscure the truncation we're trying to model.
|
||||
let dat_path = dat_dir.join(format!("{}_{}.dat", collection, vid));
|
||||
let mut dat_bytes = crate::storage::super_block::SuperBlock::default().to_bytes();
|
||||
dat_bytes.resize(256, 0u8);
|
||||
std::fs::write(&dat_path, &dat_bytes).unwrap();
|
||||
let sibling_dat_size = dat_bytes.len() as u64;
|
||||
let source_size = 10 * 1024 * 1024u64; // 10 MiB recorded in .vif
|
||||
assert!(sibling_dat_size < source_size);
|
||||
|
||||
// Partial EC: one shard plus .ecx / .ecj / .vif. The .vif
|
||||
// records the source .dat size at encode time; the safety
|
||||
// check compares the sibling .dat against this value.
|
||||
write_shard(ec_dir.to_str().unwrap(), collection, vid, 1);
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
dat_file_size: source_size as i64,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
std::fs::write(
|
||||
ec_dir.join(format!("{}_{}.vif", collection, vid)),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
ec_dir.join(format!("{}_{}.ecx", collection, vid)),
|
||||
vec![0u8; 20],
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
ec_dir.join(format!("{}_{}.ecj", collection, vid)),
|
||||
b"",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dat_dir.to_str().unwrap(),
|
||||
dat_dir.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.add_location(
|
||||
ec_dir.to_str().unwrap(),
|
||||
ec_dir.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Prune ran as part of add_location's epilogue. Confirm the
|
||||
// partial EC was left alone because the sibling .dat is too
|
||||
// small to be the encoding source.
|
||||
let ec_loc = &store.locations[1];
|
||||
let ev = ec_loc
|
||||
.find_ec_volume(VolumeId(vid))
|
||||
.expect("partial EC volume must remain mounted when the sibling .dat is smaller than the .vif source size");
|
||||
assert_eq!(ev.shard_count(), 1);
|
||||
|
||||
let ec_base = ec_dir
|
||||
.join(format!("{}_{}", collection, vid))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
assert!(
|
||||
std::path::Path::new(&format!("{}.ec01", ec_base)).exists(),
|
||||
"partial shard file must survive when the sibling .dat is truncated",
|
||||
);
|
||||
assert!(
|
||||
std::path::Path::new(&format!("{}.ecx", ec_base)).exists(),
|
||||
".ecx must survive when the sibling .dat is truncated",
|
||||
);
|
||||
}
|
||||
|
||||
/// A distributed EC volume — no `.dat` on any disk of this store —
|
||||
/// must not be touched by the prune even when shard count is below
|
||||
/// `DATA_SHARDS_COUNT`. Partial shard layouts are normal for
|
||||
/// distributed EC.
|
||||
#[test]
|
||||
fn test_prune_leaves_distributed_ec_alone() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir0 = tmp.path().join("data0");
|
||||
let dir1 = tmp.path().join("data1");
|
||||
std::fs::create_dir_all(&dir0).unwrap();
|
||||
std::fs::create_dir_all(&dir1).unwrap();
|
||||
|
||||
let collection = "logs";
|
||||
let vid = 7u32;
|
||||
|
||||
// Two shards on dir0 with .ecx, no .dat anywhere.
|
||||
write_shard(dir0.to_str().unwrap(), collection, vid, 1);
|
||||
write_shard(dir0.to_str().unwrap(), collection, vid, 2);
|
||||
write_index_files(dir0.to_str().unwrap(), collection, vid, 10, 4);
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
for dir in [&dir0, &dir1] {
|
||||
store
|
||||
.add_location(
|
||||
dir.to_str().unwrap(),
|
||||
dir.to_str().unwrap(),
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let ev = store.locations[0]
|
||||
.find_ec_volume(VolumeId(vid))
|
||||
.expect("distributed EC volume should still be mounted on dir0");
|
||||
assert_eq!(
|
||||
ev.shard_count(),
|
||||
2,
|
||||
"no .dat anywhere on this store means the partial shards must be kept as distributed EC",
|
||||
);
|
||||
|
||||
let ec_base = dir0
|
||||
.join(format!("{}_{}", collection, vid))
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
assert!(std::path::Path::new(&format!("{}.ec01", ec_base)).exists());
|
||||
assert!(std::path::Path::new(&format!("{}.ec02", ec_base)).exists());
|
||||
assert!(std::path::Path::new(&format!("{}.ecx", ec_base)).exists());
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk a disk's data directory and return the `.ec??` shard files
|
||||
|
||||
@@ -773,13 +773,6 @@ impl Volume {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn nm_or_not_found(&self) -> Result<&NeedleMap, VolumeError> {
|
||||
self.nm.as_ref().ok_or_else(|| {
|
||||
tracing::warn!(volume_id = self.id.0, "needle map not loaded");
|
||||
VolumeError::NotFound
|
||||
})
|
||||
}
|
||||
|
||||
fn load_index(&mut self) -> Result<(), VolumeError> {
|
||||
let use_redb = matches!(
|
||||
self.needle_map_kind,
|
||||
@@ -1065,7 +1058,7 @@ impl Volume {
|
||||
read_option: &mut ReadOption,
|
||||
) -> Result<i32, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
|
||||
|
||||
if nv.offset.is_zero() {
|
||||
@@ -1307,7 +1300,7 @@ impl Volume {
|
||||
read_deleted: bool,
|
||||
) -> Result<NeedleStreamInfo, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(n.id).ok_or(VolumeError::NotFound)?;
|
||||
|
||||
if nv.offset.is_zero() {
|
||||
@@ -1409,7 +1402,7 @@ impl Volume {
|
||||
&self,
|
||||
needle_id: NeedleId,
|
||||
) -> Result<(u64, u16), VolumeError> {
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let nv = nm.get(needle_id).ok_or(VolumeError::NotFound)?;
|
||||
if nv.offset.is_zero() {
|
||||
return Err(VolumeError::NotFound);
|
||||
@@ -1726,7 +1719,7 @@ impl Volume {
|
||||
/// Read all live needles from the volume (for ReadAllNeedles streaming RPC).
|
||||
pub fn read_all_needles(&self) -> Result<Vec<Needle>, VolumeError> {
|
||||
let _guard = self.data_file_access_control.read_lock();
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let version = self.version();
|
||||
let dat_size = self.current_dat_file_size()? as i64;
|
||||
let mut needles = Vec::new();
|
||||
@@ -1915,7 +1908,7 @@ impl Volume {
|
||||
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
let dat_size = self.dat_file_size().map_err(VolumeError::Io)?;
|
||||
|
||||
let mut files_checked: u64 = 0;
|
||||
@@ -1975,7 +1968,7 @@ impl Volume {
|
||||
if self.dat_file.is_none() && self.remote_dat_file.is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
let nm = self.nm_or_not_found()?;
|
||||
let nm = self.nm.as_ref().ok_or(VolumeError::NotFound)?;
|
||||
|
||||
let dat_size = self.dat_file_size().map_err(|e| VolumeError::Io(e))?;
|
||||
let version = self.version();
|
||||
@@ -3088,10 +3081,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
/// Remove all volume files from disk.
|
||||
/// Destroy removes everything related to this volume. When keep_remote_data
|
||||
/// is true the cloud-tier object backing the volume is left intact — used
|
||||
/// by moves where another server is taking over the same .vif.
|
||||
pub fn destroy(&mut self, only_empty: bool, keep_remote_data: bool) -> Result<(), VolumeError> {
|
||||
pub fn destroy(&mut self, only_empty: bool) -> Result<(), VolumeError> {
|
||||
if only_empty && self.file_count() > 0 {
|
||||
return Err(VolumeError::NotEmpty);
|
||||
}
|
||||
@@ -3103,11 +3093,7 @@ impl Volume {
|
||||
}
|
||||
|
||||
let (storage_name, storage_key) = self.remote_storage_name_key();
|
||||
if !keep_remote_data
|
||||
&& self.has_remote_file
|
||||
&& !storage_name.is_empty()
|
||||
&& !storage_key.is_empty()
|
||||
{
|
||||
if self.has_remote_file && !storage_name.is_empty() && !storage_key.is_empty() {
|
||||
let backend = crate::remote_storage::s3_tier::global_s3_tier_registry()
|
||||
.read()
|
||||
.unwrap()
|
||||
@@ -3203,17 +3189,11 @@ fn get_append_at_ns(last: u64) -> u64 {
|
||||
}
|
||||
|
||||
/// Remove all files associated with a volume.
|
||||
/// .dat/.idx removals log at info level so destructive calls are traceable.
|
||||
pub(crate) fn remove_volume_files(base: &str) {
|
||||
for ext in &[
|
||||
".dat", ".idx", ".vif", ".sdx", ".cpd", ".cpx", ".note", ".rdb",
|
||||
] {
|
||||
let path = format!("{}{}", base, ext);
|
||||
let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
|
||||
let existed = fs::remove_file(&path).is_ok();
|
||||
if existed && (*ext == ".dat" || *ext == ".idx") {
|
||||
tracing::info!("removed volume file {} (size={})", path, size);
|
||||
}
|
||||
let _ = fs::remove_file(format!("{}{}", base, ext));
|
||||
}
|
||||
// leveldb uses a directory
|
||||
let _ = fs::remove_dir_all(format!("{}.ldb", base));
|
||||
@@ -3679,7 +3659,7 @@ mod tests {
|
||||
dat_path = v.file_name(".dat");
|
||||
idx_path = v.file_name(".idx");
|
||||
assert!(Path::new(&dat_path).exists());
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
}
|
||||
|
||||
assert!(!Path::new(&dat_path).exists());
|
||||
@@ -4444,7 +4424,7 @@ mod tests {
|
||||
assert!(std::path::Path::new(&idx_path).exists());
|
||||
|
||||
// Destroy the volume
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
|
||||
// .dat and .idx should be gone
|
||||
assert!(
|
||||
@@ -4501,7 +4481,7 @@ mod tests {
|
||||
assert!(std::path::Path::new(&dat_path).exists());
|
||||
assert!(std::path::Path::new(&idx_path).exists());
|
||||
|
||||
v.destroy(false, false).unwrap();
|
||||
v.destroy(false).unwrap();
|
||||
|
||||
assert!(
|
||||
!std::path::Path::new(&dat_path).exists(),
|
||||
|
||||
@@ -101,8 +101,6 @@ fn test_state_with_guard(
|
||||
read_mode: seaweed_volume::config::ReadMode::Local,
|
||||
master_url: String::new(),
|
||||
master_urls: Vec::new(),
|
||||
seed_master_set: std::collections::HashSet::new(),
|
||||
current_master_url: tokio::sync::RwLock::new(String::new()),
|
||||
self_url: String::new(),
|
||||
http_client: reqwest::Client::new(),
|
||||
outgoing_http_scheme: "http".to_string(),
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
<junit.version>4.13.2</junit.version>
|
||||
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
|
||||
<jackson.version>2.18.6</jackson.version> <!-- Upgraded from 2.15.3 -->
|
||||
<netty.version>4.2.13.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<netty.version>4.1.132.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
|
||||
<parquet.format.version>2.12.0</parquet.format.version>
|
||||
<surefire.jvm.args>
|
||||
|
||||
+46
-45
@@ -8,37 +8,36 @@ require (
|
||||
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/stretchr/testify v1.11.1
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/grpc v1.79.3
|
||||
)
|
||||
|
||||
replace github.com/seaweedfs/seaweedfs => ../../
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.11.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.10.3 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
@@ -49,7 +48,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
@@ -63,7 +62,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
|
||||
github.com/aws/smithy-go v1.25.1 // indirect
|
||||
github.com/aws/smithy-go v1.25.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
@@ -75,9 +74,10 @@ require (
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cognusion/imaging v1.0.2 // indirect
|
||||
@@ -85,7 +85,7 @@ require (
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/diskfs/go-diskfs v1.7.0 // indirect
|
||||
github.com/dromara/dongle v1.0.1 // indirect
|
||||
@@ -100,22 +100,22 @@ require (
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.5 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.9.0 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.8.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.6 // indirect
|
||||
github.com/go-openapi/errors v0.22.4 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-resty/resty/v2 v2.17.2 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.28.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
@@ -125,8 +125,8 @@ require (
|
||||
github.com/google/btree v1.1.3 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -134,7 +134,7 @@ require (
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 // indirect
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
@@ -143,14 +143,14 @@ require (
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jhump/protoreflect v1.18.0 // indirect
|
||||
github.com/jhump/protoreflect/v2 v2.0.0-beta.1 // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb // indirect
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.14.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.13.3 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
@@ -158,18 +158,18 @@ require (
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.22 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.5 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/parquet-go/bitpack v1.0.0 // indirect
|
||||
github.com/parquet-go/jsonlite v1.0.0 // indirect
|
||||
github.com/parquet-go/parquet-go v0.28.0 // indirect
|
||||
@@ -188,22 +188,23 @@ require (
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.2 // indirect
|
||||
github.com/rclone/rclone v1.74.1 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 // indirect
|
||||
github.com/rclone/rclone v1.73.5 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
|
||||
github.com/rdleal/intervalst v1.5.0 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/rfjakob/eme v1.2.0 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/samber/lo v1.52.0 // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.2 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/smarty/assertions v1.16.0 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
@@ -216,7 +217,7 @@ require (
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twpayne/go-geom v1.6.1 // indirect
|
||||
@@ -240,15 +241,15 @@ require (
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.9 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
|
||||
golang.org/x/image v0.39.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
|
||||
golang.org/x/image v0.38.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
@@ -256,8 +257,8 @@ require (
|
||||
golang.org/x/term v0.42.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/api v0.278.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
google.golang.org/api v0.274.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
@@ -266,10 +267,10 @@ require (
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
storj.io/picobuf v0.0.4 // indirect
|
||||
storj.io/uplink v1.14.0 // indirect
|
||||
storj.io/uplink v1.13.1 // indirect
|
||||
)
|
||||
|
||||
+101
-104
@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
|
||||
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
|
||||
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA=
|
||||
cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q=
|
||||
cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM=
|
||||
cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
@@ -49,8 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4 h1:tZh20RjgfMxKBxJiIS75iTVAKIUxrST5X2dVHMTptL4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.4/go.mod h1:vGYAk36rhMVCfTP7v+RVruCR0zmPe6S+36KRpDCLySw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
|
||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
@@ -61,12 +61,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82 h1:2RfP0d2QgkFH64BjZSWd59aMsc28IyWsUuHqU0txBtY=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.3.82/go.mod h1:IPk80dOmc7VFC0DJ85xMTPmre+8xoXX6kGHAkf5jRRw=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2 h1:mJ5QbLPOm4g5qhZiVB6wbSllfpeUExftGoyPek2hk4M=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.2/go.mod h1:ngpMgwkjur1VNUjqn11LPk3o5eCyOCRbcfg/0YAY7Hc=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38 h1:YCyHs3wUXEe2BEWn40vcoyaQ2ruHNmNwkasfo3Th16A=
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.38/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 h1:lMHTplAYI9FtmCo/QOcpRxmPA5REVAct1r2riQmDQKw=
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264/go.mod h1:wGqkOzRu/ClJibvDgcfuJNAqI2nLhe8g91tPlDKRCdE=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 h1:DUnYhvC4SoC8T84rx5omnhY3+xcQg/Whyoa3mDPIMkk=
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0/go.mod h1:Q3BYO6iDA2zweQPDGbNTtqft5tDcEpm6RTuqMlPcvbw=
|
||||
github.com/IBM/sarama v1.46.0 h1:+YTM1fNd6WKMchlnLKRUB5Z0qD4M8YbvwIIPLvJD53s=
|
||||
github.com/IBM/sarama v1.46.0/go.mod h1:0lOcuQziJ1/mBGHkdp5uYrltqQuKQKM5O5FOWUQVVvo=
|
||||
github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
|
||||
@@ -82,24 +82,22 @@ github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk=
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
|
||||
github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
|
||||
github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
|
||||
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
|
||||
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
|
||||
github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI=
|
||||
github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE=
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s=
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 h1:iLDOF0rdGTrol/q8OfPIIs5kLD8XvA2q75o6Uq/tgak=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0/go.mod h1:DrEWcQJjz7t5iF2duaiyhg4jyoF0kxOD6LtECNGkZ/Q=
|
||||
github.com/aalpar/deheap v1.1.2 h1:MABHLcnjqsffb8GLkUFDigqpBBxOMz0DoKM9QfELeTw=
|
||||
github.com/aalpar/deheap v1.1.2/go.mod h1:A+nfkD4JbS05sewV0he/MYgR/90vfqyMoNNROgs+rmA=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 h1:hhdWprfSpFbN7lz3W1gM40vOgvSh1WCSMxYD6gGB4Hs=
|
||||
github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3/go.mod h1:XaUnRxSCYgL3kkgX0QHIV0D+znljPIDImxlv2kbGv0Y=
|
||||
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
|
||||
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
|
||||
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
|
||||
github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ=
|
||||
github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
|
||||
github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
|
||||
@@ -124,8 +122,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8TH
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13 h1:uMC4oL6G3MNhodo358QEqSDjrgvzV3TUQ58nyQSGq2E=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.13/go.mod h1:Cer86AE2686DvVUe57LPve3jUBmbujuaonSX8pNzGgw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1 h1:IbWiN670htmBioc+Zj32vSpJgQ2+OYSlvTvfQ1nCORQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.1/go.mod h1:tw/B596EUhBWDFGdDGuLC21fVU4A3s4/5Efy8S39W18=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
|
||||
@@ -152,8 +150,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6f
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
|
||||
github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI=
|
||||
github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U=
|
||||
github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
@@ -189,13 +187,15 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
|
||||
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
|
||||
github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
|
||||
github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0 h1:iLoIwb7BJECHTbNcmIhYDsQhoZiACWGNvEpyqQy97Dk=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.15.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 h1:ugiQwb7DwpWQnete2AZkTh94MonZKmxD7hDGy1qTzDs=
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg=
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=
|
||||
@@ -213,9 +213,8 @@ github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5z
|
||||
github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU=
|
||||
github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
|
||||
github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
|
||||
github.com/cronokirby/saferith v0.33.0 h1:TgoQlfsD4LIwx71+ChfRcIpjkw+RPOapDEVxa+LhwLo=
|
||||
github.com/cronokirby/saferith v0.33.0/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 h1:tLWBZgPg6TV67oe76W4p+aUQEWIa52wbcuiz8GFd3vo=
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
|
||||
github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
|
||||
github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -268,8 +267,8 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E=
|
||||
@@ -282,8 +281,8 @@ github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7N
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA=
|
||||
github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw=
|
||||
github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0=
|
||||
github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@@ -297,8 +296,8 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-openapi/errors v0.22.6 h1:eDxcf89O8odEnohIXwEjY1IB4ph5vmbUsBMsFNwXWPo=
|
||||
github.com/go-openapi/errors v0.22.6/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/errors v0.22.4 h1:oi2K9mHTOb5DPW2Zjdzs/NIvwi2N3fARKaTJLdNabaM=
|
||||
github.com/go-openapi/errors v0.22.4/go.mod h1:z9S8ASTUqx7+CP1Q8dD8ewGH/1JWFFLX/2PmAYNQLgk=
|
||||
github.com/go-openapi/strfmt v0.25.0 h1:7R0RX7mbKLa9EYCTHRcCuIPcaqlyQiWNPTXwClK0saQ=
|
||||
github.com/go-openapi/strfmt v0.25.0/go.mod h1:nNXct7OzbwrMY9+5tLX4I21pzcmE6ccMGXl3jFdPfn8=
|
||||
github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls=
|
||||
@@ -309,14 +308,14 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
|
||||
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
|
||||
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
|
||||
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
|
||||
github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
|
||||
github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
|
||||
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
|
||||
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
|
||||
@@ -388,12 +387,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE=
|
||||
github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA=
|
||||
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
|
||||
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
@@ -428,8 +427,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160 h1:PV6ipOJN0JIek4dqPqsTtenQmjI1SZntCUgPzhfk9gM=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260331173834-036f908d0160/go.mod h1:yRuPDnHgGmsbvopF0amMqXxr4n32GynzX5GTwFYDHaw=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd h1:dSIuz2mpJAPQfhHYtG57D0qwSkgC/vQ69gHfeyQ4kxA=
|
||||
github.com/internxt/rclone-adapter v0.0.0-20260220172730-613f4cc8b8fd/go.mod h1:vdPya4AIcDjvng4ViaAzqjegJf0VHYpYHQguFx5xBp0=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
|
||||
@@ -446,8 +445,8 @@ github.com/jhump/protoreflect v1.18.0 h1:TOz0MSR/0JOZ5kECB/0ufGnC2jdsgZ123Rd/k4Z
|
||||
github.com/jhump/protoreflect v1.18.0/go.mod h1:ezWcltJIVF4zYdIFM+D/sHV4Oh5LNU08ORzCGfwvTz8=
|
||||
github.com/jhump/protoreflect/v2 v2.0.0-beta.1 h1:Dw1rslK/VotaUGYsv53XVWITr+5RCPXfvvlGrM/+B6w=
|
||||
github.com/jhump/protoreflect/v2 v2.0.0-beta.1/go.mod h1:D9LBEowZyv8/iSu97FU2zmXG3JxVTmNw21mu63niFzU=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb h1:6vkM8gO+zFV2m21QzGYyUSq5TP0VQgP2Xz3UQyCN2kI=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb/go.mod h1:H1+whwD0Qe3YOunlXIWhh3rlvzW5cZfkMDYGQPg+KAM=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 h1:ZxO6Qr2GOXPdcW80Mcn3nemvilMPvpWqxrNfK2ZnNNs=
|
||||
github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3/go.mod h1:dvLUr/8Fs9a2OBrEnCC5duphbkz/k/mSy5OkXg3PAgI=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
@@ -470,12 +469,12 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
|
||||
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/reedsolomon v1.14.0 h1:5YSZeclzSYg5nl349+GDG/agDtQ6MZiwUYXvVKN1Jx0=
|
||||
github.com/klauspost/reedsolomon v1.14.0/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
|
||||
github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
|
||||
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
|
||||
@@ -500,8 +499,8 @@ github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolM
|
||||
github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
|
||||
github.com/lpar/date v1.0.0 h1:bq/zVqFTUmsxvd/CylidY4Udqpr9BOFrParoP6p0x/I=
|
||||
github.com/lpar/date v1.0.0/go.mod h1:KjYe0dDyMQTgpqcUz4LEIeM5VZwhggjVx/V2dtc8NSo=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM=
|
||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k=
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||
github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8=
|
||||
github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
@@ -509,8 +508,8 @@ github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stg
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk=
|
||||
github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg=
|
||||
@@ -536,12 +535,12 @@ github.com/onsi/ginkgo/v2 v2.23.3/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGm
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y=
|
||||
github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0 h1:eDkWg6ZN0uKwWzSekoFcQJhR+C+F/aVdTwr+lGHU9Qk=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.111.0/go.mod h1:8ZzvzuEG/cFLFZhxg/Mg1w19KqyXBKO3c17QIc5PkGs=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 h1:l9awEvzWvxmYhy/97A0hZ87pa7BncYXmcO/S8+rvgK0=
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0/go.mod h1:oB8jFGVc/7/zJ+DbleE8MzGHjhs2ioCz5stRTdZdIcY=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
||||
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
|
||||
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
|
||||
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
|
||||
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
|
||||
github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
|
||||
@@ -581,30 +580,30 @@ github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UH
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8=
|
||||
github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw=
|
||||
github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3 h1:Bs7RC4xCFSN0BPIYVda/BNxp0qo3NV0gB2VZqx2KIew=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.3/go.mod h1:26RAest751Ofk+F/d8xtl4UyWXrZvMQwn39U8rm/WKM=
|
||||
github.com/rclone/go-proton-api v1.0.2 h1:cJtJUab0MGJ3C6q5kiEJs3pbyhSLnOKMyYOQehA0PBc=
|
||||
github.com/rclone/go-proton-api v1.0.2/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.74.1 h1:QQI//tl4O5uooTowdKwNKcv5qCQvJ3LIBG00+TKeXw0=
|
||||
github.com/rclone/rclone v1.74.1/go.mod h1:44epaKf5w/1ZQounefr76wx4+y9NZmgfMKle9T3pzj0=
|
||||
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11 h1:4MI2alxM/Ye2gIRBlYf28JGWTipZ4Zz7yAziPKrttjs=
|
||||
github.com/rclone/Proton-API-Bridge v1.0.1-0.20260127174007-77f974840d11/go.mod h1:3HLX7dwZgvB7nt+Yl/xdzVPcargQ1yBmJEUg3n+jMKM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18 h1:Lc+d3ISfQaMJKWZOE7z4ZSY4RVmdzbn1B0IM8xN18qM=
|
||||
github.com/rclone/go-proton-api v1.0.1-0.20260127173028-eb465cac3b18/go.mod h1:LB2kCEaZMzNn3ocdz+qYfxXmuLxxN0ka62KJd2x53Bc=
|
||||
github.com/rclone/rclone v1.73.5 h1:r8a9JHYIWUqk7hNRJuMJ3cROkKfB2zmfkADg8ZLYh6I=
|
||||
github.com/rclone/rclone v1.73.5/go.mod h1:WVv8gvA/lEl/Y37e8I8yosm7ZY+Szq7ujXbJS8Ol63o=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
|
||||
github.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
|
||||
github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=
|
||||
github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
@@ -619,11 +618,11 @@ github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9
|
||||
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0=
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA=
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTCzfY=
|
||||
@@ -642,8 +641,8 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
@@ -671,8 +670,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c h1:dtcOwRimeiBFrlutmF6K94l0rxYFARNFMA+lSQ41C+M=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251120131202-6845944c051c/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 h1:rrGZv6xYk37hx0tW2sYfgbO0PqStbHqz6Bq6oc9Hurg=
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491/go.mod h1:ykucQyiE9Q2qx1wLlEtZkkNn1IURib/2O+Mvd25i1Fo=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
@@ -738,8 +737,6 @@ github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
|
||||
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
|
||||
github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
|
||||
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
|
||||
github.com/zeebo/mwc v0.0.7 h1:0NerGhCww6ZQx+/xCx5iwznftveokvto1KILpYfENZk=
|
||||
github.com/zeebo/mwc v0.0.7/go.mod h1:0B32or6moOig1YGuqMoimBpU9QK9uYaGG2bBOuddqtE=
|
||||
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
|
||||
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
|
||||
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
||||
@@ -755,8 +752,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
@@ -802,12 +799,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
|
||||
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
|
||||
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
|
||||
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -1033,8 +1030,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
|
||||
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -1057,8 +1054,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
|
||||
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
|
||||
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E=
|
||||
google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ=
|
||||
google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA=
|
||||
google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -1094,12 +1091,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE=
|
||||
google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -1112,8 +1109,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
|
||||
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
|
||||
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6 h1:ExN12ndbJ608cboPYflpTny6mXSzPrDLh0iTaVrRrds=
|
||||
google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0 h1:/KQ7VP/1bs53/aopk9QhuPyFAp9Dm9Ejix3lzYkCrDA=
|
||||
@@ -1164,8 +1161,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
|
||||
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a h1:7gSBQY3vhQMIqi3vfaEXR7mreRjcBLfVsYY0rHIN7P0=
|
||||
storj.io/common v0.0.0-20260225132117-99155641c30a/go.mod h1:kyxwKwlfH4paBZCZt/szvRB770ieAIJF73iDy2DpEHw=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c h1:UDXSrdeLJe3QFouavSW10fYdpclK0YNu3KvQHzqq2+k=
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c/go.mod h1:XNX7uykja6aco92y2y8RuqaXIDRPpt1YA2OQDKlKEUk=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 h1:8OE12DvUnB9lfZcHe7IDGsuhjrY9GBAr964PVHmhsro=
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55/go.mod h1:Y9LZaa8esL1PW2IDMqJE7CFSNq7d5bQ3RI7mGPtmKMg=
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 h1:5MZ0CyMbG6Pi0rRzUWVG6dvpXjbBYEX2oyXuj+tT+sk=
|
||||
@@ -1174,5 +1171,5 @@ storj.io/infectious v0.0.2 h1:rGIdDC/6gNYAStsxsZU79D/MqFjNyJc1tsyyj9sTl7Q=
|
||||
storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs=
|
||||
storj.io/picobuf v0.0.4 h1:qswHDla+YZ2TovGtMnU4astjvrADSIz84FXRn0qgP6o=
|
||||
storj.io/picobuf v0.0.4/go.mod h1:hSMxmZc58MS/2qSLy1I0idovlO7+6K47wIGUyRZa6mg=
|
||||
storj.io/uplink v1.14.0 h1:J1yXlt0aRr6kgLTHWXOWosNCFVfbamlcyd+CSxyIczo=
|
||||
storj.io/uplink v1.14.0/go.mod h1:2ysmjzd/1Xtz4VKoErNcSqBQz3UC9WKTVuLMV1cNu6E=
|
||||
storj.io/uplink v1.13.1 h1:C8RdW/upALoCyuF16Lod9XGCXEdbJAS+ABQy9JO/0pA=
|
||||
storj.io/uplink v1.13.1/go.mod h1:x0MQr4UfFsQBwgVWZAtEsLpuwAn6dg7G0Mpne1r516E=
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -292,60 +291,6 @@ func (v *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_serv
|
||||
return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
|
||||
if req == nil {
|
||||
return nil, fmt.Errorf("VolumeEcShardsInfo request is nil")
|
||||
}
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
|
||||
// Report whichever shards exist on disk: seeded or mounted. Collection
|
||||
// comes from the matching mount request when one exists.
|
||||
collectionByShard := make(map[uint32]string)
|
||||
for _, mr := range v.mountRequests {
|
||||
if mr == nil || mr.VolumeId != req.VolumeId {
|
||||
continue
|
||||
}
|
||||
for _, shardId := range mr.ShardIds {
|
||||
if _, ok := collectionByShard[shardId]; !ok {
|
||||
collectionByShard[shardId] = mr.Collection
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resp := &volume_server_pb.VolumeEcShardsInfoResponse{}
|
||||
prefix := fmt.Sprintf("%d.ec", req.VolumeId)
|
||||
entries, _ := os.ReadDir(v.baseDir)
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
continue
|
||||
}
|
||||
suffix := strings.TrimPrefix(name, prefix)
|
||||
if len(suffix) < 2 {
|
||||
continue
|
||||
}
|
||||
var shardId uint32
|
||||
if _, err := fmt.Sscanf(suffix[:2], "%d", &shardId); err != nil {
|
||||
continue
|
||||
}
|
||||
var size int64
|
||||
if info, err := entry.Info(); err == nil {
|
||||
size = info.Size()
|
||||
}
|
||||
resp.EcShardInfos = append(resp.EcShardInfos, &volume_server_pb.EcShardInfo{
|
||||
ShardId: shardId,
|
||||
Size: size,
|
||||
Collection: collectionByShard[shardId],
|
||||
VolumeId: req.VolumeId,
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.deleteRequests = append(v.deleteRequests, req)
|
||||
|
||||
@@ -74,33 +74,25 @@ func TestAssumeRoleWithWebIdentityValidation(t *testing.T) {
|
||||
assert.Equal(t, "MissingParameter", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("missing_role_arn_invalid_jwt_still_rejected", func(t *testing.T) {
|
||||
// Missing RoleArn is no longer a fast-fail at the HTTP layer:
|
||||
// claim-based policy mode (Phase 3b) advertises RoleArn as
|
||||
// optional so the STS service can derive the assumed-role ARN
|
||||
// from the configured policy claim. Validation now happens at
|
||||
// the STS layer once the JWT is parsed. With a bogus token the
|
||||
// JWT parse fails first, so the request is still rejected —
|
||||
// just with the JWT-parse error code instead of MissingParameter.
|
||||
t.Run("missing_role_arn", func(t *testing.T) {
|
||||
resp, err := callSTSAPI(t, url.Values{
|
||||
"Action": {"AssumeRoleWithWebIdentity"},
|
||||
"WebIdentityToken": {"fake-jwt-token"},
|
||||
"RoleSessionName": {"test-session"},
|
||||
// RoleArn omitted on purpose.
|
||||
// RoleArn is missing
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
|
||||
"Should still fail when RoleArn is missing and the JWT is invalid")
|
||||
"Should fail without RoleArn")
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
var errResp STSErrorTestResponse
|
||||
err = xml.Unmarshal(body, &errResp)
|
||||
require.NoError(t, err, "Failed to parse error response: %s", string(body))
|
||||
assert.NotEqual(t, "MissingParameter", errResp.Error.Code,
|
||||
"missing RoleArn alone must no longer surface as MissingParameter")
|
||||
assert.Equal(t, "MissingParameter", errResp.Error.Code)
|
||||
})
|
||||
|
||||
t.Run("missing_role_session_name", func(t *testing.T) {
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
# S3 Lifecycle Test Makefile
|
||||
# End-to-end test of the event-driven lifecycle worker, driven by
|
||||
# the s3.lifecycle.run-shard shell command.
|
||||
|
||||
.PHONY: help build-weed start-server stop-server test test-with-server clean health-check
|
||||
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
S3_PORT := 8333
|
||||
MASTER_PORT := 9333
|
||||
VOLUME_PORT := 8080
|
||||
FILER_PORT := 8888
|
||||
ACCESS_KEY ?= some_access_key1
|
||||
SECRET_KEY ?= some_secret_key1
|
||||
TEST_TIMEOUT := 10m
|
||||
TEST_PATTERN := TestLifecycle
|
||||
SERVER_DIR := ./test-volume-data/server-data
|
||||
|
||||
help:
|
||||
@echo "S3 Lifecycle Test Makefile"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@echo " build-weed - build the SeaweedFS binary"
|
||||
@echo " start-server - start a local 'weed mini' cluster"
|
||||
@echo " stop-server - stop the cluster"
|
||||
@echo " test - run tests against an already-running cluster"
|
||||
@echo " test-with-server - start, run tests, stop"
|
||||
@echo " health-check - is the S3 endpoint up?"
|
||||
@echo " clean - remove test artifacts"
|
||||
|
||||
build-weed:
|
||||
@echo "Building SeaweedFS binary..."
|
||||
@cd ../../../weed && go build -o weed_binary .
|
||||
@chmod +x $(WEED_BINARY)
|
||||
|
||||
start-server: build-weed
|
||||
@echo "Starting weed mini..."
|
||||
@rm -f weed-server.pid
|
||||
@mkdir -p $(SERVER_DIR)
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
|
||||
-dir=$(SERVER_DIR) \
|
||||
-s3.port=$(S3_PORT) \
|
||||
> weed-test.log 2>&1 & \
|
||||
echo $$! > weed-server.pid
|
||||
@for i in $$(seq 1 90); do \
|
||||
if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
|
||||
echo "server up after $$i s"; exit 0; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "server did not start within 90s"; \
|
||||
tail -50 weed-test.log; \
|
||||
exit 1
|
||||
|
||||
stop-server:
|
||||
@if [ -f weed-server.pid ]; then \
|
||||
PID=$$(cat weed-server.pid); \
|
||||
kill -TERM $$PID 2>/dev/null || true; \
|
||||
sleep 1; \
|
||||
kill -KILL $$PID 2>/dev/null || true; \
|
||||
rm -f weed-server.pid; \
|
||||
fi
|
||||
|
||||
health-check:
|
||||
@curl -s http://localhost:$(S3_PORT) >/dev/null && echo "S3 up on $(S3_PORT)" || (echo "S3 not reachable"; exit 1)
|
||||
|
||||
test:
|
||||
@WEED_BINARY=$$(cd ../../../weed && pwd)/weed_binary \
|
||||
S3_ENDPOINT=http://localhost:$(S3_PORT) \
|
||||
S3_GRPC_ENDPOINT=localhost:$$(($(S3_PORT) + 10000)) \
|
||||
MASTER_ENDPOINT=http://localhost:$(MASTER_PORT) \
|
||||
FILER_GRPC_ADDRESS=localhost:$$(($(FILER_PORT) + 10000)) \
|
||||
go test -v -timeout $(TEST_TIMEOUT) -run $(TEST_PATTERN)
|
||||
|
||||
test-with-server: start-server
|
||||
@$(MAKE) test || (RC=$$?; $(MAKE) stop-server; exit $$RC)
|
||||
@$(MAKE) stop-server
|
||||
|
||||
clean: stop-server
|
||||
@rm -rf $(SERVER_DIR) test-volume-data
|
||||
@rm -f weed-test.log weed-server.pid
|
||||
@@ -1,38 +0,0 @@
|
||||
# S3 Lifecycle Integration Tests
|
||||
|
||||
End-to-end test of the event-driven S3 lifecycle worker, exercised through
|
||||
the `s3.lifecycle.run-shard` shell command.
|
||||
|
||||
## Why backdate mtimes?
|
||||
|
||||
The S3 API rejects `Expiration.Days < 1`, so a literal "wait one day"
|
||||
integration test isn't workable. Each test sets up a 1-day expiration rule,
|
||||
puts the target object, then rewrites its filer entry's `Mtime` to ~30 days
|
||||
ago via `filer_pb.UpdateEntry`. From the engine's perspective the object is
|
||||
past its expiration window the moment the shell command starts.
|
||||
|
||||
## Running
|
||||
|
||||
```sh
|
||||
# build the binary, start a local mini cluster, run tests, stop it
|
||||
make test-with-server
|
||||
|
||||
# or, if a cluster is already running on the default ports
|
||||
make test
|
||||
```
|
||||
|
||||
The test runs the shell command once with `-shards 0-15` (one filer
|
||||
subscription covering all 16 shards) rather than computing the target
|
||||
object's shard up front. This keeps the test independent of the
|
||||
`ShardID(bucket, key)` hash function — only that some shard reaches the
|
||||
deletion within the polling window.
|
||||
|
||||
## Environment
|
||||
|
||||
| variable | default | description |
|
||||
|----------------------|--------------------------|----------------------------|
|
||||
| `WEED_BINARY` | _required_ | path to `weed_binary` |
|
||||
| `S3_ENDPOINT` | `http://localhost:8333` | S3 API URL |
|
||||
| `S3_GRPC_ENDPOINT` | `localhost:18333` | S3 gRPC for lifecycle dispatch |
|
||||
| `MASTER_ENDPOINT` | `http://localhost:9333` | master HTTP |
|
||||
| `FILER_GRPC_ADDRESS` | `localhost:18888` | filer gRPC for `UpdateEntry` |
|
||||
@@ -1,79 +0,0 @@
|
||||
// Bootstrap-walk integration scenario: lifecycle config added AFTER the
|
||||
// objects exist must still expire them on the first sweep.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleBootstrapWalkOnExistingObjects: PUT a batch of objects
|
||||
// FIRST, backdate them, THEN configure the lifecycle rule, THEN run
|
||||
// the worker. The worker must discover the pre-existing objects via
|
||||
// the bootstrap walk (BucketBootstrapper) — there were no meta-log
|
||||
// events the reader could have observed because the objects predate
|
||||
// the rule.
|
||||
//
|
||||
// Production scenario: an operator enables lifecycle on a bucket that
|
||||
// already holds a million objects from before the policy. Without a
|
||||
// bootstrap walk, only NEW writes would ever match; the existing
|
||||
// content would never expire.
|
||||
func TestLifecycleBootstrapWalkOnExistingObjects(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("bootstrap")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
|
||||
// 5 objects so the test exercises the walker iterating multiple
|
||||
// entries in the same bucket directory. PUT FIRST, before any
|
||||
// lifecycle config is set, so the meta-log doesn't carry events
|
||||
// that match any rule.
|
||||
const total = 5
|
||||
keys := make([]string, total)
|
||||
for i := 0; i < total; i++ {
|
||||
keys[i] = fmt.Sprintf("preexisting/obj-%02d.txt", i)
|
||||
putObject(t, c, bucket, keys[i], "content")
|
||||
backdateMtime(t, fc, bucket, keys[i], 30)
|
||||
}
|
||||
|
||||
// One out-of-prefix object that must NOT be expired. Same
|
||||
// pre-existing PUT timing.
|
||||
const survivor = "other/keep.txt"
|
||||
putObject(t, c, bucket, survivor, "keep")
|
||||
backdateMtime(t, fc, bucket, survivor, 30)
|
||||
|
||||
// NOW set the rule. Up to this point the worker (if it were
|
||||
// running event-driven only) would have nothing matching to do.
|
||||
putExpirationLifecycle(t, c, bucket, "preexisting/", 1)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// Every pre-existing in-prefix object must be expired by the
|
||||
// bootstrap walk's discovery + dispatch.
|
||||
for _, k := range keys {
|
||||
k := k
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(k),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond,
|
||||
"pre-existing %s/%s must be discovered + expired by bootstrap walk", bucket, k)
|
||||
}
|
||||
|
||||
// Out-of-prefix object stays — pin that the bootstrap walk
|
||||
// doesn't ignore the rule's prefix filter.
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(survivor),
|
||||
})
|
||||
require.NoError(t, err, "out-of-prefix object must survive the bootstrap walk")
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
// Lifecycle config update across sweeps.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleConfigUpdateBetweenSweeps: an operator changes the
|
||||
// lifecycle rule between two shell-driven sweeps. The second sweep
|
||||
// must respect the NEW rule, not a cached version of the old one.
|
||||
//
|
||||
// Each `runLifecycleShard` invocation spawns a fresh `weed shell`
|
||||
// subprocess, so cached engine state from a previous sweep doesn't
|
||||
// persist across runs. This test pins that the freshly-loaded config
|
||||
// actually changes routing — under the new prefix only matching
|
||||
// objects expire, even if there are still backdated objects sitting
|
||||
// under the old prefix.
|
||||
func TestLifecycleConfigUpdateBetweenSweeps(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("config-update")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
|
||||
// Sweep 1: rule expires anything under "first/".
|
||||
putExpirationLifecycle(t, c, bucket, "first/", 1)
|
||||
|
||||
const firstKey = "first/initial.txt"
|
||||
putObject(t, c, bucket, firstKey, "first")
|
||||
backdateMtime(t, fc, bucket, firstKey, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("sweep 1 output:\n%s", out)
|
||||
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(firstKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "sweep 1 must expire %s", firstKey)
|
||||
|
||||
// Update the rule to a different prefix. The old "first/" prefix
|
||||
// is no longer covered by any rule; objects under it must NOT be
|
||||
// expired by sweep 2 even when backdated.
|
||||
putExpirationLifecycle(t, c, bucket, "second/", 1)
|
||||
|
||||
const secondKey = "second/new.txt"
|
||||
const oldPrefixKey = "first/post-update.txt"
|
||||
putObject(t, c, bucket, secondKey, "second")
|
||||
putObject(t, c, bucket, oldPrefixKey, "stale rule")
|
||||
backdateMtime(t, fc, bucket, secondKey, 30)
|
||||
backdateMtime(t, fc, bucket, oldPrefixKey, 30)
|
||||
|
||||
out = runLifecycleShard(t)
|
||||
t.Logf("sweep 2 output:\n%s", out)
|
||||
|
||||
// Sweep 2 expires the new-prefix object.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(secondKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "sweep 2 must expire %s under the new rule", secondKey)
|
||||
|
||||
// Sweep 2 must NOT expire the old-prefix object — the rule was
|
||||
// replaced, not merged. A regression that caches old rules across
|
||||
// PutBucketLifecycleConfiguration calls would fail here.
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(oldPrefixKey),
|
||||
})
|
||||
require.NoError(t, err, "old-prefix object must survive after rule update — config replacement, not merge")
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
// Operator-removes-lifecycle-config integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleDeleteBucketLifecycleStopsDispatching: when an operator
|
||||
// runs DeleteBucketLifecycle, subsequent sweeps must NOT continue
|
||||
// expiring objects under the now-removed rule. The worker reads each
|
||||
// bucket's XML config from filer state on every run; if it cached the
|
||||
// previous config across PutBucketLifecycleConfiguration → DeleteBucket
|
||||
// Lifecycle the bucket would keep losing objects.
|
||||
//
|
||||
// Setup: positive control (rule active, backdated obj expires), then
|
||||
// DeleteBucketLifecycle, then PUT + backdate a fresh object, run the
|
||||
// worker again. The fresh object must remain — no rule, no dispatch.
|
||||
func TestLifecycleDeleteBucketLifecycleStopsDispatching(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("delete-config")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putExpirationLifecycle(t, c, bucket, "x/", 1)
|
||||
|
||||
const firstKey = "x/initial.txt"
|
||||
putObject(t, c, bucket, firstKey, "first")
|
||||
backdateMtime(t, fc, bucket, firstKey, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("sweep 1 output (rule active):\n%s", out)
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(firstKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "positive control: %s should expire while rule is active", firstKey)
|
||||
|
||||
// Now remove the rule. The worker must observe the removal on the
|
||||
// next sweep.
|
||||
_, err := c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const survivorKey = "x/post-delete.txt"
|
||||
putObject(t, c, bucket, survivorKey, "should-stay")
|
||||
backdateMtime(t, fc, bucket, survivorKey, 30)
|
||||
|
||||
out = runLifecycleShard(t)
|
||||
t.Logf("sweep 2 output (rule deleted):\n%s", out)
|
||||
|
||||
// Wait the same window; survivor MUST still exist after the worker
|
||||
// has had every opportunity to (incorrectly) act on it.
|
||||
time.Sleep(2 * time.Second)
|
||||
_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(survivorKey),
|
||||
})
|
||||
require.NoError(t, err, "after DeleteBucketLifecycle, sweeps must not expire %s", survivorKey)
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
// Empty bucket sweep — no-op safety integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleEmptyBucketSweepIsNoOp: a bucket carrying lifecycle
|
||||
// config but no objects must produce a successful sweep — no hangs,
|
||||
// no errors, no dispatches. Pinned because the bootstrap walker
|
||||
// iterates bucket directories, and an empty directory is a corner of
|
||||
// that traversal that's easy to break (e.g. a slice-bounds bug on
|
||||
// the first listing returning zero entries).
|
||||
func TestLifecycleEmptyBucketSweepIsNoOp(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
_, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("empty")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putExpirationLifecycle(t, c, bucket, "anywhere/", 1)
|
||||
|
||||
// No PUTs. The bucket is empty.
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("sweep output:\n%s", out)
|
||||
|
||||
// Sanity: the worker must report it loaded the bucket's config and
|
||||
// completed without errors. The runLifecycleShard helper already
|
||||
// fails on FATAL; pin the success markers here too so a regression
|
||||
// that produces a half-shaped output (no completion) is caught.
|
||||
require.Contains(t, out, "loaded lifecycle for", "worker must report config load")
|
||||
require.Contains(t, out, "shards 0-15 complete", "worker must complete all shards")
|
||||
require.False(t, strings.Contains(out, "FATAL"), "worker must not produce FATAL output")
|
||||
|
||||
// Bucket still exists after the sweep — the worker doesn't drop
|
||||
// empty buckets as a side effect.
|
||||
_, err := c.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: aws.String(bucket)})
|
||||
require.NoError(t, err, "empty bucket must still exist after sweep")
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
// ExpirationDate (date-based, not Days) integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleExpirationDateInThePast: an Expiration{Date: <past>} rule
|
||||
// routes through the engine's ScanAtDate mode rather than the
|
||||
// EventDriven delay-group path. The worker's dispatcher must still fire
|
||||
// when the date has already passed at the time the worker runs. Pinning
|
||||
// this path because most tests use Days-based rules; ScanAtDate is a
|
||||
// separate compile + dispatch branch (engine.decideMode case
|
||||
// ActionKindExpirationDate) that wouldn't be exercised otherwise.
|
||||
func TestLifecycleExpirationDateInThePast(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("expdate")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
|
||||
// Date in the past: AWS rejects ExpirationDate in the future from
|
||||
// being processed early but a past date is the natural integration
|
||||
// test — every object hit by the rule is immediately eligible.
|
||||
pastDate := time.Now().Add(-7 * 24 * time.Hour).UTC().Truncate(24 * time.Hour)
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{{
|
||||
ID: aws.String("expire-by-date"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String("d/")},
|
||||
Expiration: &types.LifecycleExpiration{
|
||||
Date: aws.Time(pastDate),
|
||||
},
|
||||
}},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const oldKey = "d/in-prefix.txt"
|
||||
const otherKey = "other/skip.txt"
|
||||
putObject(t, c, bucket, oldKey, "old")
|
||||
putObject(t, c, bucket, otherKey, "other")
|
||||
|
||||
// Backdate oldKey too — defense-in-depth so the test doesn't
|
||||
// depend on whether the worker also considers MinTriggerAge for
|
||||
// date kinds; a fresh object with a past date should still expire,
|
||||
// but aging it locks the assertion either way.
|
||||
backdateMtime(t, fc, bucket, oldKey, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(oldKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "%s/%s must be expired by past-date rule", bucket, oldKey)
|
||||
|
||||
// Out-of-prefix object stays — the rule's Filter.Prefix gates this.
|
||||
_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(otherKey),
|
||||
})
|
||||
require.NoError(t, err, "object outside the rule's prefix must remain")
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
// AbortIncompleteMultipartUpload integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putAbortMPULifecycle wires AbortIncompleteMultipartUpload on a prefix.
|
||||
// AWS rejects DaysAfterInitiation < 1, so the test backdates the upload
|
||||
// directory to age it past the threshold.
|
||||
func putAbortMPULifecycle(t *testing.T, c *s3.Client, bucket, prefix string, days int32) {
|
||||
t.Helper()
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("abort-stale-mpu"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
|
||||
AbortIncompleteMultipartUpload: &types.AbortIncompleteMultipartUpload{
|
||||
DaysAfterInitiation: aws.Int32(days),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// backdateUploadDir ages the .uploads/<uploadID>/ directory entry, which is
|
||||
// what the lifecycle worker keys ABORT_MPU off of (the upload's init-time
|
||||
// directory carries the destination key in Extended).
|
||||
func backdateUploadDir(t *testing.T, fc filer_pb.SeaweedFilerClient, bucket, uploadID string, daysOld int) {
|
||||
t.Helper()
|
||||
dir := bucketsPath + "/" + bucket + "/.uploads"
|
||||
resp, err := fc.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir, Name: uploadID,
|
||||
})
|
||||
require.NoError(t, err, "lookup .uploads/%s", uploadID)
|
||||
require.NotNil(t, resp.Entry)
|
||||
require.NotNil(t, resp.Entry.Attributes)
|
||||
require.True(t, resp.Entry.IsDirectory, ".uploads/%s should be a directory", uploadID)
|
||||
|
||||
resp.Entry.Attributes.Mtime = time.Now().Add(-time.Duration(daysOld) * 24 * time.Hour).Unix()
|
||||
resp.Entry.Attributes.MtimeNs = 0
|
||||
_, err = fc.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir, Entry: resp.Entry,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestLifecycleAbortIncompleteMultipartUpload: an MPU left incomplete past
|
||||
// the AbortIncompleteMultipartUpload.DaysAfterInitiation threshold must be
|
||||
// aborted by the lifecycle worker. Exercises the lifecycleAbortMPU handler
|
||||
// path which is otherwise unreachable from the prefix-based expiration
|
||||
// tests (the routing keys off of `.uploads/<id>/` directory events, not
|
||||
// regular object events).
|
||||
func TestLifecycleAbortIncompleteMultipartUpload(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("abort-mpu")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putAbortMPULifecycle(t, c, bucket, "uploads/", 1)
|
||||
|
||||
// Initiate an MPU, upload a single part, then leave it. CompleteMultipart
|
||||
// would consume the upload; AbortMultipart would explicitly abort. We do
|
||||
// neither so the lifecycle worker is the path that ends it.
|
||||
const key = "uploads/big.bin"
|
||||
createOut, err := c.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
uploadID := aws.ToString(createOut.UploadId)
|
||||
require.NotEmpty(t, uploadID)
|
||||
|
||||
// Upload one part so the directory carries the right shape.
|
||||
_, err = c.UploadPart(context.Background(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: strings.NewReader("part 1 contents"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Sanity: ListMultipartUploads sees our pending upload before the
|
||||
// worker runs.
|
||||
listOut, err := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
var foundBefore bool
|
||||
for _, u := range listOut.Uploads {
|
||||
if aws.ToString(u.UploadId) == uploadID {
|
||||
foundBefore = true
|
||||
break
|
||||
}
|
||||
}
|
||||
require.True(t, foundBefore, "upload %s should be visible before the worker runs", uploadID)
|
||||
|
||||
// Backdate the upload directory entry so the rule's DaysAfterInitiation
|
||||
// threshold is satisfied. The router walks .uploads/ and emits an
|
||||
// MPU-init event for each entry.
|
||||
backdateUploadDir(t, fc, bucket, uploadID, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// The upload must no longer appear in ListMultipartUploads.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
listOut, err := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, u := range listOut.Uploads {
|
||||
if aws.ToString(u.UploadId) == uploadID {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, 30*time.Second, 500*time.Millisecond, "upload %s must be aborted by the lifecycle worker", uploadID)
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
// Multiple-bucket integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// isS3NotFound recognizes a NotFound (NoSuchKey/404) response from the
|
||||
// AWS SDK. Treating any HeadObject error as "deleted" lets a transport
|
||||
// failure or dead endpoint mask a real bug, so callers that need to
|
||||
// prove deletion specifically should use this.
|
||||
func isS3NotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var nsk *types.NoSuchKey
|
||||
if errors.As(err, &nsk) {
|
||||
return true
|
||||
}
|
||||
var notFound *types.NotFound
|
||||
if errors.As(err, ¬Found) {
|
||||
return true
|
||||
}
|
||||
var apiErr *smithyhttp.ResponseError
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.HTTPStatusCode() == 404
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestLifecycleMultipleBucketsInOneSweep: a single shell-driven shard
|
||||
// sweep must process every bucket carrying lifecycle config, not just
|
||||
// the first one alphabetically. Pinned because the scheduler iterates
|
||||
// the buckets directory and a regression that returns early after the
|
||||
// first match would silently disable lifecycle for every later bucket.
|
||||
//
|
||||
// Two buckets, each with its own 1-day prefix-expiration rule and one
|
||||
// backdated object. After the worker runs, both objects must be gone.
|
||||
func TestLifecycleMultipleBucketsInOneSweep(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucketA := uniqueBucket("multi-a")
|
||||
bucketB := uniqueBucket("multi-b")
|
||||
mustCreateBucket(t, c, bucketA)
|
||||
mustCreateBucket(t, c, bucketB)
|
||||
|
||||
putExpirationLifecycle(t, c, bucketA, "exp/", 1)
|
||||
putExpirationLifecycle(t, c, bucketB, "exp/", 1)
|
||||
|
||||
const keyA = "exp/a.txt"
|
||||
const keyB = "exp/b.txt"
|
||||
putObject(t, c, bucketA, keyA, "a")
|
||||
putObject(t, c, bucketB, keyB, "b")
|
||||
backdateMtime(t, fc, bucketA, keyA, 30)
|
||||
backdateMtime(t, fc, bucketB, keyB, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// Both buckets must have their objects expired in this single sweep.
|
||||
for _, c2 := range []struct {
|
||||
bucket, key string
|
||||
}{
|
||||
{bucketA, keyA},
|
||||
{bucketB, keyB},
|
||||
} {
|
||||
c2 := c2
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(c2.bucket), Key: aws.String(c2.key),
|
||||
})
|
||||
// Pin: only count 404 / NoSuchKey as deletion. Any other
|
||||
// error (transport failure, dead endpoint, auth) would
|
||||
// otherwise mask a real bug as a "passed" test.
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond,
|
||||
"%s/%s must be expired by the multi-bucket sweep", c2.bucket, c2.key)
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
// Multiple-rules-per-bucket integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleMultipleRulesInOneBucket: a single bucket carries two
|
||||
// independent Expiration rules with disjoint prefix filters and
|
||||
// different Days thresholds. Each rule must fire only on its prefix
|
||||
// and ignore the other; objects outside both prefixes must survive.
|
||||
//
|
||||
// Pinned because compile.Compile builds one CompiledAction per rule
|
||||
// per kind, all sharing the same bucket index. A bug that lets one
|
||||
// rule's prefix or threshold leak into the other (e.g. last-write-
|
||||
// wins on a shared map) would silently expire wrong objects.
|
||||
func TestLifecycleMultipleRulesInOneBucket(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("multi-rule")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("logs-1d"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String("logs/")},
|
||||
Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
|
||||
},
|
||||
{
|
||||
ID: aws.String("tmp-7d"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String("tmp/")},
|
||||
Expiration: &types.LifecycleExpiration{Days: aws.Int32(7)},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const logsKey = "logs/access.log"
|
||||
const tmpKey = "tmp/scratch.bin"
|
||||
const otherKey = "data/keep.bin"
|
||||
|
||||
putObject(t, c, bucket, logsKey, "log entry")
|
||||
putObject(t, c, bucket, tmpKey, "scratch")
|
||||
putObject(t, c, bucket, otherKey, "important")
|
||||
|
||||
// Backdate logs/ past the 1d rule, tmp/ past the 7d rule, data/
|
||||
// past both thresholds — but data/ matches no rule, so it must
|
||||
// survive regardless of age.
|
||||
backdateMtime(t, fc, bucket, logsKey, 30)
|
||||
backdateMtime(t, fc, bucket, tmpKey, 30)
|
||||
backdateMtime(t, fc, bucket, otherKey, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// logs/ rule fires.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(logsKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "logs/ rule must expire %s", logsKey)
|
||||
|
||||
// tmp/ rule fires.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(tmpKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "tmp/ rule must expire %s", tmpKey)
|
||||
|
||||
// data/ matches NEITHER rule and must survive — pinning that the
|
||||
// per-rule prefix gate is independent and additive, not a global
|
||||
// "any rule's prefix" check.
|
||||
_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(otherKey),
|
||||
})
|
||||
require.NoError(t, err, "object outside both rule prefixes must survive")
|
||||
}
|
||||
@@ -1,110 +0,0 @@
|
||||
// NewerNoncurrentVersions integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putNewerNoncurrentLifecycle keeps the N most recent noncurrent versions
|
||||
// and expires the rest. NoncurrentDays is required by the AWS XML schema
|
||||
// alongside NewerNoncurrentVersions; keeping it at 1 means the rule fires
|
||||
// as soon as a version moves past the keep-count and is older than 1 day.
|
||||
func putNewerNoncurrentLifecycle(t *testing.T, c *s3.Client, bucket, prefix string, keep int32) {
|
||||
t.Helper()
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("keep-newest-noncurrent"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
|
||||
NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
|
||||
NewerNoncurrentVersions: aws.Int32(keep),
|
||||
NoncurrentDays: aws.Int32(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestLifecycleNewerNoncurrentVersions: NewerNoncurrentVersions=1 keeps
|
||||
// only the most recent noncurrent version; older noncurrent versions
|
||||
// must be deleted. PUT v1, v2, v3, v4 (v4 current; v1-v3 noncurrent),
|
||||
// backdate v1, v2, v3, run the worker, expect v3 (newest noncurrent)
|
||||
// and v4 (current) to remain, v1 and v2 to be removed.
|
||||
//
|
||||
// This routes through routePointerTransition's "needs full expansion"
|
||||
// path — distinct from the per-version NoncurrentDays path because
|
||||
// the rule depends on per-version rank, not just per-version age.
|
||||
func TestLifecycleNewerNoncurrentVersions(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("newer-nc")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putVersioningEnabled(t, c, bucket)
|
||||
putNewerNoncurrentLifecycle(t, c, bucket, "n/", 1)
|
||||
|
||||
const key = "n/obj.txt"
|
||||
puts := make([]string, 4) // versionIds in PUT order
|
||||
for i := range puts {
|
||||
out, err := c.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
Body: strings.NewReader(time.Now().Format(time.RFC3339Nano) + "-" + string(rune('a'+i))),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
puts[i] = aws.ToString(out.VersionId)
|
||||
require.NotEmpty(t, puts[i])
|
||||
}
|
||||
v1, v2, v3, v4 := puts[0], puts[1], puts[2], puts[3]
|
||||
require.NotEqual(t, v1, v2)
|
||||
require.NotEqual(t, v3, v4)
|
||||
|
||||
// Backdate every noncurrent (v1, v2, v3) so all three satisfy the
|
||||
// NoncurrentDays>=1 floor; the keep-newest-1 logic then chooses
|
||||
// which to drop.
|
||||
for _, vid := range []string{v1, v2, v3} {
|
||||
backdateVersionedMtime(t, fc, bucket, key, vid, 30)
|
||||
}
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// v1 and v2 (older noncurrent) must be expired.
|
||||
for _, vid := range []string{v1, v2} {
|
||||
vid := vid
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(vid),
|
||||
})
|
||||
return err != nil
|
||||
}, 30*time.Second, 500*time.Millisecond, "older noncurrent version %s must be expired", vid)
|
||||
}
|
||||
|
||||
// v3 (newest noncurrent within the keep=1 window) must remain.
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v3),
|
||||
})
|
||||
require.NoError(t, err, "v3 (newest noncurrent within keep=1) must survive")
|
||||
|
||||
// v4 (current) must remain — current versions are immune to
|
||||
// NoncurrentVersionExpiration regardless of how the keep-count
|
||||
// arithmetic plays out.
|
||||
currentHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "current version v4 must remain")
|
||||
require.Equal(t, v4, aws.ToString(currentHead.VersionId))
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
// Object Lock + lifecycle integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleSkipsObjectLockedObjects: an object under Object Lock
|
||||
// retention must NOT be expired by the lifecycle worker. The handler's
|
||||
// enforceObjectLockProtections check (s3api_internal_lifecycle.go:47)
|
||||
// returns an error which the dispatcher classifies as
|
||||
// SKIPPED_OBJECT_LOCK. Pinning the integration end-to-end so a
|
||||
// regression in either the lock state lookup or the dispatcher's
|
||||
// outcome handling is caught.
|
||||
//
|
||||
// Companion positive control: a second object in the same bucket
|
||||
// without retention is expired by the same sweep — proves the rule
|
||||
// is firing AND that the lock check is selective, not a blanket skip.
|
||||
func TestLifecycleSkipsObjectLockedObjects(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("objlock")
|
||||
// Object Lock requires the bucket to be created with the flag set;
|
||||
// it can't be enabled retroactively on an existing bucket. Use the
|
||||
// raw API rather than mustCreateBucket here so we can pass the
|
||||
// ObjectLockEnabledForBucket option.
|
||||
_, err := c.CreateBucket(context.Background(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
ObjectLockEnabledForBucket: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
// Best-effort cleanup mirrors mustCreateBucket; locked objects
|
||||
// may resist deletion but the test process continues regardless.
|
||||
c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{Bucket: aws.String(bucket)})
|
||||
listOut, _ := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
|
||||
if listOut != nil {
|
||||
for _, v := range listOut.Versions {
|
||||
// Bypass governance retention to free the bucket for delete.
|
||||
c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: v.Key,
|
||||
VersionId: v.VersionId,
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
}
|
||||
for _, m := range listOut.DeleteMarkers {
|
||||
c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket), Key: m.Key, VersionId: m.VersionId,
|
||||
})
|
||||
}
|
||||
}
|
||||
c.DeleteBucket(context.Background(), &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
|
||||
})
|
||||
|
||||
putExpirationLifecycle(t, c, bucket, "lock/", 1)
|
||||
|
||||
const lockedKey = "lock/protected.txt"
|
||||
const freeKey = "lock/free.txt"
|
||||
|
||||
// Locked object: PUT with GOVERNANCE retention until 1h from now.
|
||||
lockedPut, err := c.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(lockedKey),
|
||||
Body: strings.NewReader("locked"),
|
||||
ObjectLockMode: types.ObjectLockModeGovernance,
|
||||
ObjectLockRetainUntilDate: aws.Time(time.Now().Add(time.Hour)),
|
||||
})
|
||||
require.NoError(t, err, "PUT with retention must succeed on a lock-enabled bucket")
|
||||
require.NotEmpty(t, aws.ToString(lockedPut.VersionId))
|
||||
|
||||
// Free object: PUT without retention. Object Lock requires
|
||||
// versioning, so the bucket is implicitly versioned and every PUT
|
||||
// produces a versionId. Capture both for the version-aware
|
||||
// backdate path.
|
||||
freePut, err := c.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(freeKey), Body: strings.NewReader("free"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
freeVersionID := aws.ToString(freePut.VersionId)
|
||||
require.NotEmpty(t, freeVersionID)
|
||||
|
||||
// Backdate both so they would otherwise both expire under the
|
||||
// 1-day rule. The lock check is what distinguishes them.
|
||||
// Versioning-enabled buckets store entries under .versions/v_<id>,
|
||||
// not at the bare key path, so use backdateVersionedMtime.
|
||||
backdateVersionedMtime(t, fc, bucket, lockedKey, aws.ToString(lockedPut.VersionId), 30)
|
||||
backdateVersionedMtime(t, fc, bucket, freeKey, freeVersionID, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// Free object expires (positive control).
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(freeKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "unlocked object must expire (control)")
|
||||
|
||||
// Locked object survives — Object Lock retention overrides the
|
||||
// lifecycle rule. HEAD without versionId returns the still-current
|
||||
// object data, not a 404.
|
||||
_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(lockedKey),
|
||||
})
|
||||
require.NoError(t, err, "object under GOVERNANCE retention must survive the sweep")
|
||||
}
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
// Size-filter integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestLifecycleSizeFilterGreaterThan: ObjectSizeGreaterThan is a strict >
|
||||
// gate (filterAllows in match.go uses ev.Size <= rule.FilterSizeGreaterThan
|
||||
// to reject). Two backdated objects on the same prefix — only the one
|
||||
// strictly larger than the threshold is expired.
|
||||
func TestLifecycleSizeFilterGreaterThan(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("size-filter")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
|
||||
const threshold = 100
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{{
|
||||
ID: aws.String("size-gt"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{
|
||||
And: &types.LifecycleRuleAndOperator{
|
||||
Prefix: aws.String("sz/"),
|
||||
ObjectSizeGreaterThan: aws.Int64(threshold),
|
||||
},
|
||||
},
|
||||
Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
|
||||
}},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const smallKey = "sz/small.txt" // size <= threshold → must remain
|
||||
const largeKey = "sz/large.txt" // size > threshold → must expire
|
||||
putObject(t, c, bucket, smallKey, strings.Repeat("a", threshold)) // exactly at boundary; gate is strictly >, so rejects
|
||||
putObject(t, c, bucket, largeKey, strings.Repeat("a", threshold+50))
|
||||
|
||||
backdateMtime(t, fc, bucket, smallKey, 30)
|
||||
backdateMtime(t, fc, bucket, largeKey, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// Larger-than-threshold object expired.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(largeKey),
|
||||
})
|
||||
return err != nil
|
||||
}, 30*time.Second, 500*time.Millisecond, "object larger than threshold must expire")
|
||||
|
||||
// Boundary object (size == threshold) survives — strictly-greater gate.
|
||||
_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(smallKey),
|
||||
})
|
||||
require.NoError(t, err, "object at exact threshold must remain (gate is strictly >)")
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
// Suspended-versioning Expiration integration scenario.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putVersioningSuspended flips Status=Suspended. PUTs after this point
|
||||
// overwrite the null version in place rather than creating new versionIds.
|
||||
func putVersioningSuspended(t *testing.T, c *s3.Client, bucket string) {
|
||||
t.Helper()
|
||||
_, err := c.PutBucketVersioning(context.Background(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucket),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusSuspended},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestLifecycleSuspendedVersioningExpiration: on a suspended-versioning
|
||||
// bucket, Expiration takes a different code path than the Enabled and
|
||||
// Off cases. lifecycleDispatch's VersioningSuspended branch first
|
||||
// deletes the null version (via deleteSpecificObjectVersion(versionId="null"))
|
||||
// and then writes a fresh delete marker on top.
|
||||
//
|
||||
// Setup: enable then suspend versioning so any noncurrent versions can
|
||||
// exist alongside the null one; PUT once-with-versioning + once-after-
|
||||
// suspend (so we have a noncurrent version + a null version), backdate
|
||||
// the null, run the worker. The null must be deleted and replaced by a
|
||||
// delete marker; the prior noncurrent version stays addressable.
|
||||
func TestLifecycleSuspendedVersioningExpiration(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("suspended")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putVersioningEnabled(t, c, bucket)
|
||||
|
||||
const key = "s/obj.txt"
|
||||
// First PUT under Enabled: produces a real versionId.
|
||||
putObject(t, c, bucket, key, "v1")
|
||||
headEnabled, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
v1 := aws.ToString(headEnabled.VersionId)
|
||||
require.NotEmpty(t, v1)
|
||||
require.NotEqual(t, "null", v1)
|
||||
|
||||
// Suspend versioning, then PUT again. The new write becomes the
|
||||
// "null" version that overwrites the bare-key entry; v1 demotes to
|
||||
// a noncurrent version.
|
||||
putVersioningSuspended(t, c, bucket)
|
||||
putObject(t, c, bucket, key, "null-version")
|
||||
|
||||
// Set the Expiration rule AFTER the writes so the rule
|
||||
// configuration race against PUTs is deterministic. Days=1 for
|
||||
// AWS-spec compliance; backdating the null entry makes the worker
|
||||
// fire immediately.
|
||||
putExpirationLifecycle(t, c, bucket, "s/", 1)
|
||||
|
||||
// Backdate the bare-key entry (the null version): on suspended
|
||||
// buckets the null lives at the bare path, NOT under .versions/.
|
||||
backdateMtime(t, fc, bucket, key, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// HEAD without versionId hits the latest, which after the worker
|
||||
// runs should be a delete marker (404 NoSuchKey).
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
return err != nil
|
||||
}, 30*time.Second, 500*time.Millisecond, "delete marker must dominate latest after suspended-versioning expiration")
|
||||
|
||||
// The noncurrent v1 (created under Enabled) must remain — Expiration
|
||||
// in the suspended branch only touches the null version, not other
|
||||
// versions in the .versions/ folder.
|
||||
directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v1),
|
||||
})
|
||||
require.NoError(t, err, "noncurrent v1 must survive — suspended Expiration only touches the null")
|
||||
require.NotNil(t, directHead)
|
||||
|
||||
// ListObjectVersions must show: v1 still present, a fresh delete
|
||||
// marker as latest, and no more "null" version.
|
||||
listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucket), Prefix: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
var sawV1, sawMarker bool
|
||||
for _, v := range listOut.Versions {
|
||||
if aws.ToString(v.Key) == key {
|
||||
vid := aws.ToString(v.VersionId)
|
||||
if vid == v1 {
|
||||
sawV1 = true
|
||||
}
|
||||
require.NotEqual(t, "null", vid, "null version must have been removed")
|
||||
}
|
||||
}
|
||||
for _, m := range listOut.DeleteMarkers {
|
||||
if aws.ToString(m.Key) == key && aws.ToBool(m.IsLatest) {
|
||||
sawMarker = true
|
||||
}
|
||||
}
|
||||
require.True(t, sawV1, "v1 must still appear in ListObjectVersions")
|
||||
require.True(t, sawMarker, "fresh delete marker must dominate latest")
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
// Package lifecycle is the end-to-end test for the event-driven S3
|
||||
// lifecycle worker, driven by the s3.lifecycle.run-shard shell command.
|
||||
//
|
||||
// The S3 API rejects Expiration.Days < 1, so a literal "wait one day"
|
||||
// integration test is unworkable. Instead, the test backdates the target
|
||||
// object's mtime via the filer's UpdateEntry RPC: from the engine's
|
||||
// perspective the object is past its expiration window the moment the
|
||||
// shell command starts, and the dispatcher fires immediately.
|
||||
//
|
||||
// Variables (set by the Makefile):
|
||||
//
|
||||
// WEED_BINARY - path to the built `weed` binary
|
||||
// S3_ENDPOINT - http://host:port for the S3 API
|
||||
// FILER_GRPC_ADDRESS - host:port of the filer's gRPC listener
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultS3Endpoint = "http://localhost:8333"
|
||||
defaultS3GrpcEndpoint = "localhost:18333"
|
||||
defaultMasterEndpt = "http://localhost:9333"
|
||||
defaultFilerGRPC = "localhost:18888"
|
||||
bucketLifecycleXMLKey = "s3-bucket-lifecycle-configuration-xml"
|
||||
bucketsPath = "/buckets"
|
||||
accessKey = "some_access_key1"
|
||||
secretKey = "some_secret_key1"
|
||||
region = "us-east-1"
|
||||
)
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func s3Client(t *testing.T) *s3.Client {
|
||||
t.Helper()
|
||||
endpoint := envOr("S3_ENDPOINT", defaultS3Endpoint)
|
||||
cfg, err := config.LoadDefaultConfig(context.Background(),
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
|
||||
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
|
||||
func(service, region string, _ ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{URL: endpoint, SigningRegion: region, HostnameImmutable: true}, nil
|
||||
})),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
|
||||
}
|
||||
|
||||
func filerClient(t *testing.T) (filer_pb.SeaweedFilerClient, func()) {
|
||||
t.Helper()
|
||||
addr := envOr("FILER_GRPC_ADDRESS", defaultFilerGRPC)
|
||||
conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
require.NoError(t, err)
|
||||
return filer_pb.NewSeaweedFilerClient(conn), func() { conn.Close() }
|
||||
}
|
||||
|
||||
// uniqueBucket returns a fresh bucket name; the shell command picks up all
|
||||
// buckets with lifecycle config, so two parallel-running tests would conflict
|
||||
// on cursor state. Each test gets its own bucket.
|
||||
func uniqueBucket(prefix string) string {
|
||||
return fmt.Sprintf("lc-%s-%d", prefix, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func mustCreateBucket(t *testing.T, c *s3.Client, name string) {
|
||||
t.Helper()
|
||||
_, err := c.CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(name)})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
// Drop the lifecycle configuration first so any subsequent
|
||||
// shell-driven shard sweep stops loading rules for this bucket
|
||||
// — without this, a later test's run-shard would pick up the
|
||||
// dead bucket's config and produce phantom dispatches.
|
||||
c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{Bucket: aws.String(name)})
|
||||
|
||||
// Empty every version + delete marker (versioning-aware buckets
|
||||
// hold state that ListObjectsV2 doesn't surface). Best-effort:
|
||||
// errors are tolerated because the bucket might already be
|
||||
// half-torn-down.
|
||||
listOut, _ := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{Bucket: aws.String(name)})
|
||||
if listOut != nil {
|
||||
for _, v := range listOut.Versions {
|
||||
c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(name), Key: v.Key, VersionId: v.VersionId,
|
||||
})
|
||||
}
|
||||
for _, m := range listOut.DeleteMarkers {
|
||||
c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(name), Key: m.Key, VersionId: m.VersionId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Catch any non-versioned objects ListObjectVersions missed.
|
||||
objs, _ := c.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{Bucket: aws.String(name)})
|
||||
if objs != nil {
|
||||
for _, o := range objs.Contents {
|
||||
c.DeleteObject(context.Background(), &s3.DeleteObjectInput{Bucket: aws.String(name), Key: o.Key})
|
||||
}
|
||||
}
|
||||
|
||||
// Abort any in-flight multipart uploads — the AbortMPU lifecycle
|
||||
// test leaves these intentionally; without an explicit Abort the
|
||||
// bucket DELETE refuses with NotEmpty.
|
||||
mpus, _ := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{Bucket: aws.String(name)})
|
||||
if mpus != nil {
|
||||
for _, u := range mpus.Uploads {
|
||||
c.AbortMultipartUpload(context.Background(), &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(name), Key: u.Key, UploadId: u.UploadId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
c.DeleteBucket(context.Background(), &s3.DeleteBucketInput{Bucket: aws.String(name)})
|
||||
})
|
||||
}
|
||||
|
||||
func putExpirationLifecycle(t *testing.T, c *s3.Client, bucket, prefix string, days int32) {
|
||||
t.Helper()
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("expire-prefix"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
|
||||
Expiration: &types.LifecycleExpiration{
|
||||
Days: aws.Int32(days),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func putObject(t *testing.T, c *s3.Client, bucket, key, body string) {
|
||||
t.Helper()
|
||||
_, err := c.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(body),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// backdateMtime rewrites the object's filer entry attributes so its Mtime
|
||||
// is daysOld days in the past. This sidesteps the AWS-spec minimum of one
|
||||
// day for Expiration.Days, letting the lifecycle dispatcher fire on demand.
|
||||
func backdateMtime(t *testing.T, fc filer_pb.SeaweedFilerClient, bucket, key string, daysOld int) {
|
||||
t.Helper()
|
||||
dir, name := splitBucketKey(bucket, key)
|
||||
resp, err := fc.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
require.NoError(t, err, "lookup %s/%s", dir, name)
|
||||
require.NotNil(t, resp.Entry)
|
||||
require.NotNil(t, resp.Entry.Attributes)
|
||||
|
||||
resp.Entry.Attributes.Mtime = time.Now().Add(-time.Duration(daysOld) * 24 * time.Hour).Unix()
|
||||
resp.Entry.Attributes.MtimeNs = 0
|
||||
_, err = fc.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: resp.Entry,
|
||||
})
|
||||
require.NoError(t, err, "update %s/%s", dir, name)
|
||||
}
|
||||
|
||||
func splitBucketKey(bucket, key string) (dir, name string) {
|
||||
full := bucketsPath + "/" + bucket + "/" + key
|
||||
if i := strings.LastIndex(full, "/"); i >= 0 {
|
||||
return full[:i], full[i+1:]
|
||||
}
|
||||
return full, ""
|
||||
}
|
||||
|
||||
// runShellCommand invokes `weed shell` with a one-shot s3.lifecycle.run-shard
|
||||
// command piped via stdin and returns the combined stdout+stderr.
|
||||
func runShellCommand(t *testing.T, command string) string {
|
||||
t.Helper()
|
||||
binary := envOr("WEED_BINARY", "")
|
||||
require.NotEmpty(t, binary, "WEED_BINARY must be set")
|
||||
masterEndpoint := envOr("MASTER_ENDPOINT", defaultMasterEndpt)
|
||||
master := strings.TrimPrefix(masterEndpoint, "http://")
|
||||
if u, err := url.Parse(masterEndpoint); err == nil && u.Host != "" {
|
||||
master = u.Host
|
||||
}
|
||||
cmd := exec.Command(binary, "shell", "-master="+master)
|
||||
cmd.Stdin = strings.NewReader(command + "\nexit\n")
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &out
|
||||
err := cmd.Run()
|
||||
output := out.String()
|
||||
if err != nil {
|
||||
t.Logf("shell command output:\n%s", output)
|
||||
t.Fatalf("shell exec failed: %v", err)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
// TestLifecycleExpirationFiresOnBackdatedObject is the end-to-end
|
||||
// validation: a 1-day expiration rule plus an object whose mtime has been
|
||||
// backdated to 30 days ago must result in deletion when the shell command
|
||||
// runs the matching shard.
|
||||
func TestLifecycleExpirationFiresOnBackdatedObject(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("expire")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putExpirationLifecycle(t, c, bucket, "expire/", 1)
|
||||
|
||||
const oldKey = "expire/old.txt"
|
||||
const freshKey = "keep/fresh.txt"
|
||||
putObject(t, c, bucket, oldKey, "old")
|
||||
putObject(t, c, bucket, freshKey, "fresh")
|
||||
backdateMtime(t, fc, bucket, oldKey, 30)
|
||||
|
||||
// One subscription handles every shard via -shards 0-15; this stays
|
||||
// independent of which (bucket, key) hash lands the target on.
|
||||
// -runtime caps the run by wall-clock so the subprocess exits even
|
||||
// when the in-shard event count never reaches -events.
|
||||
out := runShellCommand(t, fmt.Sprintf(
|
||||
"s3.lifecycle.run-shard -shards 0-15 -s3 %s -events 0 -dispatch 200ms -checkpoint 5s -runtime 10s",
|
||||
envOr("S3_GRPC_ENDPOINT", defaultS3GrpcEndpoint),
|
||||
))
|
||||
t.Logf("shell command output:\n%s", out)
|
||||
require.NotContains(t, out, "FATAL", "shell output:\n%s", out)
|
||||
|
||||
// Allow the dispatched delete to land in the filer + propagate to S3.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(oldKey),
|
||||
})
|
||||
return err != nil
|
||||
}, 30*time.Second, 500*time.Millisecond, "expected %s/%s to be deleted", bucket, oldKey)
|
||||
|
||||
// The fresh, in-prefix-but-recent object must remain (1d rule, mtime
|
||||
// is now). Best-evidence check after waiting for the same poll window.
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(freshKey),
|
||||
})
|
||||
require.NoError(t, err, "%s should still exist", freshKey)
|
||||
}
|
||||
@@ -1,456 +0,0 @@
|
||||
// Versioning + delete-marker + noncurrent integration scenarios for the
|
||||
// event-driven lifecycle worker. Same backdating trick as the base test:
|
||||
// AWS rejects Days < 1, so the test ages the object via the filer's
|
||||
// UpdateEntry RPC and runs the shell command once with -runtime 10s.
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// putVersioningEnabled flips Status=Enabled on the bucket so subsequent
|
||||
// PUTs produce versioned siblings rather than overwriting.
|
||||
func putVersioningEnabled(t *testing.T, c *s3.Client, bucket string) {
|
||||
t.Helper()
|
||||
_, err := c.PutBucketVersioning(context.Background(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucket),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// putNoncurrentExpirationLifecycle sets a NoncurrentVersionExpiration rule
|
||||
// targeting a prefix. Days=1 because AWS rejects 0; the test backdates
|
||||
// the noncurrent version.
|
||||
func putNoncurrentExpirationLifecycle(t *testing.T, c *s3.Client, bucket, prefix string, days int32) {
|
||||
t.Helper()
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("expire-noncurrent"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
|
||||
NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
|
||||
NoncurrentDays: aws.Int32(days),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// backdateVersionedMtime ages a specific .versions/v_<id> entry. The
|
||||
// version files live under <buckets>/<bucket>/<key>.versions/v_<versionId>.
|
||||
//
|
||||
// Also clears the ExtNoncurrentSinceNsKey stamp if present. The stamp
|
||||
// records when this version was demoted by its successor's PUT; tests
|
||||
// that backdate a version's own mtime aren't expressing a coherent claim
|
||||
// about the demotion moment, so leaving a real-now stamp would let it
|
||||
// dominate the backdated mtime and stop the rule from firing. Clearing
|
||||
// makes the lifecycle engine fall back to the sibling-mtime derivation,
|
||||
// which is what these tests were originally written against. Production
|
||||
// PUTs always write a stamp; this is a test-only escape hatch.
|
||||
func backdateVersionedMtime(t *testing.T, fc filer_pb.SeaweedFilerClient, bucket, key, versionID string, daysOld int) {
|
||||
t.Helper()
|
||||
dir := bucketsPath + "/" + bucket + "/" + key + ".versions"
|
||||
name := "v_" + versionID
|
||||
resp, err := fc.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir, Name: name,
|
||||
})
|
||||
require.NoError(t, err, "lookup version %s/%s", dir, name)
|
||||
require.NotNil(t, resp.Entry)
|
||||
require.NotNil(t, resp.Entry.Attributes)
|
||||
|
||||
resp.Entry.Attributes.Mtime = time.Now().Add(-time.Duration(daysOld) * 24 * time.Hour).Unix()
|
||||
resp.Entry.Attributes.MtimeNs = 0
|
||||
delete(resp.Entry.Extended, s3_constants.ExtNoncurrentSinceNsKey)
|
||||
_, err = fc.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir, Entry: resp.Entry,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func runLifecycleShard(t *testing.T) string {
|
||||
t.Helper()
|
||||
// -refresh 1s drives an inter-pass loop inside the 10s runtime so
|
||||
// rules that need two passes to settle (e.g.
|
||||
// TestLifecycleExpiredDeleteMarkerCleanup: pass 1 removes v1, pass 2
|
||||
// removes the now-orphaned marker) get the second pass.
|
||||
out := runShellCommand(t, fmt.Sprintf(
|
||||
"s3.lifecycle.run-shard -shards 0-15 -s3 %s -events 0 -refresh 1s -runtime 10s",
|
||||
envOr("S3_GRPC_ENDPOINT", defaultS3GrpcEndpoint),
|
||||
))
|
||||
require.NotContains(t, out, "FATAL", "shell output:\n%s", out)
|
||||
return out
|
||||
}
|
||||
|
||||
// TestLifecycleVersionedBucketCreatesDeleteMarker: an Expiration rule on a
|
||||
// versioned bucket must produce a delete marker (latest version after
|
||||
// expiration) rather than removing the version itself. Pre-fix this would
|
||||
// have unconditionally deleted the live version.
|
||||
func TestLifecycleVersionedBucketCreatesDeleteMarker(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("vexp")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putVersioningEnabled(t, c, bucket)
|
||||
putExpirationLifecycle(t, c, bucket, "expire/", 1)
|
||||
|
||||
const key = "expire/old.txt"
|
||||
putObject(t, c, bucket, key, "v1")
|
||||
|
||||
// Look up the version we just created so we can backdate it. The bare
|
||||
// filer entry under <bucket>/<key> is the latest-version pointer in
|
||||
// SeaweedFS's versioned layout.
|
||||
headOut, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, headOut.VersionId)
|
||||
versionID := aws.ToString(headOut.VersionId)
|
||||
require.NotEmpty(t, versionID)
|
||||
backdateVersionedMtime(t, fc, bucket, key, versionID, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// HEAD without versionId returns the latest version. After the worker
|
||||
// runs, the latest must be a delete marker (NoSuchKey on plain HEAD).
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "expected delete marker to become latest for %s/%s", bucket, key)
|
||||
|
||||
// The original version must still be addressable directly — Expiration
|
||||
// on a versioned bucket creates a marker, it doesn't drop the version.
|
||||
directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(versionID),
|
||||
})
|
||||
require.NoError(t, err, "original version must still exist")
|
||||
require.NotNil(t, directHead)
|
||||
|
||||
// ListObjectVersions should now show a delete marker dominating the
|
||||
// version. The IsLatest=true marker proves the worker created a fresh
|
||||
// marker rather than aging the existing version.
|
||||
listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucket), Prefix: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
var sawMarker bool
|
||||
for _, m := range listOut.DeleteMarkers {
|
||||
if aws.ToString(m.Key) == key && aws.ToBool(m.IsLatest) {
|
||||
sawMarker = true
|
||||
break
|
||||
}
|
||||
}
|
||||
require.True(t, sawMarker, "ListObjectVersions must show a delete marker as latest for %s", key)
|
||||
}
|
||||
|
||||
// TestLifecycleNoncurrentVersionExpiration: NoncurrentVersionExpiration
|
||||
// only fires on noncurrent versions. PUT v1, PUT v2 (so v1 → noncurrent),
|
||||
// backdate v1, run worker. v1 must be removed; v2 stays current.
|
||||
func TestLifecycleNoncurrentVersionExpiration(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("noncurrent")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putVersioningEnabled(t, c, bucket)
|
||||
putNoncurrentExpirationLifecycle(t, c, bucket, "v/", 1)
|
||||
|
||||
const key = "v/obj.txt"
|
||||
// First PUT.
|
||||
put1, err := c.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), Body: strings.NewReader("v1"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
v1 := aws.ToString(put1.VersionId)
|
||||
|
||||
// Second PUT promotes v1 to noncurrent.
|
||||
put2, err := c.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), Body: strings.NewReader("v2"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
v2 := aws.ToString(put2.VersionId)
|
||||
require.NotEqual(t, v1, v2)
|
||||
|
||||
// NoncurrentDays is clocked from the SUCCESSOR's mtime — i.e. when
|
||||
// the version became noncurrent — not from the displaced version's
|
||||
// own mtime. Backdating only v1 leaves the noncurrent clock (v2's
|
||||
// mtime) at "now" and the rule never fires. Age both with v1 older
|
||||
// than v2 so the ordering is realistic.
|
||||
backdateVersionedMtime(t, fc, bucket, key, v1, 31)
|
||||
backdateVersionedMtime(t, fc, bucket, key, v2, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// v1 must be gone.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v1),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "noncurrent v1 must be expired")
|
||||
|
||||
// v2 must still be addressable BY VERSION ID — pinning that the
|
||||
// noncurrent dispatch didn't accidentally remove the current
|
||||
// version's file.
|
||||
directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v2),
|
||||
})
|
||||
require.NoError(t, err, "current version v2 (versionId=%s) must still be addressable directly", v2)
|
||||
|
||||
// And HEAD without versionId (i.e., the latest) must hit v2 — pinning
|
||||
// that the bare-key pointer wasn't removed and a delete marker
|
||||
// wasn't fabricated as a side effect of the noncurrent delete. On
|
||||
// failure we dump ListObjectVersions so the failure log shows
|
||||
// exactly what's there (versions vs delete markers vs bare).
|
||||
currentHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
listOut, listErr := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucket), Prefix: aws.String(key),
|
||||
})
|
||||
if listErr != nil || listOut == nil {
|
||||
t.Logf("HeadObject(latest) failed: %v\nListObjectVersions err=%v", err, listErr)
|
||||
} else {
|
||||
t.Logf("HeadObject(latest) failed: %v\nListObjectVersions err=%v versions=%d markers=%d", err, listErr, len(listOut.Versions), len(listOut.DeleteMarkers))
|
||||
for _, v := range listOut.Versions {
|
||||
t.Logf(" version key=%s id=%s isLatest=%v", aws.ToString(v.Key), aws.ToString(v.VersionId), aws.ToBool(v.IsLatest))
|
||||
}
|
||||
for _, m := range listOut.DeleteMarkers {
|
||||
t.Logf(" marker key=%s id=%s isLatest=%v", aws.ToString(m.Key), aws.ToString(m.VersionId), aws.ToBool(m.IsLatest))
|
||||
}
|
||||
}
|
||||
require.NoError(t, err, "HEAD(latest) on %s/%s must succeed; v2 (%s) must remain current", bucket, key, v2)
|
||||
}
|
||||
require.Equal(t, v2, aws.ToString(currentHead.VersionId), "latest pointer must still resolve to v2")
|
||||
_ = directHead
|
||||
}
|
||||
|
||||
// TestLifecycleExpiredDeleteMarkerCleanup: a sole-survivor delete marker
|
||||
// must be removed by an ExpiredObjectDeleteMarker=true rule. Setup: PUT v1,
|
||||
// DELETE (creates a marker that becomes latest, with v1 as noncurrent),
|
||||
// expire v1 via NoncurrentVersionExpiration so the marker is alone, then
|
||||
// the marker rule fires.
|
||||
//
|
||||
// We exercise this in two passes: pass 1 (Noncurrent rule on a sibling
|
||||
// prefix) cleans v1; pass 2 (ExpiredObjectDeleteMarker rule, also on the
|
||||
// same prefix) removes the now-orphaned marker. Since the lifecycle XML
|
||||
// only carries one set of rules per bucket, we wire both.
|
||||
func TestLifecycleExpiredDeleteMarkerCleanup(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("marker")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
putVersioningEnabled(t, c, bucket)
|
||||
|
||||
// Combined rule: noncurrent expiration + delete-marker cleanup, both
|
||||
// on the same prefix.
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{
|
||||
{
|
||||
ID: aws.String("expire-noncurrent"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String("m/")},
|
||||
NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
|
||||
NoncurrentDays: aws.Int32(1),
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: aws.String("expire-marker"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String("m/")},
|
||||
Expiration: &types.LifecycleExpiration{
|
||||
ExpiredObjectDeleteMarker: aws.Bool(true),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const key = "m/obj.txt"
|
||||
put1, err := c.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key), Body: strings.NewReader("v1"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
v1 := aws.ToString(put1.VersionId)
|
||||
|
||||
// Plain DELETE with versioning on creates a marker; the marker becomes
|
||||
// latest, v1 demotes to noncurrent.
|
||||
_, err = c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Backdate v1 so the noncurrent rule expires it.
|
||||
backdateVersionedMtime(t, fc, bucket, key, v1, 30)
|
||||
|
||||
// Backdate the delete marker too. ListObjectVersions tells us the
|
||||
// marker's versionId.
|
||||
listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucket), Prefix: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
var markerID string
|
||||
for _, m := range listOut.DeleteMarkers {
|
||||
if aws.ToString(m.Key) == key {
|
||||
markerID = aws.ToString(m.VersionId)
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, markerID, "expected to find the delete marker we just created")
|
||||
backdateVersionedMtime(t, fc, bucket, key, markerID, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// Both v1 and the marker must be gone — the .versions/<key>/ folder
|
||||
// should hold nothing for this key.
|
||||
require.Eventuallyf(t, func() bool {
|
||||
listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucket), Prefix: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, v := range listOut.Versions {
|
||||
if aws.ToString(v.Key) == key {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, m := range listOut.DeleteMarkers {
|
||||
if aws.ToString(m.Key) == key {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, 30*time.Second, 500*time.Millisecond, "every version and marker for %s must be gone", key)
|
||||
}
|
||||
|
||||
// TestLifecycleDisabledRuleSkipsObject: a rule with Status=Disabled must
|
||||
// not produce dispatches, even on a backdated object that would otherwise
|
||||
// match. Negative test for the engine's enabled-status gate.
|
||||
func TestLifecycleDisabledRuleSkipsObject(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("disabled")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{{
|
||||
ID: aws.String("disabled-rule"),
|
||||
Status: types.ExpirationStatusDisabled,
|
||||
Filter: &types.LifecycleRuleFilter{Prefix: aws.String("d/")},
|
||||
Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
|
||||
}},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const key = "d/obj.txt"
|
||||
putObject(t, c, bucket, key, "still-here")
|
||||
backdateMtime(t, fc, bucket, key, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
// Wait the same window; the object MUST still exist after.
|
||||
time.Sleep(2 * time.Second) // give the worker a chance to (incorrectly) act
|
||||
_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Disabled rule must not delete the object")
|
||||
}
|
||||
|
||||
// TestLifecycleTagFilter: a rule with a tag filter must only match objects
|
||||
// carrying that tag. Two backdated objects, one tagged, one not — only the
|
||||
// tagged one is removed.
|
||||
func TestLifecycleTagFilter(t *testing.T) {
|
||||
c := s3Client(t)
|
||||
fc, fcClose := filerClient(t)
|
||||
defer fcClose()
|
||||
|
||||
bucket := uniqueBucket("tagfilter")
|
||||
mustCreateBucket(t, c, bucket)
|
||||
_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
|
||||
Bucket: aws.String(bucket),
|
||||
LifecycleConfiguration: &types.BucketLifecycleConfiguration{
|
||||
Rules: []types.LifecycleRule{{
|
||||
ID: aws.String("tag-only"),
|
||||
Status: types.ExpirationStatusEnabled,
|
||||
Filter: &types.LifecycleRuleFilter{
|
||||
And: &types.LifecycleRuleAndOperator{
|
||||
Prefix: aws.String("t/"),
|
||||
Tags: []types.Tag{
|
||||
{Key: aws.String("env"), Value: aws.String("temp")},
|
||||
},
|
||||
},
|
||||
},
|
||||
Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
|
||||
}},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
const taggedKey = "t/tagged.txt"
|
||||
const untaggedKey = "t/untagged.txt"
|
||||
putObject(t, c, bucket, taggedKey, "tagged")
|
||||
putObject(t, c, bucket, untaggedKey, "untagged")
|
||||
|
||||
// Stamp the tag on one of them.
|
||||
_, err = c.PutObjectTagging(context.Background(), &s3.PutObjectTaggingInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(taggedKey),
|
||||
Tagging: &types.Tagging{TagSet: []types.Tag{
|
||||
{Key: aws.String("env"), Value: aws.String("temp")},
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
backdateMtime(t, fc, bucket, taggedKey, 30)
|
||||
backdateMtime(t, fc, bucket, untaggedKey, 30)
|
||||
|
||||
out := runLifecycleShard(t)
|
||||
t.Logf("shell output:\n%s", out)
|
||||
|
||||
require.Eventuallyf(t, func() bool {
|
||||
_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(taggedKey),
|
||||
})
|
||||
return isS3NotFound(err)
|
||||
}, 30*time.Second, 500*time.Millisecond, "tagged object must be expired")
|
||||
|
||||
_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket), Key: aws.String(untaggedKey),
|
||||
})
|
||||
require.NoError(t, err, "untagged object must remain")
|
||||
}
|
||||
@@ -704,27 +704,13 @@ func uniqueName(prefix string) string {
|
||||
// --- Test setup helpers ---
|
||||
|
||||
func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
// Allocate two extra ports for admin HTTP + gRPC. Without explicit
|
||||
// values mini falls back to 23646/33646, which collide across the
|
||||
// sequential subtests in this package: each subtest spins up a fresh
|
||||
// mini in a goroutine, but the previous mini's admin goroutine is
|
||||
// still binding 23646 — so the next test's admin can never become
|
||||
// ready and mini.go fatals on its readiness check.
|
||||
ports := testutil.MustAllocatePorts(t, 10)
|
||||
ports := testutil.MustAllocatePorts(t, 8)
|
||||
masterPort, masterGrpcPort := ports[0], ports[1]
|
||||
volumePort, volumeGrpcPort := ports[2], ports[3]
|
||||
filerPort, filerGrpcPort := ports[4], ports[5]
|
||||
s3Port, s3GrpcPort := ports[6], ports[7]
|
||||
adminPort, adminGrpcPort := ports[8], ports[9]
|
||||
|
||||
// Manually-managed temp dir (not t.TempDir()) so we control removal order:
|
||||
// the dir is removed inside Stop() AFTER the mini goroutine has fully
|
||||
// exited. Otherwise t.TempDir()'s RemoveAll cleanup races the admin
|
||||
// plugin worker, which keeps creating files under admin/plugin/job_types/
|
||||
// for ~1s after subtests finish, producing flaky "directory not empty"
|
||||
// failures (see seaweedfs CI run 25352039081).
|
||||
testDir, err := os.MkdirTemp("", "seaweed-policy-mini-")
|
||||
require.NoError(t, err)
|
||||
testDir := t.TempDir()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s3Endpoint := fmt.Sprintf("http://127.0.0.1:%d", s3Port)
|
||||
@@ -743,7 +729,7 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
|
||||
// Disable authentication for tests
|
||||
securityToml := filepath.Join(testDir, "security.toml")
|
||||
err = os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
|
||||
err := os.WriteFile(securityToml, []byte("# Empty security config\n"), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Configure credential store for IAM tests.
|
||||
@@ -786,8 +772,6 @@ enabled = true
|
||||
"-filer.port.grpc=" + strconv.Itoa(filerGrpcPort),
|
||||
"-s3.port=" + strconv.Itoa(s3Port),
|
||||
"-s3.port.grpc=" + strconv.Itoa(s3GrpcPort),
|
||||
"-admin.port=" + strconv.Itoa(adminPort),
|
||||
"-admin.port.grpc=" + strconv.Itoa(adminGrpcPort),
|
||||
"-webdav.port=0",
|
||||
"-admin.ui=false",
|
||||
"-master.volumeSizeLimitMB=32",
|
||||
@@ -799,12 +783,7 @@ enabled = true
|
||||
for _, cmd := range command.Commands {
|
||||
if cmd.Name() == "mini" && cmd.Run != nil {
|
||||
cmd.Flag.Parse(os.Args[1:])
|
||||
// MiniClusterCtx makes runMini observe our cancel: master/
|
||||
// volume/filer get this as their shutdownCtx, and the clients
|
||||
// state (admin/s3/webdav/plugin worker) chains from it.
|
||||
command.MiniClusterCtx = ctx
|
||||
cmd.Run(cmd, cmd.Flag.Args())
|
||||
command.MiniClusterCtx = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -813,8 +792,6 @@ enabled = true
|
||||
// Wait for S3
|
||||
if !testutil.WaitForService(cluster.s3Endpoint, 60*time.Second) {
|
||||
cancel()
|
||||
cluster.wg.Wait()
|
||||
os.RemoveAll(testDir)
|
||||
return nil, fmt.Errorf("timeout waiting for S3 at %s", cluster.s3Endpoint)
|
||||
}
|
||||
|
||||
@@ -822,8 +799,6 @@ enabled = true
|
||||
if os.Getenv("VOLUME_SERVER_IMPL") == "rust" {
|
||||
if err := cluster.startRustVolumeServer(t); err != nil {
|
||||
cancel()
|
||||
cluster.wg.Wait()
|
||||
os.RemoveAll(testDir)
|
||||
return nil, fmt.Errorf("failed to start Rust volume server: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -907,25 +882,10 @@ func (c *TestCluster) Stop() {
|
||||
if c.cancel != nil {
|
||||
c.cancel()
|
||||
}
|
||||
|
||||
// Wait for the mini goroutine to fully return before removing the data
|
||||
// dir. runMini observes MiniClusterCtx and returns once cancel fires; the
|
||||
// admin/s3/webdav/plugin-worker shutdown is gated by the same ctx via
|
||||
// resetMiniClients. The deadline is generous because admin shutdown can
|
||||
// take several seconds when graceful-stops are draining.
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
c.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(15 * time.Second):
|
||||
fmt.Println("Warning: TestCluster.Stop timed out waiting for mini goroutine")
|
||||
if c.isRunning {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Reset all mini flags so a subsequent in-process startMiniCluster sees
|
||||
// fresh state.
|
||||
// Simplified stop
|
||||
for _, cmd := range command.Commands {
|
||||
if cmd.Name() == "mini" {
|
||||
cmd.Flag.VisitAll(func(f *flag.Flag) {
|
||||
@@ -934,10 +894,6 @@ func (c *TestCluster) Stop() {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if c.dataDir != "" {
|
||||
os.RemoveAll(c.dataDir)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
|
||||
@@ -127,7 +127,6 @@ start-primary: check-deps
|
||||
-webdav.port=$(PRIMARY_WEBDAV_PORT) \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-s3.config=s3_config.json \
|
||||
-volume.allowUntrustedRemoteEndpoints \
|
||||
-dir=$(PRIMARY_DIR) \
|
||||
-ip=127.0.0.1 \
|
||||
-ip.bind=127.0.0.1 \
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN pip install --no-cache-dir \
|
||||
"deltalake==0.21.0" \
|
||||
"pyarrow==17.0.0" \
|
||||
"requests==2.32.3"
|
||||
|
||||
WORKDIR /app
|
||||
COPY delta_rs_writer.py /app/delta_rs_writer.py
|
||||
|
||||
ENTRYPOINT ["python3", "/app/delta_rs_writer.py"]
|
||||
@@ -1,104 +0,0 @@
|
||||
# Unity Catalog OSS integration tests
|
||||
|
||||
These tests run Unity Catalog OSS in Docker against an embedded SeaweedFS
|
||||
S3 endpoint. The `server.properties` mirrors the upstream playground at
|
||||
[`mds-in-a-box/unitycatalog-playground`](https://github.com/data-engineering-helpers/mds-in-a-box/tree/main/unitycatalog-playground).
|
||||
|
||||
| Test | Variant | Status |
|
||||
| --- | --- | --- |
|
||||
| `TestUnityCatalogDeltaIntegration` | static keys, `aws.masterRoleArn=` empty | passes; covers catalog/schema/EXTERNAL Delta CRUD against SeaweedFS-backed warehouse and asserts that UC's `/temporary-table-credentials` *cannot* vend usable creds with this configuration -- exactly the gap the playground reports. |
|
||||
| `TestUnityCatalogMasterRoleIntegration` | `aws.masterRoleArn=arn:aws:iam::000000000000:role/UnityCatalogVendedRole` | passes; proves SeaweedFS' STS endpoint accepts `sts:AssumeRole` for the role UC would use (Go SDK round-trip), and that UC starts and accepts CRUD when wired with the master-role config. UC's own StsClient still talks to real AWS regardless of `aws.endpoint` / `AWS_ENDPOINT_URL_STS` (UC bug, see below); that hop is logged via `t.Logf` rather than asserted. |
|
||||
| `TestUnityCatalogDeltaRsRoundTrip` | static keys + `delta-rs` Python client | passes; resolves table metadata through UC and writes/reads a real Delta table at the registered `storage_location` using `python:3.11-slim + deltalake` with the SeaweedFS test credentials. |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker available locally (the tests call `docker run` / `docker build` directly).
|
||||
- A `weed` binary at the repo root (`weed/weed`) or on `$PATH`.
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
go test -timeout 15m \
|
||||
-run 'TestUnityCatalog' \
|
||||
./test/s3tables/unity_catalog/...
|
||||
```
|
||||
|
||||
Pin a specific Unity Catalog image (defaults to
|
||||
`unitycatalog/unitycatalog:v0.4.0`):
|
||||
|
||||
```bash
|
||||
UC_IMAGE=unitycatalog/unitycatalog:main \
|
||||
go test -timeout 15m -run TestUnityCatalogDeltaIntegration \
|
||||
./test/s3tables/unity_catalog/...
|
||||
```
|
||||
|
||||
The tests self-skip when Docker is unavailable or no `weed` binary is on
|
||||
the path; running under `-short` also skips them.
|
||||
|
||||
## Why the static-key path can't vend usable creds
|
||||
|
||||
UC OSS' `AwsCredentialVendor.createPerBucketCredentialGenerator`:
|
||||
|
||||
```java
|
||||
if (config.getSessionToken() != null && !config.getSessionToken().isEmpty()) {
|
||||
return new AwsCredentialGenerator.StaticAwsCredentialGenerator(config);
|
||||
}
|
||||
return createStsCredentialGenerator(config);
|
||||
```
|
||||
|
||||
With `aws.masterRoleArn=` empty and `s3.sessionToken.0=` empty (this
|
||||
test's configuration), `/temporary-table-credentials` short-circuits with
|
||||
`"S3 bucket configuration not found."` before UC fires any STS call.
|
||||
Setting a stub `s3.sessionToken.0` switches UC to
|
||||
`StaticAwsCredentialGenerator` and the endpoint returns the static keys,
|
||||
but the response carries that stub session token -- SeaweedFS won't
|
||||
recognize it on the next S3 call, so the vended creds aren't usable for
|
||||
table I/O. Clients have to fall back to the static keys directly.
|
||||
|
||||
With `aws.masterRoleArn` set, UC's `AwsCredentialGenerator.StsAwsCredentialGenerator`
|
||||
builds the StsClient with only `.region(...)` and `.credentialsProvider(...)` --
|
||||
no `.endpointOverride()`. The SDK's generic env-var resolution doesn't kick in
|
||||
for that builder shape, so even with `AWS_ENDPOINT_URL_STS=...` (or the
|
||||
matching `aws.endpointUrlSts` Java property, or the catch-all
|
||||
`AWS_ENDPOINT_URL=...`) the StsClient still targets real AWS and gets back
|
||||
`InvalidClientTokenId`. Verified by pointing the env var at port 1: UC reports
|
||||
the same AWS-issued 403 that it reports against SeaweedFS, and a sniffer in
|
||||
front of SeaweedFS' STS port records zero traffic. SeaweedFS' STS handler
|
||||
itself works -- the Go SDK round-trip in `assumeRoleViaSeaweedFS` proves that
|
||||
against the same SeaweedFS instance.
|
||||
|
||||
UC's own AWS credential-vending tests don't catch this because they mock
|
||||
`StsClient` away entirely -- `BaseCRUDTestWithMockCredentials` injects a
|
||||
custom `stsClientBuilderSupplier` returning an `EchoAwsStsClient` that
|
||||
synthesizes credentials in-process, and `CloudCredentialVendorTest` uses
|
||||
`Mockito.mockStatic(StsClient.class)`. No test ever exercises the wire
|
||||
path between UC's Java SDK and a real STS endpoint, so the missing
|
||||
`endpointOverride` slipped through.
|
||||
|
||||
Fix is upstream in
|
||||
[unitycatalog/unitycatalog#1532](https://github.com/unitycatalog/unitycatalog/pull/1532),
|
||||
which adds an `aws.endpoint` property and applies it to both the StsClient
|
||||
and the S3Client builders. Until that lands, the master-role test logs
|
||||
the failure but does not assert it.
|
||||
|
||||
## What the tests actually validate today
|
||||
|
||||
- Unity Catalog accepts a SeaweedFS-backed `server.properties` and starts.
|
||||
- Catalog / schema / EXTERNAL Delta table CRUD all work against the
|
||||
SeaweedFS warehouse via the UC REST API.
|
||||
- SeaweedFS' STS endpoint correctly issues `sts:AssumeRole` credentials
|
||||
for the `UnityCatalogVendedRole` and those credentials are accepted on
|
||||
S3 round-trips (Go AWS SDK).
|
||||
- Delta-RS resolves a UC table's `storage_location` and can write/read Delta
|
||||
data through the SeaweedFS S3 endpoint with the test credentials.
|
||||
|
||||
## What is still pending
|
||||
|
||||
Nothing on the SeaweedFS side. The remaining gap (UC's StsClient ignoring
|
||||
endpoint config) needs a UC OSS patch upstream.
|
||||
|
||||
## MANAGED tables
|
||||
|
||||
Not exercised. UC OSS gates them behind `server.managed-table.enabled=true`
|
||||
and a two-step staging flow (`POST /staging-tables` then `POST /tables`);
|
||||
EXTERNAL Delta is the simpler path and what these tests cover.
|
||||
@@ -1,184 +0,0 @@
|
||||
package unity_catalog
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ucClient struct {
|
||||
base string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func newUCClient(base string) *ucClient {
|
||||
return &ucClient{base: base + ucAPIBase, http: &http.Client{Timeout: 30 * time.Second}}
|
||||
}
|
||||
|
||||
type ucCreateCatalog struct {
|
||||
Name string `json:"name"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
StorageRoot string `json:"storage_root,omitempty"`
|
||||
}
|
||||
|
||||
type ucCatalogInfo struct {
|
||||
Name string `json:"name"`
|
||||
StorageRoot string `json:"storage_root"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type ucCreateSchema struct {
|
||||
Name string `json:"name"`
|
||||
CatalogName string `json:"catalog_name"`
|
||||
}
|
||||
|
||||
type ucSchemaInfo struct {
|
||||
Name string `json:"name"`
|
||||
CatalogName string `json:"catalog_name"`
|
||||
FullName string `json:"full_name"`
|
||||
SchemaID string `json:"schema_id"`
|
||||
}
|
||||
|
||||
type ucColumn struct {
|
||||
Name string `json:"name"`
|
||||
TypeText string `json:"type_text"`
|
||||
TypeJSON string `json:"type_json,omitempty"`
|
||||
TypeName string `json:"type_name"`
|
||||
Position int `json:"position"`
|
||||
Nullable bool `json:"nullable"`
|
||||
}
|
||||
|
||||
type ucCreateTable struct {
|
||||
Name string `json:"name"`
|
||||
CatalogName string `json:"catalog_name"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
TableType string `json:"table_type"`
|
||||
DataSourceFormat string `json:"data_source_format"`
|
||||
Columns []ucColumn `json:"columns"`
|
||||
StorageLocation string `json:"storage_location,omitempty"`
|
||||
}
|
||||
|
||||
type ucTableInfo struct {
|
||||
Name string `json:"name"`
|
||||
CatalogName string `json:"catalog_name"`
|
||||
SchemaName string `json:"schema_name"`
|
||||
TableType string `json:"table_type"`
|
||||
DataSourceFormat string `json:"data_source_format"`
|
||||
Columns []ucColumn `json:"columns"`
|
||||
StorageLocation string `json:"storage_location"`
|
||||
TableID string `json:"table_id"`
|
||||
}
|
||||
|
||||
type ucListTablesResp struct {
|
||||
Tables []ucTableInfo `json:"tables"`
|
||||
}
|
||||
|
||||
type ucAwsCreds struct {
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
SessionToken string `json:"session_token"`
|
||||
}
|
||||
|
||||
type ucTempCreds struct {
|
||||
AwsTempCredentials *ucAwsCreds `json:"aws_temp_credentials"`
|
||||
ExpirationTime int64 `json:"expiration_time"`
|
||||
}
|
||||
|
||||
func (c *ucClient) createCatalog(ctx context.Context, in ucCreateCatalog) (*ucCatalogInfo, error) {
|
||||
var out ucCatalogInfo
|
||||
return &out, c.do(ctx, http.MethodPost, "/catalogs", in, &out)
|
||||
}
|
||||
|
||||
func (c *ucClient) deleteCatalog(ctx context.Context, name string) error {
|
||||
q := url.Values{"force": []string{"true"}}
|
||||
return c.do(ctx, http.MethodDelete, "/catalogs/"+url.PathEscape(name)+"?"+q.Encode(), nil, nil)
|
||||
}
|
||||
|
||||
func (c *ucClient) createSchema(ctx context.Context, in ucCreateSchema) (*ucSchemaInfo, error) {
|
||||
var out ucSchemaInfo
|
||||
return &out, c.do(ctx, http.MethodPost, "/schemas", in, &out)
|
||||
}
|
||||
|
||||
func (c *ucClient) deleteSchema(ctx context.Context, fullName string) error {
|
||||
q := url.Values{"force": []string{"true"}}
|
||||
return c.do(ctx, http.MethodDelete, "/schemas/"+url.PathEscape(fullName)+"?"+q.Encode(), nil, nil)
|
||||
}
|
||||
|
||||
func (c *ucClient) createTable(ctx context.Context, in ucCreateTable) (*ucTableInfo, error) {
|
||||
var out ucTableInfo
|
||||
return &out, c.do(ctx, http.MethodPost, "/tables", in, &out)
|
||||
}
|
||||
|
||||
func (c *ucClient) getTable(ctx context.Context, fullName string) (*ucTableInfo, error) {
|
||||
var out ucTableInfo
|
||||
return &out, c.do(ctx, http.MethodGet, "/tables/"+url.PathEscape(fullName), nil, &out)
|
||||
}
|
||||
|
||||
func (c *ucClient) listTables(ctx context.Context, catalog, schema string) ([]ucTableInfo, error) {
|
||||
var out ucListTablesResp
|
||||
q := url.Values{
|
||||
"catalog_name": []string{catalog},
|
||||
"schema_name": []string{schema},
|
||||
}
|
||||
path := "/tables?" + q.Encode()
|
||||
if err := c.do(ctx, http.MethodGet, path, nil, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Tables, nil
|
||||
}
|
||||
|
||||
func (c *ucClient) deleteTable(ctx context.Context, fullName string) error {
|
||||
return c.do(ctx, http.MethodDelete, "/tables/"+url.PathEscape(fullName), nil, nil)
|
||||
}
|
||||
|
||||
func (c *ucClient) generateTemporaryTableCredentials(ctx context.Context, tableID, op string) (*ucTempCreds, error) {
|
||||
body := map[string]string{"table_id": tableID, "operation": op}
|
||||
var out ucTempCreds
|
||||
return &out, c.do(ctx, http.MethodPost, "/temporary-table-credentials", body, &out)
|
||||
}
|
||||
|
||||
func (c *ucClient) do(ctx context.Context, method, path string, in any, out any) error {
|
||||
var body io.Reader
|
||||
if in != nil {
|
||||
buf, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal %s %s: %w", method, path, err)
|
||||
}
|
||||
body = bytes.NewReader(buf)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if in != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// 10 MiB cap is well above any reasonable Unity Catalog response and
|
||||
// keeps a runaway server from OOMing the test runner.
|
||||
respBytes, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %s %s: %w", method, path, err)
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("%s %s: status %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(respBytes)))
|
||||
}
|
||||
if out == nil || len(respBytes) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(respBytes, out); err != nil {
|
||||
return fmt.Errorf("decode %s %s: %w (body=%s)", method, path, err, string(respBytes))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package unity_catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
// TestUnityCatalogDeltaRsRoundTrip writes and reads a real Delta table at
|
||||
// the registered storage_location using the delta-rs Python library inside
|
||||
// a Docker container. The script resolves table metadata from UC and uses the
|
||||
// test SeaweedFS credentials as delta-rs storage_options.
|
||||
func TestUnityCatalogDeltaRsRoundTrip(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("docker not available")
|
||||
}
|
||||
|
||||
env := newTestEnv(t)
|
||||
defer env.cleanup(t)
|
||||
|
||||
t.Log(">>> starting SeaweedFS for delta-rs test...")
|
||||
env.startSeaweedFS(t, "")
|
||||
t.Log(">>> SeaweedFS ready")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
s3c := env.newHostS3Client(t, ctx)
|
||||
if _, err := s3c.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(ucWarehouse)}); err != nil {
|
||||
t.Fatalf("create warehouse bucket: %v", err)
|
||||
}
|
||||
|
||||
t.Log(">>> starting Unity Catalog server...")
|
||||
env.startUnityCatalog(t, ctx, ucServerOpts{})
|
||||
t.Log(">>> Unity Catalog ready")
|
||||
|
||||
uc := newUCClient(fmt.Sprintf("http://127.0.0.1:%d", env.ucHostPort))
|
||||
|
||||
suffix := time.Now().UnixNano()
|
||||
catalogName := fmt.Sprintf("seaweed_uc_delta_rs_%d", suffix)
|
||||
schemaName := fmt.Sprintf("ns_%d", suffix)
|
||||
tableName := fmt.Sprintf("delta_rs_%d", suffix)
|
||||
tableLocation := fmt.Sprintf("s3://%s/%s/%s/%s", ucWarehouse, ucWarehouseKey, schemaName, tableName)
|
||||
|
||||
defer func() {
|
||||
_ = uc.deleteTable(context.Background(), catalogName+"."+schemaName+"."+tableName)
|
||||
_ = uc.deleteSchema(context.Background(), catalogName+"."+schemaName)
|
||||
_ = uc.deleteCatalog(context.Background(), catalogName)
|
||||
}()
|
||||
|
||||
if _, err := uc.createCatalog(ctx, ucCreateCatalog{
|
||||
Name: catalogName,
|
||||
StorageRoot: fmt.Sprintf("s3://%s/%s", ucWarehouse, ucWarehouseKey),
|
||||
}); err != nil {
|
||||
t.Fatalf("create catalog: %v", err)
|
||||
}
|
||||
if _, err := uc.createSchema(ctx, ucCreateSchema{Name: schemaName, CatalogName: catalogName}); err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
created, err := uc.createTable(ctx, ucCreateTable{
|
||||
Name: tableName,
|
||||
CatalogName: catalogName,
|
||||
SchemaName: schemaName,
|
||||
TableType: "EXTERNAL",
|
||||
DataSourceFormat: "DELTA",
|
||||
Columns: []ucColumn{
|
||||
{Name: "id", TypeText: "long", TypeName: "LONG", TypeJSON: `{"name":"id","type":"long","nullable":true,"metadata":{}}`, Position: 0, Nullable: true},
|
||||
{Name: "value", TypeText: "string", TypeName: "STRING", TypeJSON: `{"name":"value","type":"string","nullable":true,"metadata":{}}`, Position: 1, Nullable: true},
|
||||
},
|
||||
StorageLocation: tableLocation,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create table: %v", err)
|
||||
}
|
||||
|
||||
testDir, err := filepath.Abs(".")
|
||||
if err != nil {
|
||||
t.Fatalf("abs: %v", err)
|
||||
}
|
||||
imageTag := fmt.Sprintf("seaweed-uc-delta-rs:%d", time.Now().UnixNano())
|
||||
|
||||
build := exec.CommandContext(ctx, "docker", "build", "-t", imageTag, "-f", "Dockerfile.delta-rs", ".")
|
||||
build.Dir = testDir
|
||||
if out, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("docker build delta-rs image: %v\n%s", err, out)
|
||||
}
|
||||
defer func() { _ = exec.Command("docker", "rmi", "-f", imageTag).Run() }()
|
||||
|
||||
ucURLForContainer := fmt.Sprintf("http://host.docker.internal:%d", env.ucHostPort)
|
||||
s3EndpointForContainer := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
|
||||
cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-e", "UC_URL="+ucURLForContainer,
|
||||
"-e", "S3_ENDPOINT="+s3EndpointForContainer,
|
||||
"-e", "UC_CATALOG="+catalogName,
|
||||
"-e", "UC_SCHEMA="+schemaName,
|
||||
"-e", "UC_TABLE="+tableName,
|
||||
"-e", "UC_TABLE_ID="+created.TableID,
|
||||
"-e", "AWS_ACCESS_KEY_ID="+env.accessKey,
|
||||
"-e", "AWS_SECRET_ACCESS_KEY="+env.secretKey,
|
||||
imageTag,
|
||||
)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("delta-rs container failed: %v\noutput:\n%s", err, out)
|
||||
}
|
||||
got := string(out)
|
||||
if !strings.Contains(got, "DELTA_RS_OK rows=3") {
|
||||
t.Fatalf("expected DELTA_RS_OK rows=3 in output; got:\n%s", got)
|
||||
}
|
||||
t.Logf("delta-rs output: %s", strings.TrimSpace(got))
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
"""Round-trip a Delta table through Unity Catalog using delta-rs.
|
||||
|
||||
Reads connection settings from environment variables:
|
||||
|
||||
UC_URL Unity Catalog REST base, e.g. http://host.docker.internal:8080
|
||||
S3_ENDPOINT SeaweedFS S3 endpoint reachable from the container
|
||||
UC_CATALOG Catalog name to operate against
|
||||
UC_SCHEMA Schema name
|
||||
UC_TABLE Table name
|
||||
UC_TABLE_ID Optional: pre-fetched table_id; falls back to GET /tables/{full_name}
|
||||
AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
|
||||
Optional: direct S3 credentials. If omitted, the script
|
||||
asks UC for temporary table credentials.
|
||||
|
||||
Steps:
|
||||
|
||||
1. GET the table info from UC.
|
||||
2. Resolve S3 credentials from env or UC temporary-table-credentials.
|
||||
3. write_deltalake() a small pyarrow Table to storage_location.
|
||||
4. Read it back with DeltaTable() and assert row count.
|
||||
|
||||
Prints `DELTA_RS_OK rows=<n>` on success and exits 0; any failure exits 1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
import pyarrow as pa
|
||||
import requests
|
||||
from deltalake import DeltaTable, write_deltalake
|
||||
|
||||
|
||||
def must_env(name: str) -> str:
|
||||
value = os.environ.get(name)
|
||||
if not value:
|
||||
print(f"ERROR: required env var {name} is not set", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
return value
|
||||
|
||||
|
||||
def main() -> int:
|
||||
uc_url = must_env("UC_URL").rstrip("/")
|
||||
s3_endpoint = must_env("S3_ENDPOINT")
|
||||
catalog = must_env("UC_CATALOG")
|
||||
schema = must_env("UC_SCHEMA")
|
||||
table = must_env("UC_TABLE")
|
||||
table_id = os.environ.get("UC_TABLE_ID", "").strip()
|
||||
|
||||
full_name = f"{catalog}.{schema}.{table}"
|
||||
api = f"{uc_url}/api/2.1/unity-catalog"
|
||||
|
||||
encoded_full_name = urllib.parse.quote(full_name, safe="")
|
||||
r = requests.get(f"{api}/tables/{encoded_full_name}", timeout=10)
|
||||
r.raise_for_status()
|
||||
info = r.json()
|
||||
if not table_id:
|
||||
table_id = info.get("table_id") or ""
|
||||
storage_location = info.get("storage_location") or ""
|
||||
|
||||
if not table_id:
|
||||
print("ERROR: table_id is empty in UC response", file=sys.stderr)
|
||||
return 1
|
||||
if not storage_location:
|
||||
print("ERROR: storage_location is empty in UC response", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
access_key = os.environ.get("AWS_ACCESS_KEY_ID", "").strip()
|
||||
secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "").strip()
|
||||
session_token = os.environ.get("AWS_SESSION_TOKEN", "").strip()
|
||||
if bool(access_key) != bool(secret_key):
|
||||
print(
|
||||
"ERROR: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set together",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
if not access_key:
|
||||
creds_resp = requests.post(
|
||||
f"{api}/temporary-table-credentials",
|
||||
json={"table_id": table_id, "operation": "READ_WRITE"},
|
||||
timeout=15,
|
||||
)
|
||||
creds_resp.raise_for_status()
|
||||
creds = creds_resp.json().get("aws_temp_credentials") or {}
|
||||
access_key = creds.get("access_key_id") or ""
|
||||
secret_key = creds.get("secret_access_key") or ""
|
||||
session_token = creds.get("session_token") or ""
|
||||
if not access_key or not secret_key:
|
||||
print(
|
||||
"ERROR: missing aws_temp_credentials; "
|
||||
f"response status {creds_resp.status_code}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
storage_options = {
|
||||
"AWS_ACCESS_KEY_ID": access_key,
|
||||
"AWS_SECRET_ACCESS_KEY": secret_key,
|
||||
"AWS_REGION": "us-east-1",
|
||||
"AWS_ENDPOINT_URL": s3_endpoint,
|
||||
"AWS_ALLOW_HTTP": "true",
|
||||
# delta-rs requires this for non-S3 backends that lack atomic rename.
|
||||
"AWS_S3_ALLOW_UNSAFE_RENAME": "true",
|
||||
}
|
||||
if session_token:
|
||||
storage_options["AWS_SESSION_TOKEN"] = session_token
|
||||
|
||||
arrow_table = pa.table(
|
||||
{
|
||||
"id": pa.array([1, 2, 3], type=pa.int64()),
|
||||
"value": pa.array(["alpha", "beta", "gamma"], type=pa.string()),
|
||||
}
|
||||
)
|
||||
|
||||
write_deltalake(
|
||||
storage_location,
|
||||
arrow_table,
|
||||
storage_options=storage_options,
|
||||
mode="overwrite",
|
||||
)
|
||||
|
||||
dt = DeltaTable(storage_location, storage_options=storage_options)
|
||||
read_back = dt.to_pyarrow_table()
|
||||
rows = read_back.num_rows
|
||||
print(f"DELTA_RS_OK rows={rows} version={dt.version()}")
|
||||
return 0 if rows == arrow_table.num_rows else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = main()
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
# deltalake/pyarrow can abort during native teardown after a successful
|
||||
# round-trip in the short-lived test container. Preserve main's result.
|
||||
os._exit(exit_code)
|
||||
@@ -1,201 +0,0 @@
|
||||
package unity_catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sts"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
// TestUnityCatalogMasterRoleIntegration covers the master-role configuration
|
||||
// the upstream playground notes as not-yet-working
|
||||
// (https://github.com/data-engineering-helpers/mds-in-a-box/blob/main/unitycatalog-playground/etc/conf/server.properties#L45).
|
||||
//
|
||||
// It verifies two things end-to-end:
|
||||
//
|
||||
// 1. SeaweedFS' STS endpoint accepts sts:AssumeRole for the
|
||||
// UnityCatalogVendedRole, returning real STS-vended credentials. This
|
||||
// mirrors what the lakekeeper / polaris tests already exercise via the Go
|
||||
// AWS SDK and is the SeaweedFS-side prerequisite for UC's master-role
|
||||
// flow.
|
||||
//
|
||||
// 2. Unity Catalog OSS starts and accepts catalog/schema/EXTERNAL Delta
|
||||
// table CRUD when configured with `aws.masterRoleArn` set to that role
|
||||
// and AWS_ENDPOINT_URL_STS pointed at SeaweedFS.
|
||||
//
|
||||
// What it does NOT verify is the third hop -- UC's Java StsClient
|
||||
// reaching SeaweedFS' STS handler. UC's
|
||||
// AwsCredentialGenerator.StsAwsCredentialGenerator builds the StsClient
|
||||
// without an endpointOverride and doesn't propagate aws.endpoint to it, so
|
||||
// the SDK's generic AWS_ENDPOINT_URL_STS resolution doesn't kick in and the
|
||||
// request always targets real AWS, returning InvalidClientTokenId. Verified
|
||||
// by pointing AWS_ENDPOINT_URL_STS at port 1: same 403 either way, and a
|
||||
// sniffer in front of SeaweedFS records zero traffic. UC's own tests mock
|
||||
// StsClient out (BaseCRUDTestWithMockCredentials' EchoAwsStsClient,
|
||||
// Mockito.mockStatic in CloudCredentialVendorTest), so this never showed
|
||||
// up upstream. Fix is in unitycatalog/unitycatalog#1532; that bit is
|
||||
// logged via t.Logf and not asserted.
|
||||
func TestUnityCatalogMasterRoleIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("docker not available")
|
||||
}
|
||||
|
||||
env := newTestEnv(t)
|
||||
defer env.cleanup(t)
|
||||
|
||||
t.Log(">>> starting SeaweedFS with STS-enabled IAM config...")
|
||||
env.startSeaweedFS(t, stsEnabledIAMConfig(env.accessKey, env.secretKey))
|
||||
t.Log(">>> SeaweedFS ready")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
s3c := env.newHostS3Client(t, ctx)
|
||||
if _, err := s3c.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(ucWarehouse)}); err != nil {
|
||||
t.Fatalf("create warehouse bucket: %v", err)
|
||||
}
|
||||
|
||||
// Slice 1: prove SeaweedFS STS works for the role UC would use.
|
||||
t.Run("SeaweedFsAssumesUnityCatalogVendedRole", func(t *testing.T) {
|
||||
stsEndpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
|
||||
creds, err := assumeRoleViaSeaweedFS(ctx, stsEndpoint, env.accessKey, env.secretKey, ucVendedRoleArn)
|
||||
if err != nil {
|
||||
t.Fatalf("AssumeRole on SeaweedFS STS: %v", err)
|
||||
}
|
||||
if creds.SessionToken == "" {
|
||||
t.Fatalf("expected non-empty session_token from STS, got %+v", creds)
|
||||
}
|
||||
if creds.AccessKeyID == env.accessKey {
|
||||
t.Fatalf("STS returned the static admin access key; AssumeRole did not vend a new one")
|
||||
}
|
||||
|
||||
// Confirm the vended creds actually work for an S3 round-trip.
|
||||
s3v := env.newHostS3ClientWithCreds(t, ctx, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)
|
||||
probeKey := "ns-probe/master-role-probe.txt"
|
||||
if _, err := s3v.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(ucWarehouse),
|
||||
Key: aws.String(probeKey),
|
||||
Body: nil,
|
||||
}); err != nil {
|
||||
t.Fatalf("PutObject with sts-vended creds: %v", err)
|
||||
}
|
||||
_, _ = s3v.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(ucWarehouse),
|
||||
Key: aws.String(probeKey),
|
||||
})
|
||||
})
|
||||
|
||||
// Slice 2: UC starts with master-role config and CRUD works.
|
||||
stsEndpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
t.Log(">>> starting Unity Catalog server (master role)...")
|
||||
env.startUnityCatalog(t, ctx, ucServerOpts{
|
||||
MasterRoleArn: ucVendedRoleArn,
|
||||
ExtraEnv: map[string]string{
|
||||
"AWS_ENDPOINT_URL": stsEndpoint,
|
||||
"AWS_ENDPOINT_URL_STS": stsEndpoint,
|
||||
"AWS_ACCESS_KEY_ID": env.accessKey,
|
||||
"AWS_SECRET_ACCESS_KEY": env.secretKey,
|
||||
"AWS_REGION": "us-east-1",
|
||||
},
|
||||
})
|
||||
t.Log(">>> Unity Catalog ready")
|
||||
|
||||
uc := newUCClient(fmt.Sprintf("http://127.0.0.1:%d", env.ucHostPort))
|
||||
suffix := time.Now().UnixNano()
|
||||
catalogName := fmt.Sprintf("seaweed_uc_role_%d", suffix)
|
||||
schemaName := fmt.Sprintf("ns_%d", suffix)
|
||||
tableName := fmt.Sprintf("events_%d", suffix)
|
||||
tableLocation := fmt.Sprintf("s3://%s/%s/%s/%s", ucWarehouse, ucWarehouseKey, schemaName, tableName)
|
||||
|
||||
defer func() {
|
||||
_ = uc.deleteTable(context.Background(), catalogName+"."+schemaName+"."+tableName)
|
||||
_ = uc.deleteSchema(context.Background(), catalogName+"."+schemaName)
|
||||
_ = uc.deleteCatalog(context.Background(), catalogName)
|
||||
}()
|
||||
|
||||
t.Run("UnityCatalogAcceptsMasterRoleConfig", func(t *testing.T) {
|
||||
if _, err := uc.createCatalog(ctx, ucCreateCatalog{
|
||||
Name: catalogName,
|
||||
StorageRoot: fmt.Sprintf("s3://%s/%s", ucWarehouse, ucWarehouseKey),
|
||||
}); err != nil {
|
||||
t.Fatalf("create catalog: %v", err)
|
||||
}
|
||||
if _, err := uc.createSchema(ctx, ucCreateSchema{Name: schemaName, CatalogName: catalogName}); err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
created, err := uc.createTable(ctx, ucCreateTable{
|
||||
Name: tableName,
|
||||
CatalogName: catalogName,
|
||||
SchemaName: schemaName,
|
||||
TableType: "EXTERNAL",
|
||||
DataSourceFormat: "DELTA",
|
||||
Columns: []ucColumn{
|
||||
{Name: "id", TypeText: "long", TypeName: "LONG", TypeJSON: `{"name":"id","type":"long","nullable":true,"metadata":{}}`, Position: 0, Nullable: true},
|
||||
},
|
||||
StorageLocation: tableLocation,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create external delta table: %v", err)
|
||||
}
|
||||
if created.TableID == "" {
|
||||
t.Fatalf("expected table_id in CreateTable response, got %+v", created)
|
||||
}
|
||||
|
||||
// The actual UC -> SeaweedFS STS handoff is the still-unsolved slice.
|
||||
// Log what happens, but don't fail the test on it: SeaweedFS' STS is
|
||||
// known-good (slice 1 above), and UC's CRUD is known-good (this slice).
|
||||
_, ucCredErr := uc.generateTemporaryTableCredentials(ctx, created.TableID, "READ_WRITE")
|
||||
if ucCredErr != nil {
|
||||
t.Logf("UC StsClient still hits real AWS (UC bug, fix in unitycatalog/unitycatalog#1532): %v", ucCredErr)
|
||||
} else {
|
||||
t.Logf("UC StsClient honors endpoint override; unitycatalog/unitycatalog#1532 (or equivalent) is in this image.")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// assumeRoleViaSeaweedFS calls sts:AssumeRole against SeaweedFS' STS endpoint
|
||||
// using the Go AWS SDK, mirroring how lakekeeper/polaris tests prove
|
||||
// SeaweedFS' STS path works.
|
||||
func assumeRoleViaSeaweedFS(ctx context.Context, endpoint, accessKey, secretKey, roleArn string) (aws.Credentials, error) {
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion("us-east-1"),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
|
||||
)
|
||||
if err != nil {
|
||||
return aws.Credentials{}, fmt.Errorf("load aws config: %w", err)
|
||||
}
|
||||
|
||||
// Override the STS endpoint via service-specific BaseEndpoint instead of
|
||||
// the deprecated EndpointResolverWithOptions wired through the global
|
||||
// config. This is the supported path in aws-sdk-go-v2 since v1.21.
|
||||
client := sts.NewFromConfig(cfg, func(o *sts.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
})
|
||||
resp, err := client.AssumeRole(ctx, &sts.AssumeRoleInput{
|
||||
RoleArn: aws.String(roleArn),
|
||||
RoleSessionName: aws.String("uc-master-role-test"),
|
||||
})
|
||||
if err != nil {
|
||||
return aws.Credentials{}, err
|
||||
}
|
||||
if resp.Credentials == nil {
|
||||
return aws.Credentials{}, fmt.Errorf("AssumeRole returned no credentials")
|
||||
}
|
||||
return aws.Credentials{
|
||||
AccessKeyID: aws.ToString(resp.Credentials.AccessKeyId),
|
||||
SecretAccessKey: aws.ToString(resp.Credentials.SecretAccessKey),
|
||||
SessionToken: aws.ToString(resp.Credentials.SessionToken),
|
||||
Source: "seaweedfs-sts",
|
||||
}, nil
|
||||
}
|
||||
@@ -1,379 +0,0 @@
|
||||
package unity_catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
ucImageDefault = "unitycatalog/unitycatalog:v0.4.0"
|
||||
ucContainerCfg = "/home/unitycatalog/etc/conf/server.properties"
|
||||
ucStartupGrace = 90 * time.Second
|
||||
ucAPIBase = "/api/2.1/unity-catalog"
|
||||
ucWarehouse = "lakehouse"
|
||||
ucWarehouseKey = "warehouse"
|
||||
|
||||
// Role used by the master-role STS-vended variant of the test. The trust
|
||||
// policy is wide open so any caller can assume it; in production UC
|
||||
// would set this ARN as `aws.masterRoleArn`.
|
||||
ucVendedRoleArn = "arn:aws:iam::000000000000:role/UnityCatalogVendedRole"
|
||||
ucVendedRoleName = "UnityCatalogVendedRole"
|
||||
)
|
||||
|
||||
type testEnv struct {
|
||||
seaweedDir string
|
||||
weedBinary string
|
||||
dataDir string
|
||||
bindIP string
|
||||
s3Port int
|
||||
s3GrpcPort int
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
filerPort int
|
||||
filerGrpcPort int
|
||||
volumePort int
|
||||
volumeGrpcPort int
|
||||
weedProcess *exec.Cmd
|
||||
weedCancel context.CancelFunc
|
||||
|
||||
accessKey string
|
||||
secretKey string
|
||||
|
||||
ucImage string
|
||||
ucContainerID string
|
||||
ucHostPort int
|
||||
}
|
||||
|
||||
// ucServerOpts customizes the Unity Catalog server.properties and runtime
|
||||
// environment. Defaults match the upstream playground (static keys, no master
|
||||
// role).
|
||||
type ucServerOpts struct {
|
||||
// MasterRoleArn populates aws.masterRoleArn. Empty means UC falls back to
|
||||
// static aws.accessKey / aws.secretKey for storage operations.
|
||||
MasterRoleArn string
|
||||
// ExtraEnv adds environment variables to the UC container, useful for
|
||||
// AWS_ENDPOINT_URL_STS-style overrides.
|
||||
ExtraEnv map[string]string
|
||||
}
|
||||
|
||||
func newTestEnv(t *testing.T) *testEnv {
|
||||
t.Helper()
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("getwd: %v", err)
|
||||
}
|
||||
seaweedDir := wd
|
||||
for i := 0; i < 8; i++ {
|
||||
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
|
||||
break
|
||||
}
|
||||
seaweedDir = filepath.Dir(seaweedDir)
|
||||
}
|
||||
|
||||
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
|
||||
if _, err := os.Stat(weedBinary); err != nil {
|
||||
weedBinary = "weed"
|
||||
if _, err := exec.LookPath(weedBinary); err != nil {
|
||||
t.Skip("weed binary not found, skipping integration test")
|
||||
}
|
||||
}
|
||||
|
||||
dataDir, err := os.MkdirTemp("", "seaweed-uc-test-*")
|
||||
if err != nil {
|
||||
t.Fatalf("mkdtemp: %v", err)
|
||||
}
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
ports := testutil.MustAllocatePorts(t, 9)
|
||||
masterPort, masterGrpcPort := ports[0], ports[1]
|
||||
volumePort, volumeGrpcPort := ports[2], ports[3]
|
||||
filerPort, filerGrpcPort := ports[4], ports[5]
|
||||
s3Port, s3GrpcPort := ports[6], ports[7]
|
||||
ucHostPort := ports[8]
|
||||
|
||||
image := os.Getenv("UC_IMAGE")
|
||||
if image == "" {
|
||||
image = ucImageDefault
|
||||
}
|
||||
|
||||
return &testEnv{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
s3Port: s3Port,
|
||||
s3GrpcPort: s3GrpcPort,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
filerPort: filerPort,
|
||||
filerGrpcPort: filerGrpcPort,
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
accessKey: "admin",
|
||||
secretKey: "admin",
|
||||
ucImage: image,
|
||||
ucHostPort: ucHostPort,
|
||||
}
|
||||
}
|
||||
|
||||
// startSeaweedFS starts a `weed mini` instance. If iamJSON is empty, a minimal
|
||||
// admin-only IAM config is used. When iamJSON is non-empty, it is passed to
|
||||
// both -s3.config and -s3.iam.config so the STS handler is enabled (mirroring
|
||||
// the lakekeeper test).
|
||||
func (env *testEnv) startSeaweedFS(t *testing.T, iamJSON string) {
|
||||
t.Helper()
|
||||
|
||||
enableSTS := iamJSON != ""
|
||||
if iamJSON == "" {
|
||||
iamJSON = fmt.Sprintf(`{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [{"accessKey": %q, "secretKey": %q}],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
}
|
||||
]
|
||||
}`, env.accessKey, env.secretKey)
|
||||
}
|
||||
|
||||
iamConfigPath := filepath.Join(env.dataDir, "iam.json")
|
||||
if err := os.WriteFile(iamConfigPath, []byte(iamJSON), 0644); err != nil {
|
||||
t.Fatalf("write iam config: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
weedArgs := []string{"-v", "4", "mini",
|
||||
"-master.port", fmt.Sprintf("%d", env.masterPort),
|
||||
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
|
||||
"-volume.port", fmt.Sprintf("%d", env.volumePort),
|
||||
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
|
||||
"-filer.port", fmt.Sprintf("%d", env.filerPort),
|
||||
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
|
||||
"-s3.port", fmt.Sprintf("%d", env.s3Port),
|
||||
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
|
||||
"-s3.config", iamConfigPath,
|
||||
"-ip", env.bindIP,
|
||||
"-ip.bind", "0.0.0.0",
|
||||
"-dir", env.dataDir,
|
||||
}
|
||||
if enableSTS {
|
||||
weedArgs = append(weedArgs, "-s3.iam.config", iamConfigPath, "-s3.iam.readOnly=false")
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, weedArgs...)
|
||||
cmd.Dir = env.dataDir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("start weed mini: %v", err)
|
||||
}
|
||||
env.weedProcess = cmd
|
||||
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
|
||||
t.Fatalf("S3 API at 127.0.0.1:%d did not become ready", env.s3Port)
|
||||
}
|
||||
}
|
||||
|
||||
// startUnityCatalog launches the Unity Catalog OSS server in Docker against
|
||||
// the running SeaweedFS instance. It mirrors the upstream playground's
|
||||
// server.properties layout and bind-mounts only that single file (matching
|
||||
// docker-compose.yaml from the playground).
|
||||
func (env *testEnv) startUnityCatalog(t *testing.T, ctx context.Context, opts ucServerOpts) {
|
||||
t.Helper()
|
||||
|
||||
s3EndpointForContainer := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
|
||||
props := strings.Join([]string{
|
||||
"server.env=test",
|
||||
"server.authorization=disable",
|
||||
"server.cookie-timeout=PT1H",
|
||||
"server.managed-table.enabled=false",
|
||||
fmt.Sprintf("aws.masterRoleArn=%s", opts.MasterRoleArn),
|
||||
fmt.Sprintf("aws.accessKey=%s", env.accessKey),
|
||||
fmt.Sprintf("aws.secretKey=%s", env.secretKey),
|
||||
"aws.region=us-east-1",
|
||||
fmt.Sprintf("aws.endpoint=%s", s3EndpointForContainer),
|
||||
// UC keys perBucketS3Configs by NormalizedURL.from(bucketPath) and
|
||||
// looks up using the storageBase, which is "s3://<bucket>" (scheme +
|
||||
// authority only). The playground's "s3://lakehouse/warehouse" never
|
||||
// matches because of that asymmetry; the bucket-only form works.
|
||||
fmt.Sprintf("s3.bucketPath.0=s3://%s", ucWarehouse),
|
||||
"s3.region.0=us-east-1",
|
||||
fmt.Sprintf("s3.awsRoleArn.0=%s", opts.MasterRoleArn),
|
||||
fmt.Sprintf("s3.accessKey.0=%s", env.accessKey),
|
||||
fmt.Sprintf("s3.secretKey.0=%s", env.secretKey),
|
||||
fmt.Sprintf("s3.endpoint.0=%s", s3EndpointForContainer),
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
confDir := filepath.Join(env.dataDir, "uc-conf")
|
||||
if err := os.MkdirAll(confDir, 0755); err != nil {
|
||||
t.Fatalf("mkdir uc-conf: %v", err)
|
||||
}
|
||||
propsPath := filepath.Join(confDir, "server.properties")
|
||||
if err := os.WriteFile(propsPath, []byte(props), 0644); err != nil {
|
||||
t.Fatalf("write server.properties: %v", err)
|
||||
}
|
||||
|
||||
containerName := fmt.Sprintf("seaweed-unity-catalog-%d", time.Now().UnixNano())
|
||||
|
||||
args := []string{
|
||||
"run", "-d", "--rm",
|
||||
"--name", containerName,
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-p", fmt.Sprintf("%d:8080", env.ucHostPort),
|
||||
"-v", fmt.Sprintf("%s:%s:ro", propsPath, ucContainerCfg),
|
||||
"-e", "JAVA_OPTS=-Xmx1g",
|
||||
}
|
||||
for k, v := range opts.ExtraEnv {
|
||||
args = append(args, "-e", fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
args = append(args, env.ucImage)
|
||||
|
||||
out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("docker run unity-catalog: %v\n%s", err, out)
|
||||
}
|
||||
// docker run -d prefixes pull progress / warnings before the final container
|
||||
// ID line when the image isn't cached locally. Trim to the last whitespace-
|
||||
// separated token so the ID survives a fresh CI runner.
|
||||
fields := strings.Fields(string(out))
|
||||
if len(fields) == 0 {
|
||||
t.Fatalf("docker run produced no output; combined: %q", string(out))
|
||||
}
|
||||
env.ucContainerID = fields[len(fields)-1]
|
||||
t.Logf("unity-catalog container id: %s", env.ucContainerID)
|
||||
|
||||
probe := fmt.Sprintf("http://127.0.0.1:%d%s/catalogs", env.ucHostPort, ucAPIBase)
|
||||
deadline := time.Now().Add(ucStartupGrace)
|
||||
var lastErr error
|
||||
for time.Now().Before(deadline) {
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, probe, nil)
|
||||
resp, err := (&http.Client{Timeout: 3 * time.Second}).Do(req)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
return
|
||||
}
|
||||
lastErr = fmt.Errorf("status %d", resp.StatusCode)
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("ctx done while waiting for unity catalog: %v", ctx.Err())
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
logs, _ := exec.Command("docker", "logs", "--tail", "200", env.ucContainerID).CombinedOutput()
|
||||
t.Fatalf("unity catalog did not become ready: %v\nrecent logs:\n%s", lastErr, logs)
|
||||
}
|
||||
|
||||
func (env *testEnv) cleanup(t *testing.T) {
|
||||
if env.ucContainerID != "" {
|
||||
if logs, err := exec.Command("docker", "logs", "--tail", "200", env.ucContainerID).CombinedOutput(); err == nil {
|
||||
t.Logf("unity-catalog tail logs:\n%s", logs)
|
||||
}
|
||||
_ = exec.Command("docker", "rm", "-f", env.ucContainerID).Run()
|
||||
}
|
||||
if env.weedCancel != nil {
|
||||
env.weedCancel()
|
||||
}
|
||||
if env.weedProcess != nil {
|
||||
_ = env.weedProcess.Wait()
|
||||
}
|
||||
if env.dataDir != "" {
|
||||
_ = os.RemoveAll(env.dataDir)
|
||||
}
|
||||
}
|
||||
|
||||
// -- S3 client helpers ---------------------------------------------------------
|
||||
|
||||
func (env *testEnv) newHostS3Client(t *testing.T, ctx context.Context) *s3.Client {
|
||||
t.Helper()
|
||||
return env.newHostS3ClientWithCreds(t, ctx, env.accessKey, env.secretKey, "")
|
||||
}
|
||||
|
||||
func (env *testEnv) newHostS3ClientWithCreds(t *testing.T, ctx context.Context, ak, sk, token string) *s3.Client {
|
||||
t.Helper()
|
||||
endpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion("us-east-1"),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(ak, sk, token)),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("load aws config: %v", err)
|
||||
}
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(endpoint)
|
||||
o.UsePathStyle = true
|
||||
})
|
||||
}
|
||||
|
||||
// stsEnabledIAMConfig returns an iam.json that defines an admin user, the
|
||||
// UnityCatalogVendedRole role with a permissive trust policy, and a FullAccess
|
||||
// policy attached to it. This is the SeaweedFS-side counterpart of the
|
||||
// `aws.masterRoleArn` configuration on the Unity Catalog server.
|
||||
func stsEnabledIAMConfig(accessKey, secretKey string) string {
|
||||
return fmt.Sprintf(`{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [{"accessKey": %q, "secretKey": %q}],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
}
|
||||
],
|
||||
"sts": {
|
||||
"tokenDuration": "12h",
|
||||
"maxSessionLength": "24h",
|
||||
"issuer": "seaweedfs-sts",
|
||||
"signingKey": "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz"
|
||||
},
|
||||
"roles": [
|
||||
{
|
||||
"roleName": %q,
|
||||
"roleArn": %q,
|
||||
"trustPolicy": {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "sts:AssumeRole"
|
||||
}
|
||||
]
|
||||
},
|
||||
"attachedPolicies": ["FullAccess"]
|
||||
}
|
||||
],
|
||||
"policies": [
|
||||
{
|
||||
"name": "FullAccess",
|
||||
"document": {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Action": "*", "Resource": "*"}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}`, accessKey, secretKey, ucVendedRoleName, ucVendedRoleArn)
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
// Package unity_catalog provides integration tests that run Unity Catalog
|
||||
// OSS in Docker against a SeaweedFS S3 backend. The base test mirrors the
|
||||
// configuration used by the upstream UC playground at
|
||||
// https://github.com/data-engineering-helpers/mds-in-a-box/tree/main/unitycatalog-playground:
|
||||
// static aws.accessKey/aws.secretKey, no master role, EXTERNAL Delta tables.
|
||||
package unity_catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/testutil"
|
||||
)
|
||||
|
||||
// TestUnityCatalogDeltaIntegration brings up SeaweedFS, runs Unity Catalog OSS
|
||||
// in Docker against it, and exercises catalog/schema/table CRUD plus
|
||||
// temporary-table-credentials and storage I/O via vended credentials. The UC
|
||||
// server is configured with static keys (aws.masterRoleArn empty), matching
|
||||
// the playground's working configuration when targeting SeaweedFS.
|
||||
func TestUnityCatalogDeltaIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in -short mode")
|
||||
}
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("docker not available")
|
||||
}
|
||||
|
||||
env := newTestEnv(t)
|
||||
defer env.cleanup(t)
|
||||
|
||||
t.Log(">>> starting SeaweedFS for Unity Catalog test...")
|
||||
env.startSeaweedFS(t, "")
|
||||
t.Log(">>> SeaweedFS ready")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
s3c := env.newHostS3Client(t, ctx)
|
||||
if _, err := s3c.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(ucWarehouse)}); err != nil {
|
||||
t.Fatalf("create warehouse bucket: %v", err)
|
||||
}
|
||||
|
||||
t.Log(">>> starting Unity Catalog server (static keys)...")
|
||||
env.startUnityCatalog(t, ctx, ucServerOpts{})
|
||||
t.Log(">>> Unity Catalog ready")
|
||||
|
||||
uc := newUCClient(fmt.Sprintf("http://127.0.0.1:%d", env.ucHostPort))
|
||||
|
||||
suffix := time.Now().UnixNano()
|
||||
catalogName := fmt.Sprintf("seaweed_uc_%d", suffix)
|
||||
schemaName := fmt.Sprintf("ns_%d", suffix)
|
||||
tableName := fmt.Sprintf("events_%d", suffix)
|
||||
tableLocation := fmt.Sprintf("s3://%s/%s/%s/%s", ucWarehouse, ucWarehouseKey, schemaName, tableName)
|
||||
|
||||
defer func() {
|
||||
_ = uc.deleteTable(context.Background(), catalogName+"."+schemaName+"."+tableName)
|
||||
_ = uc.deleteSchema(context.Background(), catalogName+"."+schemaName)
|
||||
_ = uc.deleteCatalog(context.Background(), catalogName)
|
||||
}()
|
||||
|
||||
t.Run("CreateCatalog", func(t *testing.T) {
|
||||
got, err := uc.createCatalog(ctx, ucCreateCatalog{
|
||||
Name: catalogName,
|
||||
Comment: "seaweedfs integration test",
|
||||
StorageRoot: fmt.Sprintf("s3://%s/%s", ucWarehouse, ucWarehouseKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create catalog: %v", err)
|
||||
}
|
||||
if got.Name != catalogName {
|
||||
t.Fatalf("catalog name = %q, want %q", got.Name, catalogName)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CreateSchema", func(t *testing.T) {
|
||||
got, err := uc.createSchema(ctx, ucCreateSchema{
|
||||
Name: schemaName,
|
||||
CatalogName: catalogName,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
if got.Name != schemaName || got.CatalogName != catalogName {
|
||||
t.Fatalf("schema = %+v, want name=%q catalog=%q", got, schemaName, catalogName)
|
||||
}
|
||||
})
|
||||
|
||||
var createdTable ucTableInfo
|
||||
t.Run("CreateExternalDeltaTable", func(t *testing.T) {
|
||||
columns := []ucColumn{
|
||||
{Name: "id", TypeText: "long", TypeName: "LONG", TypeJSON: `{"name":"id","type":"long","nullable":true,"metadata":{}}`, Position: 0, Nullable: true},
|
||||
{Name: "value", TypeText: "string", TypeName: "STRING", TypeJSON: `{"name":"value","type":"string","nullable":true,"metadata":{}}`, Position: 1, Nullable: true},
|
||||
}
|
||||
got, err := uc.createTable(ctx, ucCreateTable{
|
||||
Name: tableName,
|
||||
CatalogName: catalogName,
|
||||
SchemaName: schemaName,
|
||||
TableType: "EXTERNAL",
|
||||
DataSourceFormat: "DELTA",
|
||||
Columns: columns,
|
||||
StorageLocation: tableLocation,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create table: %v", err)
|
||||
}
|
||||
if got.TableType != "EXTERNAL" || got.DataSourceFormat != "DELTA" {
|
||||
t.Fatalf("table types = %s/%s, want EXTERNAL/DELTA", got.TableType, got.DataSourceFormat)
|
||||
}
|
||||
if got.StorageLocation == "" {
|
||||
t.Fatalf("table storage_location is empty in response: %+v", got)
|
||||
}
|
||||
createdTable = *got
|
||||
})
|
||||
|
||||
t.Run("GetTable", func(t *testing.T) {
|
||||
full := catalogName + "." + schemaName + "." + tableName
|
||||
got, err := uc.getTable(ctx, full)
|
||||
if err != nil {
|
||||
t.Fatalf("get table: %v", err)
|
||||
}
|
||||
if got.Name != tableName {
|
||||
t.Fatalf("get table name = %q, want %q", got.Name, tableName)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ListTables", func(t *testing.T) {
|
||||
tables, err := uc.listTables(ctx, catalogName, schemaName)
|
||||
if err != nil {
|
||||
t.Fatalf("list tables: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, tbl := range tables {
|
||||
if tbl.Name == tableName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("created table %q not found in list", tableName)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("TemporaryTableCredentialsRejected", func(t *testing.T) {
|
||||
// With aws.masterRoleArn empty AND no s3.sessionToken.0 set, UC OSS
|
||||
// always tries to AssumeRole via its internal StsClient (see
|
||||
// AwsCredentialVendor.createPerBucketCredentialGenerator). Against a
|
||||
// non-AWS endpoint, that call doesn't reach a real STS, so UC returns
|
||||
// "S3 bucket configuration not found." or an STS-side error. This is
|
||||
// the gap users hit at <https://github.com/data-engineering-helpers/mds-in-a-box/blob/main/unitycatalog-playground/etc/conf/server.properties#L45>:
|
||||
// "with simple S3 access and secret keys, Unity Catalog does not seem
|
||||
// to work."
|
||||
//
|
||||
// The assertion is therefore inverted: we expect a non-nil error from
|
||||
// /temporary-table-credentials with this configuration. A future
|
||||
// variant can pin s3.sessionToken.0 (UC's StaticAwsCredentialGenerator
|
||||
// path) once SeaweedFS' SigV4 path tolerates the vended session token.
|
||||
if createdTable.TableID == "" {
|
||||
t.Fatalf("created table has empty table_id; cannot request temporary credentials")
|
||||
}
|
||||
_, err := uc.generateTemporaryTableCredentials(ctx, createdTable.TableID, "READ_WRITE")
|
||||
if err == nil {
|
||||
t.Fatalf("expected /temporary-table-credentials to fail with the static-key playground configuration; it succeeded unexpectedly")
|
||||
}
|
||||
t.Logf("expected failure (UC static-key path requires AWS STS): %v", err)
|
||||
})
|
||||
|
||||
t.Run("DeleteTableSchemaCatalog", func(t *testing.T) {
|
||||
if err := uc.deleteTable(ctx, catalogName+"."+schemaName+"."+tableName); err != nil {
|
||||
t.Fatalf("delete table: %v", err)
|
||||
}
|
||||
if err := uc.deleteSchema(ctx, catalogName+"."+schemaName); err != nil {
|
||||
t.Fatalf("delete schema: %v", err)
|
||||
}
|
||||
if err := uc.deleteCatalog(ctx, catalogName); err != nil {
|
||||
t.Fatalf("delete catalog: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -184,38 +184,6 @@ func (c *Cluster) Stop() {
|
||||
})
|
||||
}
|
||||
|
||||
// RestartVolumeServer kills the volume server and starts a new one against
|
||||
// the same data dirs and ports. The master keeps running. The previous run's
|
||||
// volume.log is moved to volume.log.previous so the first run's logs survive
|
||||
// a second-run startup failure.
|
||||
func (c *Cluster) RestartVolumeServer() {
|
||||
c.testingTB.Helper()
|
||||
stopProcess(c.volumeCmd)
|
||||
c.volumeCmd = nil
|
||||
// Rotate the log; absent on the first call after a clean start, which
|
||||
// is fine — startVolume will create it. Any real filesystem failure
|
||||
// surfaces immediately on the next os.Create in startVolume.
|
||||
oldLog := filepath.Join(c.logsDir, "volume.log")
|
||||
_ = os.Rename(oldLog, filepath.Join(c.logsDir, "volume.log.previous"))
|
||||
if err := c.startVolume(c.volumeDataDirs); err != nil {
|
||||
c.testingTB.Fatalf("restart volume server: %v", err)
|
||||
}
|
||||
if err := c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
|
||||
c.testingTB.Fatalf("wait for volume admin readiness after restart: %v\nvolume log tail:\n%s", err, c.tailLog("volume.log"))
|
||||
}
|
||||
if err := c.waitForTCP(c.VolumeGRPCAddress()); err != nil {
|
||||
c.testingTB.Fatalf("wait for volume grpc readiness after restart: %v\nvolume log tail:\n%s", err, c.tailLog("volume.log"))
|
||||
}
|
||||
}
|
||||
|
||||
// StopVolumeServer kills the volume server but leaves the master and data
|
||||
// dirs alone. Pair with RestartVolumeServer or Stop.
|
||||
func (c *Cluster) StopVolumeServer() {
|
||||
c.testingTB.Helper()
|
||||
stopProcess(c.volumeCmd)
|
||||
c.volumeCmd = nil
|
||||
}
|
||||
|
||||
func (c *Cluster) startMaster(dataDir string) error {
|
||||
logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
|
||||
if err != nil {
|
||||
@@ -264,10 +232,6 @@ func (c *Cluster) startVolume(dataDirs []string) error {
|
||||
"-readMode=" + c.profile.ReadMode,
|
||||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
|
||||
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
|
||||
// Integration tests deliberately exercise loopback S3 endpoints
|
||||
// (the test rig boots weed-mini next to the volume server); allow
|
||||
// the SSRF guard to be bypassed for them.
|
||||
"-volume.allowUntrustedRemoteEndpoints",
|
||||
}
|
||||
if c.profile.InflightUploadTimeout > 0 {
|
||||
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
|
||||
|
||||
@@ -256,8 +256,6 @@ func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
|
||||
"-readMode=" + c.profile.ReadMode,
|
||||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
|
||||
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
|
||||
// Integration tests deliberately exercise loopback S3 endpoints; allow the SSRF guard to be bypassed for them.
|
||||
"-volume.allowUntrustedRemoteEndpoints",
|
||||
}
|
||||
if c.profile.InflightUploadTimeout > 0 {
|
||||
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
|
||||
|
||||
@@ -227,8 +227,6 @@ func (c *MultiVolumeCluster) startVolume(index int, dataDir string) error {
|
||||
"-readMode=" + c.profile.ReadMode,
|
||||
"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
|
||||
"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
|
||||
// Integration tests deliberately exercise loopback S3 endpoints; allow the SSRF guard to be bypassed for them.
|
||||
"-volume.allowUntrustedRemoteEndpoints",
|
||||
}
|
||||
if c.profile.InflightUploadTimeout > 0 {
|
||||
args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
|
||||
|
||||
@@ -8,9 +8,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
@@ -324,18 +321,15 @@ func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Volume servers only track their masters as ping peers, so drive the
|
||||
// success path through the master target and rely on the master->volume
|
||||
// admission for cross-type coverage.
|
||||
pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: cluster.MasterType,
|
||||
Target: clusterHarness.MasterAddress(),
|
||||
TargetType: cluster.VolumeServerType,
|
||||
Target: clusterHarness.VolumeServerAddress(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Ping master from volume server failed: %v", err)
|
||||
t.Fatalf("Ping target volume server failed: %v", err)
|
||||
}
|
||||
if pingResp.GetRemoteTimeNs() == 0 {
|
||||
t.Fatalf("expected remote timestamp from ping master")
|
||||
t.Fatalf("expected remote timestamp from ping target volume server")
|
||||
}
|
||||
|
||||
if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
|
||||
@@ -405,21 +399,20 @@ func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// The volume server gates Ping on a known peer list. Unknown target types
|
||||
// and addresses outside that list are rejected with InvalidArgument
|
||||
// before any outbound dial is attempted.
|
||||
_, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
unknownResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: "unknown-type",
|
||||
Target: "127.0.0.1:12345",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("Ping unknown target type should be rejected by admission")
|
||||
if err != nil {
|
||||
t.Fatalf("Ping unknown target type should not return grpc error, got: %v", err)
|
||||
}
|
||||
if got := status.Code(err); got != codes.InvalidArgument {
|
||||
t.Fatalf("Ping unknown target type expected InvalidArgument, got %s: %v", got, err)
|
||||
if unknownResp.GetRemoteTimeNs() != 0 {
|
||||
t.Fatalf("Ping unknown target type expected remote_time_ns=0, got %d", unknownResp.GetRemoteTimeNs())
|
||||
}
|
||||
if unknownResp.GetStopTimeNs() < unknownResp.GetStartTimeNs() {
|
||||
t.Fatalf("Ping unknown target type expected stop_time_ns >= start_time_ns")
|
||||
}
|
||||
|
||||
// Empty target stays as an unauthenticated self-liveness probe.
|
||||
emptyTargetResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("Ping empty target should not return grpc error, got: %v", err)
|
||||
@@ -431,30 +424,26 @@ func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
|
||||
t.Fatalf("Ping empty target expected stop_time_ns >= start_time_ns")
|
||||
}
|
||||
|
||||
// 127.0.0.1:1 is not in the seed/current master set, so admission rejects
|
||||
// the call before it can reach the network.
|
||||
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: cluster.MasterType,
|
||||
Target: "127.0.0.1:1",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("Ping master target should be rejected when not in the known-peer set")
|
||||
t.Fatalf("Ping master target should fail when target is unreachable")
|
||||
}
|
||||
if got := status.Code(err); got != codes.InvalidArgument {
|
||||
t.Fatalf("Ping unknown master expected InvalidArgument, got %s: %v", got, err)
|
||||
if !strings.Contains(err.Error(), "ping master") {
|
||||
t.Fatalf("Ping master unreachable error mismatch: %v", err)
|
||||
}
|
||||
|
||||
// Volume servers do not carry a peer-filer list at all; any filer target
|
||||
// is rejected at admission regardless of reachability.
|
||||
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: cluster.FilerType,
|
||||
Target: "127.0.0.1:1",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("Ping filer target should be rejected by admission")
|
||||
t.Fatalf("Ping filer target should fail when target is unreachable")
|
||||
}
|
||||
if got := status.Code(err); got != codes.InvalidArgument {
|
||||
t.Fatalf("Ping filer expected InvalidArgument, got %s: %v", got, err)
|
||||
if !strings.Contains(err.Error(), "ping filer") {
|
||||
t.Fatalf("Ping filer unreachable error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -490,10 +479,6 @@ func TestPingFilerTargetSuccess(t *testing.T) {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
// Volume servers do not maintain a peer-filer index, so drive the
|
||||
// filer-presence success path through the volume server's master peer:
|
||||
// reaching that master in a filer-bearing cluster is sufficient signal
|
||||
// that filer joined the cluster without breaking volume-server ping.
|
||||
clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
@@ -502,16 +487,16 @@ func TestPingFilerTargetSuccess(t *testing.T) {
|
||||
defer cancel()
|
||||
|
||||
resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: cluster.MasterType,
|
||||
Target: clusterHarness.MasterAddress(),
|
||||
TargetType: cluster.FilerType,
|
||||
Target: clusterHarness.FilerServerAddress(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Ping master from filer-bearing cluster failed: %v", err)
|
||||
t.Fatalf("Ping filer target success path failed: %v", err)
|
||||
}
|
||||
if resp.GetRemoteTimeNs() == 0 {
|
||||
t.Fatalf("Ping master expected non-zero remote time")
|
||||
t.Fatalf("Ping filer target expected non-zero remote time")
|
||||
}
|
||||
if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
|
||||
t.Fatalf("Ping master expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
|
||||
t.Fatalf("Ping filer target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,487 +0,0 @@
|
||||
package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
)
|
||||
|
||||
// TestEcLifecycleAcrossMultipleDisks drives encode, mount, read, drop-dat,
|
||||
// cross-disk redistribute + restart + reconcile, blob delete, and rebuild on
|
||||
// a 2-disk volume server.
|
||||
func TestEcLifecycleAcrossMultipleDisks(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
const (
|
||||
dataDirCount = 2
|
||||
volumeID = uint32(9500)
|
||||
collection = "ec-multi-disk"
|
||||
)
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
|
||||
dataDirs := clusterHarness.VolumeDataDirs()
|
||||
if len(dataDirs) != dataDirCount {
|
||||
t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
|
||||
}
|
||||
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
||||
|
||||
// populate
|
||||
type needleSpec struct {
|
||||
fid string
|
||||
payload []byte
|
||||
}
|
||||
needles := []needleSpec{
|
||||
{framework.NewFileID(volumeID, 9501, 0xAA000001), bytesOfLen(11, 0xA1)},
|
||||
{framework.NewFileID(volumeID, 9502, 0xAA000002), bytesOfLen(1024, 0xA2)},
|
||||
{framework.NewFileID(volumeID, 9503, 0xAA000003), bytesOfLen(63*1024+17, 0xA3)},
|
||||
{framework.NewFileID(volumeID, 9504, 0xAA000004), bytesOfLen(2, 0xA4)},
|
||||
{framework.NewFileID(volumeID, 9505, 0xAA000005), bytesOfLen(513*1024, 0xA5)},
|
||||
}
|
||||
httpClient := framework.NewHTTPClient()
|
||||
for _, n := range needles {
|
||||
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// encode — every shard lands on the .dat's disk
|
||||
if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
}); err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate: %v", err)
|
||||
}
|
||||
postGenerateLayout := scanShardLayout(t, dataDirs, collection, volumeID)
|
||||
if len(postGenerateLayout[0]) != erasure_coding.TotalShardsCount {
|
||||
t.Fatalf("post-generate: expected all %d shards on disk 0, got per-disk layout %v",
|
||||
erasure_coding.TotalShardsCount, postGenerateLayout)
|
||||
}
|
||||
if len(postGenerateLayout[1]) != 0 {
|
||||
t.Fatalf("post-generate: disk 1 should be empty, got %v", postGenerateLayout[1])
|
||||
}
|
||||
// .ecj is created lazily on first EC delete
|
||||
for _, side := range []string{".ecx", ".vif"} {
|
||||
if !fileExistsIn(dataDirs[0], collection, volumeID, side) {
|
||||
t.Fatalf("post-generate: expected %s on disk 0", side)
|
||||
}
|
||||
}
|
||||
|
||||
// mount + read via EC
|
||||
if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
ShardIds: dataShardIds(),
|
||||
}); err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount data shards: %v", err)
|
||||
}
|
||||
for _, n := range needles {
|
||||
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-mount")
|
||||
}
|
||||
|
||||
// drop the original volume so EC shards are the only source
|
||||
if _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
|
||||
VolumeId: volumeID,
|
||||
}); err != nil {
|
||||
t.Fatalf("VolumeDelete (drop .dat): %v", err)
|
||||
}
|
||||
if fileExistsIn(dataDirs[0], collection, volumeID, ".dat") {
|
||||
t.Fatalf("VolumeDelete left .dat in place on disk 0")
|
||||
}
|
||||
for _, n := range needles {
|
||||
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-dat-delete")
|
||||
}
|
||||
|
||||
// move half the shards onto disk 1, leaving .ecx on disk 0
|
||||
clusterHarness.StopVolumeServer()
|
||||
const splitAt = 7
|
||||
for shard := 0; shard < splitAt; shard++ {
|
||||
movedFile(t, dataDirs[0], dataDirs[1], collection, volumeID, erasure_coding.ToExt(shard))
|
||||
}
|
||||
postMoveLayout := scanShardLayout(t, dataDirs, collection, volumeID)
|
||||
if len(postMoveLayout[0]) != erasure_coding.TotalShardsCount-splitAt {
|
||||
t.Fatalf("post-move: expected %d shards on disk 0, got %v", erasure_coding.TotalShardsCount-splitAt, postMoveLayout[0])
|
||||
}
|
||||
if len(postMoveLayout[1]) != splitAt {
|
||||
t.Fatalf("post-move: expected %d shards on disk 1, got %v", splitAt, postMoveLayout[1])
|
||||
}
|
||||
if !fileExistsIn(dataDirs[0], collection, volumeID, ".ecx") {
|
||||
t.Fatalf("post-move: .ecx must stay on disk 0 to exercise the cross-disk reconcile path")
|
||||
}
|
||||
if fileExistsIn(dataDirs[1], collection, volumeID, ".ecx") {
|
||||
t.Fatalf("post-move: .ecx leaked onto disk 1; reconcile path would not be exercised")
|
||||
}
|
||||
|
||||
clusterHarness.RestartVolumeServer()
|
||||
conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn2.Close()
|
||||
|
||||
// VolumeEcShardsInfo only sees one disk's EcVolume; filesystem layout is
|
||||
// the ground truth for the whole-store shard count.
|
||||
postReconcileLayout := scanShardLayout(t, dataDirs, collection, volumeID)
|
||||
if got, want := totalShardsInLayout(postReconcileLayout), erasure_coding.TotalShardsCount; got != want {
|
||||
t.Fatalf("post-reconcile: total shards on disk mismatch: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
|
||||
}
|
||||
if got, want := len(postReconcileLayout[0]), erasure_coding.TotalShardsCount-splitAt; got != want {
|
||||
t.Fatalf("post-reconcile: disk 0 shard count drift: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
|
||||
}
|
||||
if got, want := len(postReconcileLayout[1]), splitAt; got != want {
|
||||
t.Fatalf("post-reconcile: disk 1 shard count drift: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
|
||||
}
|
||||
if _, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
|
||||
VolumeId: volumeID,
|
||||
}); err != nil {
|
||||
t.Fatalf("VolumeEcShardsInfo after redistribute restart: %v", err)
|
||||
}
|
||||
for _, n := range needles {
|
||||
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-cross-disk-reconcile")
|
||||
}
|
||||
|
||||
// blob delete on a cross-disk layout
|
||||
deletedNeedle := needles[2]
|
||||
deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+deletedNeedle.fid, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("build delete request: %v", err)
|
||||
}
|
||||
deleteResp := framework.DoRequest(t, httpClient, deleteReq)
|
||||
_ = framework.ReadAllAndClose(t, deleteResp)
|
||||
if deleteResp.StatusCode != http.StatusAccepted {
|
||||
t.Fatalf("HTTP DELETE %s expected 202, got %d", deletedNeedle.fid, deleteResp.StatusCode)
|
||||
}
|
||||
readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), deletedNeedle.fid)
|
||||
_ = framework.ReadAllAndClose(t, readResp)
|
||||
if readResp.StatusCode == http.StatusOK {
|
||||
t.Fatalf("HTTP GET on deleted EC needle expected non-200, got %d", readResp.StatusCode)
|
||||
}
|
||||
for _, n := range needles {
|
||||
if n.fid == deletedNeedle.fid {
|
||||
continue
|
||||
}
|
||||
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-blob-delete")
|
||||
}
|
||||
|
||||
// rebuild a shard whose inputs are split across two disks
|
||||
clusterHarness.StopVolumeServer()
|
||||
const repairTargetShard = 9 // sits on disk 0 after the split
|
||||
shardPath := filepath.Join(dataDirs[0], shardFileName(collection, volumeID, repairTargetShard))
|
||||
if _, statErr := os.Stat(shardPath); statErr != nil {
|
||||
t.Fatalf("repair setup: shard %d not on disk 0 (%s): %v", repairTargetShard, shardPath, statErr)
|
||||
}
|
||||
if err := os.Remove(shardPath); err != nil {
|
||||
t.Fatalf("repair setup: remove shard %d on disk 0: %v", repairTargetShard, err)
|
||||
}
|
||||
clusterHarness.RestartVolumeServer()
|
||||
conn3, grpcClient3 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn3.Close()
|
||||
|
||||
rebuildResp, err := grpcClient3.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsRebuild after cross-disk shard loss: %v", err)
|
||||
}
|
||||
if !containsUint32(rebuildResp.GetRebuiltShardIds(), repairTargetShard) {
|
||||
t.Fatalf("VolumeEcShardsRebuild expected to rebuild shard %d, got %v",
|
||||
repairTargetShard, rebuildResp.GetRebuiltShardIds())
|
||||
}
|
||||
if _, statErr := os.Stat(shardPath); statErr != nil {
|
||||
t.Fatalf("rebuild did not restore shard %d on disk 0 (%s): %v", repairTargetShard, shardPath, statErr)
|
||||
}
|
||||
|
||||
if _, err := grpcClient3.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
ShardIds: []uint32{repairTargetShard},
|
||||
}); err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount rebuilt shard %d: %v", repairTargetShard, err)
|
||||
}
|
||||
for _, n := range needles {
|
||||
if n.fid == deletedNeedle.fid {
|
||||
continue
|
||||
}
|
||||
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-repair")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcPartialShardsOnSiblingDiskCleanedUpOnRestart seeds a healthy .dat on
|
||||
// disk 0, plants the on-disk footprint of an interrupted EC encode on disk 1
|
||||
// (one shard plus .ecx / .ecj / .vif, no .dat), and asserts the startup prune
|
||||
// wipes disk 1 without touching disk 0.
|
||||
func TestEcPartialShardsOnSiblingDiskCleanedUpOnRestart(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
const (
|
||||
dataDirCount = 2
|
||||
volumeID = uint32(9501)
|
||||
collection = "ec-9478"
|
||||
partialShard = 1
|
||||
)
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
|
||||
dataDirs := clusterHarness.VolumeDataDirs()
|
||||
if len(dataDirs) != dataDirCount {
|
||||
t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
|
||||
}
|
||||
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
needleFid := framework.NewFileID(volumeID, 7777, 0xCAFEBABE)
|
||||
needlePayload := bytesOfLen(4096, 0xC7)
|
||||
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), needleFid, needlePayload)
|
||||
_ = framework.ReadAllAndClose(t, upResp)
|
||||
if upResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("seed upload expected 201, got %d", upResp.StatusCode)
|
||||
}
|
||||
|
||||
datPath := filepath.Join(dataDirs[0], datFileName(collection, volumeID))
|
||||
datInfo, err := os.Stat(datPath)
|
||||
if err != nil {
|
||||
t.Fatalf("seed setup: .dat must exist on disk 0 (%s): %v", datPath, err)
|
||||
}
|
||||
// prune's size guard refuses cleanup if the sibling .dat looks truncated
|
||||
if datInfo.Size() == 0 {
|
||||
t.Fatalf("seed setup: .dat on disk 0 is zero bytes")
|
||||
}
|
||||
|
||||
clusterHarness.StopVolumeServer()
|
||||
plantPartialEc(t, dataDirs[1], collection, volumeID, partialShard, datInfo.Size())
|
||||
|
||||
preRestartLayout := scanShardLayout(t, dataDirs, collection, volumeID)
|
||||
if len(preRestartLayout[0]) != 0 {
|
||||
t.Fatalf("seed setup: disk 0 should hold no EC shards before restart, got %v", preRestartLayout[0])
|
||||
}
|
||||
if len(preRestartLayout[1]) != 1 {
|
||||
t.Fatalf("seed setup: disk 1 should hold exactly one planted shard, got %v", preRestartLayout[1])
|
||||
}
|
||||
|
||||
clusterHarness.RestartVolumeServer()
|
||||
|
||||
conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn2.Close()
|
||||
|
||||
// the volume admin port answers before the startup prune finishes
|
||||
postPruneLayout := waitForLayout(t, dataDirs, collection, volumeID, func(layout map[int][]int) bool {
|
||||
return len(layout[1]) == 0
|
||||
}, 10*time.Second)
|
||||
|
||||
if len(postPruneLayout[1]) != 0 {
|
||||
t.Fatalf("partial EC shards survived restart on disk 1: %v", postPruneLayout[1])
|
||||
}
|
||||
// .vif is intentionally left behind: with no .ecx / shard to anchor it,
|
||||
// the next startup ignores it.
|
||||
for _, side := range []string{".ecx", ".ecj"} {
|
||||
if fileExistsIn(dataDirs[1], collection, volumeID, side) {
|
||||
t.Fatalf("%s survived prune on disk 1", side)
|
||||
}
|
||||
}
|
||||
if !fileExistsIn(dataDirs[0], collection, volumeID, ".dat") {
|
||||
t.Fatalf("prune wiped the healthy .dat on disk 0")
|
||||
}
|
||||
if !fileExistsIn(dataDirs[0], collection, volumeID, ".idx") {
|
||||
t.Fatalf("prune wiped the healthy .idx on disk 0")
|
||||
}
|
||||
if _, err = grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
|
||||
VolumeId: volumeID,
|
||||
}); err == nil {
|
||||
t.Fatalf("VolumeEcShardsInfo returned success after prune; the partial EC mount should be gone")
|
||||
}
|
||||
verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), needleFid, needlePayload, "after-prune")
|
||||
}
|
||||
|
||||
func bytesOfLen(n int, seed byte) []byte {
|
||||
out := make([]byte, n)
|
||||
for i := range out {
|
||||
out[i] = byte(int(seed) + i*31)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dataShardIds() []uint32 {
|
||||
ids := make([]uint32, erasure_coding.DataShardsCount)
|
||||
for i := range ids {
|
||||
ids[i] = uint32(i)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func datFileName(collection string, volumeID uint32) string {
|
||||
if collection == "" {
|
||||
return fmt.Sprintf("%d.dat", volumeID)
|
||||
}
|
||||
return fmt.Sprintf("%s_%d.dat", collection, volumeID)
|
||||
}
|
||||
|
||||
func shardFileName(collection string, volumeID uint32, shardID int) string {
|
||||
if collection == "" {
|
||||
return fmt.Sprintf("%d%s", volumeID, erasure_coding.ToExt(shardID))
|
||||
}
|
||||
return fmt.Sprintf("%s_%d%s", collection, volumeID, erasure_coding.ToExt(shardID))
|
||||
}
|
||||
|
||||
func sideFileName(collection string, volumeID uint32, ext string) string {
|
||||
if collection == "" {
|
||||
return fmt.Sprintf("%d%s", volumeID, ext)
|
||||
}
|
||||
return fmt.Sprintf("%s_%d%s", collection, volumeID, ext)
|
||||
}
|
||||
|
||||
func fileExistsIn(dir, collection string, volumeID uint32, ext string) bool {
|
||||
_, err := os.Stat(filepath.Join(dir, sideFileName(collection, volumeID, ext)))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func scanShardLayout(t testing.TB, dataDirs []string, collection string, volumeID uint32) map[int][]int {
|
||||
t.Helper()
|
||||
out := make(map[int][]int, len(dataDirs))
|
||||
for diskIdx, dir := range dataDirs {
|
||||
out[diskIdx] = nil
|
||||
for shard := 0; shard < erasure_coding.TotalShardsCount; shard++ {
|
||||
if _, err := os.Stat(filepath.Join(dir, shardFileName(collection, volumeID, shard))); err == nil {
|
||||
out[diskIdx] = append(out[diskIdx], shard)
|
||||
}
|
||||
}
|
||||
sort.Ints(out[diskIdx])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func waitForLayout(t testing.TB, dataDirs []string, collection string, volumeID uint32, pred func(map[int][]int) bool, timeout time.Duration) map[int][]int {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var layout map[int][]int
|
||||
for time.Now().Before(deadline) {
|
||||
layout = scanShardLayout(t, dataDirs, collection, volumeID)
|
||||
if pred(layout) {
|
||||
return layout
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return layout
|
||||
}
|
||||
|
||||
func movedFile(t testing.TB, fromDir, toDir, collection string, volumeID uint32, ext string) {
|
||||
t.Helper()
|
||||
name := sideFileName(collection, volumeID, ext)
|
||||
src := filepath.Join(fromDir, name)
|
||||
dst := filepath.Join(toDir, name)
|
||||
if err := os.Rename(src, dst); err != nil {
|
||||
t.Fatalf("move %s → %s: %v", src, dst, err)
|
||||
}
|
||||
}
|
||||
|
||||
func totalShardsInLayout(layout map[int][]int) int {
|
||||
n := 0
|
||||
for _, ids := range layout {
|
||||
n += len(ids)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func containsUint32(s []uint32, v uint32) bool {
|
||||
for _, x := range s {
|
||||
if x == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func verifyHTTPRead(t testing.TB, client *http.Client, base, fid string, want []byte, stage string) {
|
||||
t.Helper()
|
||||
resp := framework.ReadBytes(t, client, base, fid)
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("[%s] GET %s expected 200, got %d", stage, fid, resp.StatusCode)
|
||||
}
|
||||
if len(body) != len(want) {
|
||||
t.Fatalf("[%s] GET %s size mismatch: got %d, want %d", stage, fid, len(body), len(want))
|
||||
}
|
||||
for i := range want {
|
||||
if body[i] != want[i] {
|
||||
t.Fatalf("[%s] GET %s byte %d mismatch: got %d, want %d", stage, fid, i, body[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// plantPartialEc writes the footprint of an interrupted EC encode (one shard
|
||||
// sized for datFileSize, plus .ecx / .ecj / .vif, no .dat) into dir.
|
||||
func plantPartialEc(t testing.TB, dir, collection string, volumeID uint32, shardID int, datFileSize int64) {
|
||||
t.Helper()
|
||||
shardPath := filepath.Join(dir, shardFileName(collection, volumeID, shardID))
|
||||
if f, err := os.Create(shardPath); err == nil {
|
||||
expectedShardSize := plantedShardSize(datFileSize)
|
||||
if err := f.Truncate(expectedShardSize); err != nil {
|
||||
f.Close()
|
||||
t.Fatalf("truncate planted shard: %v", err)
|
||||
}
|
||||
f.Close()
|
||||
} else {
|
||||
t.Fatalf("create planted shard %s: %v", shardPath, err)
|
||||
}
|
||||
for _, side := range []struct {
|
||||
ext string
|
||||
content []byte
|
||||
}{
|
||||
{".ecx", []byte("dummy ecx")},
|
||||
{".ecj", nil},
|
||||
{".vif", nil},
|
||||
} {
|
||||
p := filepath.Join(dir, sideFileName(collection, volumeID, side.ext))
|
||||
f, err := os.Create(p)
|
||||
if err != nil {
|
||||
t.Fatalf("create planted %s: %v", side.ext, err)
|
||||
}
|
||||
if side.content != nil {
|
||||
if _, err := f.Write(side.content); err != nil {
|
||||
f.Close()
|
||||
t.Fatalf("write planted %s: %v", side.ext, err)
|
||||
}
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// mirrors calculateExpectedShardSize in weed/storage/disk_location_ec.go
|
||||
func plantedShardSize(datFileSize int64) int64 {
|
||||
const dataShards = int64(erasure_coding.DataShardsCount)
|
||||
largeBatch := int64(erasure_coding.ErasureCodingLargeBlockSize) * dataShards
|
||||
numLarge := datFileSize / largeBatch
|
||||
shardSize := numLarge * int64(erasure_coding.ErasureCodingLargeBlockSize)
|
||||
remaining := datFileSize - numLarge*largeBatch
|
||||
if remaining > 0 {
|
||||
smallBatch := int64(erasure_coding.ErasureCodingSmallBlockSize) * dataShards
|
||||
numSmall := (remaining + smallBatch - 1) / smallBatch
|
||||
shardSize += numSmall * int64(erasure_coding.ErasureCodingSmallBlockSize)
|
||||
}
|
||||
return shardSize
|
||||
}
|
||||
@@ -14,9 +14,7 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -740,9 +738,6 @@ func (s *AdminServer) parseOrBuildClusterContext(raw json.RawMessage) (*plugin_p
|
||||
if len(contextMessage.VolumeGrpcAddresses) == 0 {
|
||||
contextMessage.VolumeGrpcAddresses = append(contextMessage.VolumeGrpcAddresses, fallback.VolumeGrpcAddresses...)
|
||||
}
|
||||
if len(contextMessage.S3GrpcAddresses) == 0 {
|
||||
contextMessage.S3GrpcAddresses = append(contextMessage.S3GrpcAddresses, fallback.S3GrpcAddresses...)
|
||||
}
|
||||
if contextMessage.Metadata == nil {
|
||||
contextMessage.Metadata = map[string]string{}
|
||||
}
|
||||
@@ -756,7 +751,6 @@ func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterConte
|
||||
MasterGrpcAddresses: make([]string, 0),
|
||||
FilerGrpcAddresses: make([]string, 0),
|
||||
VolumeGrpcAddresses: make([]string, 0),
|
||||
S3GrpcAddresses: make([]string, 0),
|
||||
Metadata: map[string]string{
|
||||
"source": "admin",
|
||||
},
|
||||
@@ -800,34 +794,9 @@ func (s *AdminServer) buildDefaultPluginClusterContext() *plugin_pb.ClusterConte
|
||||
glog.V(1).Infof("failed to build default plugin volume context: %v", err)
|
||||
}
|
||||
|
||||
s3Seen := map[string]struct{}{}
|
||||
if err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
|
||||
ClientType: cluster.S3Type,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, node := range resp.GetClusterNodes() {
|
||||
address := strings.TrimSpace(node.GetAddress())
|
||||
if address == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := s3Seen[address]; exists {
|
||||
continue
|
||||
}
|
||||
s3Seen[address] = struct{}{}
|
||||
clusterContext.S3GrpcAddresses = append(clusterContext.S3GrpcAddresses, address)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
glog.V(1).Infof("failed to list cluster S3 nodes: %v", err)
|
||||
}
|
||||
|
||||
sort.Strings(clusterContext.MasterGrpcAddresses)
|
||||
sort.Strings(clusterContext.FilerGrpcAddresses)
|
||||
sort.Strings(clusterContext.VolumeGrpcAddresses)
|
||||
sort.Strings(clusterContext.S3GrpcAddresses)
|
||||
|
||||
return clusterContext
|
||||
}
|
||||
@@ -936,8 +905,8 @@ func applyDescriptorDefaultsToPersistedConfig(
|
||||
if descriptor.AdminRuntimeDefaults != nil {
|
||||
runtime := config.AdminRuntime
|
||||
defaults := descriptor.AdminRuntimeDefaults
|
||||
if runtime.DetectionIntervalMinutes <= 0 {
|
||||
runtime.DetectionIntervalMinutes = defaults.DetectionIntervalMinutes
|
||||
if runtime.DetectionIntervalSeconds <= 0 {
|
||||
runtime.DetectionIntervalSeconds = defaults.DetectionIntervalSeconds
|
||||
}
|
||||
if runtime.DetectionTimeoutSeconds <= 0 {
|
||||
runtime.DetectionTimeoutSeconds = defaults.DetectionTimeoutSeconds
|
||||
|
||||
@@ -137,7 +137,7 @@ func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testi
|
||||
},
|
||||
},
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
DetectionIntervalMinutes: 60,
|
||||
DetectionIntervalSeconds: 60,
|
||||
DetectionTimeoutSeconds: 300,
|
||||
},
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testi
|
||||
if !ok || scriptKind.StringValue == "" {
|
||||
t.Fatalf("expected non-empty script default, got=%+v", script)
|
||||
}
|
||||
if config.AdminRuntime.DetectionIntervalMinutes != 60 {
|
||||
if config.AdminRuntime.DetectionIntervalSeconds != 60 {
|
||||
t.Fatalf("expected runtime detection interval default to be backfilled")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +123,6 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMet
|
||||
DeletedBytes: volInfo.DeletedByteCount,
|
||||
LastModified: time.Unix(int64(volInfo.ModifiedAtSecond), 0),
|
||||
IsReadOnly: volInfo.ReadOnly,
|
||||
HasRemoteCopy: volInfo.RemoteStorageName != "",
|
||||
IsECVolume: false, // Will be determined from volume structure
|
||||
ReplicaCount: 1, // Will be counted
|
||||
ExpectedReplicas: int(volInfo.ReplicaPlacement),
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
// Job types whose ExecuteJobRequest needs per-worker rate-allocation
|
||||
// metadata injected. Keyed by the job-type string so plugin.go's
|
||||
// generic dispatch path stays job-agnostic.
|
||||
//
|
||||
// To add a new job type to the share-allocation pipeline: register an
|
||||
// entry here that knows how to read its admin-config field(s) and
|
||||
// produce the metadata keys/values the worker reads.
|
||||
|
||||
// s3LifecycleClusterDeletesPerSecondKey, s3LifecycleClusterDeletesBurstKey,
|
||||
// s3LifecycleMetadataDeletesPerSecond, and s3LifecycleMetadataDeletesBurst
|
||||
// are the contract between admin and worker. The values must match the
|
||||
// constants exported from weed/worker/tasks/s3_lifecycle/cluster_rate_limit.go
|
||||
// — duplicated here as plain strings rather than imported so the admin
|
||||
// plugin package doesn't pull a dependency on the worker handler
|
||||
// package. A mismatch on either side would silently disable rate
|
||||
// limiting; tests pin the constants in both packages against the same
|
||||
// values.
|
||||
const (
|
||||
s3LifecycleJobType = "s3_lifecycle"
|
||||
s3LifecycleClusterDeletesPerSecondKey = "cluster_deletes_per_second"
|
||||
s3LifecycleClusterDeletesBurstKey = "cluster_deletes_burst"
|
||||
s3LifecycleMetadataDeletesPerSecond = "s3_lifecycle.deletes_per_second"
|
||||
s3LifecycleMetadataDeletesBurst = "s3_lifecycle.deletes_burst"
|
||||
)
|
||||
|
||||
// decorateClusterContextForJob returns a new ClusterContext with any
|
||||
// per-job-type metadata the admin needs to inject before the
|
||||
// ExecuteJobRequest is sent. Returns the input cc unchanged when no
|
||||
// decoration applies.
|
||||
//
|
||||
// Today only s3_lifecycle decorates; the function exists so a future
|
||||
// job type's plumbing slots in alongside without touching
|
||||
// executeJobWithExecutor.
|
||||
//
|
||||
// maxJobsPerDetection is the job-type's AdminRuntimeConfig.MaxJobsPerDetection
|
||||
// — the cap on how many parallel instances of this job the scheduler will
|
||||
// dispatch per detection cycle. For singleton jobs (s3_lifecycle has
|
||||
// MaxJobsPerDetection=1) only one worker is ever active, so the cluster
|
||||
// budget must go to that worker undivided. For parallel-dispatch jobs the
|
||||
// budget divides across the actually-running set, not across every
|
||||
// capable worker. The divisor is min(executors, maxJobsPerDetection),
|
||||
// clamped to ≥1.
|
||||
func (r *Plugin) decorateClusterContextForJob(cc *plugin_pb.ClusterContext, jobType string, adminConfigValues map[string]*plugin_pb.ConfigValue, maxJobsPerDetection int) *plugin_pb.ClusterContext {
|
||||
if cc == nil {
|
||||
return cc
|
||||
}
|
||||
if jobType != s3LifecycleJobType {
|
||||
return cc
|
||||
}
|
||||
rps := readNonNegativeInt(adminConfigValues, s3LifecycleClusterDeletesPerSecondKey)
|
||||
burst := readNonNegativeInt(adminConfigValues, s3LifecycleClusterDeletesBurstKey)
|
||||
if rps <= 0 {
|
||||
// Operator hasn't configured a cluster cap; nothing to allocate.
|
||||
// The worker treats missing metadata keys as "unlimited," which
|
||||
// is the legacy behavior.
|
||||
return cc
|
||||
}
|
||||
executors := r.registry.CountCapableExecutors(jobType)
|
||||
if executors <= 0 {
|
||||
// No executors means the job won't dispatch at all; metadata
|
||||
// would be discarded. Log so the case is visible in ops.
|
||||
glog.V(2).Infof("decorateClusterContext: %s rps=%d but no execute-capable workers; skipping allocation", jobType, rps)
|
||||
return cc
|
||||
}
|
||||
// Divide by the number of *concurrently-running* workers, not the
|
||||
// number of capable ones. A singleton job (maxJobs=1) gets the full
|
||||
// budget on its single active worker.
|
||||
activeWorkers := executors
|
||||
if maxJobsPerDetection > 0 && maxJobsPerDetection < activeWorkers {
|
||||
activeWorkers = maxJobsPerDetection
|
||||
}
|
||||
perWorkerRps := float64(rps) / float64(activeWorkers)
|
||||
perWorkerBurst := 0
|
||||
if burst > 0 {
|
||||
perWorkerBurst = burst / activeWorkers
|
||||
if perWorkerBurst < 1 {
|
||||
perWorkerBurst = 1
|
||||
}
|
||||
}
|
||||
|
||||
// Clone so we don't mutate the shared base context. The metadata
|
||||
// map is small; a fresh allocation per ExecuteJob is fine.
|
||||
out := cloneClusterContext(cc)
|
||||
if out.Metadata == nil {
|
||||
out.Metadata = map[string]string{}
|
||||
}
|
||||
out.Metadata[s3LifecycleMetadataDeletesPerSecond] = strconv.FormatFloat(perWorkerRps, 'f', -1, 64)
|
||||
if perWorkerBurst > 0 {
|
||||
out.Metadata[s3LifecycleMetadataDeletesBurst] = strconv.Itoa(perWorkerBurst)
|
||||
}
|
||||
glog.V(3).Infof("decorateClusterContext: %s rps=%d burst=%d executors=%d maxJobs=%d active=%d -> per-worker rps=%g burst=%d",
|
||||
jobType, rps, burst, executors, maxJobsPerDetection, activeWorkers, perWorkerRps, perWorkerBurst)
|
||||
return out
|
||||
}
|
||||
|
||||
// cloneClusterContext returns a shallow-but-safe copy: the top-level
|
||||
// fields are reassigned, and the Metadata map is duplicated so the
|
||||
// caller can mutate it without racing other consumers of the input.
|
||||
// Slices of strings (master/filer/volume/s3 addresses) are copied by
|
||||
// reference — those are treated as immutable elsewhere in the codebase.
|
||||
func cloneClusterContext(in *plugin_pb.ClusterContext) *plugin_pb.ClusterContext {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := &plugin_pb.ClusterContext{
|
||||
MasterGrpcAddresses: in.MasterGrpcAddresses,
|
||||
FilerGrpcAddresses: in.FilerGrpcAddresses,
|
||||
VolumeGrpcAddresses: in.VolumeGrpcAddresses,
|
||||
S3GrpcAddresses: in.S3GrpcAddresses,
|
||||
}
|
||||
if in.Metadata != nil {
|
||||
out.Metadata = make(map[string]string, len(in.Metadata))
|
||||
for k, v := range in.Metadata {
|
||||
out.Metadata[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readNonNegativeInt reads an int64 admin config value, treating
|
||||
// missing fields and non-int kinds as 0. Negative values are clamped
|
||||
// to 0 since the AdminConfigForm declares MinValue=0 on both fields.
|
||||
func readNonNegativeInt(values map[string]*plugin_pb.ConfigValue, field string) int {
|
||||
v, ok := values[field]
|
||||
if !ok || v == nil {
|
||||
return 0
|
||||
}
|
||||
switch k := v.Kind.(type) {
|
||||
case *plugin_pb.ConfigValue_Int64Value:
|
||||
if k.Int64Value < 0 {
|
||||
return 0
|
||||
}
|
||||
return int(k.Int64Value)
|
||||
case *plugin_pb.ConfigValue_DoubleValue:
|
||||
if k.DoubleValue < 0 {
|
||||
return 0
|
||||
}
|
||||
return int(k.DoubleValue)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// pluginWithExecutors returns a Plugin whose registry contains n
|
||||
// non-stale execute-capable workers for jobType. Helper for the
|
||||
// allocator tests. Bypasses UpsertFromHello so tests don't have to
|
||||
// build a full Hello message.
|
||||
func pluginWithExecutors(t *testing.T, jobType string, n int) *Plugin {
|
||||
t.Helper()
|
||||
reg := NewRegistry()
|
||||
now := time.Now()
|
||||
for i := 0; i < n; i++ {
|
||||
id := "worker-" + string(rune('a'+i))
|
||||
reg.sessions[id] = &WorkerSession{
|
||||
WorkerID: id,
|
||||
LastSeenAt: now,
|
||||
ConnectedAt: now,
|
||||
Capabilities: map[string]*plugin_pb.JobTypeCapability{
|
||||
jobType: {CanExecute: true},
|
||||
},
|
||||
}
|
||||
}
|
||||
return &Plugin{registry: reg}
|
||||
}
|
||||
|
||||
// adminConfig builds an int64 admin config map for the given fields.
|
||||
func adminConfig(pairs ...interface{}) map[string]*plugin_pb.ConfigValue {
|
||||
if len(pairs)%2 != 0 {
|
||||
panic("adminConfig expects key/value pairs")
|
||||
}
|
||||
out := map[string]*plugin_pb.ConfigValue{}
|
||||
for i := 0; i < len(pairs); i += 2 {
|
||||
key := pairs[i].(string)
|
||||
switch v := pairs[i+1].(type) {
|
||||
case int:
|
||||
out[key] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(v)}}
|
||||
case int64:
|
||||
out[key] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: v}}
|
||||
case float64:
|
||||
out[key] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: v}}
|
||||
default:
|
||||
panic("adminConfig: unsupported value type")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_NonS3LifecycleIsPassThrough(t *testing.T) {
|
||||
// Any job type other than s3_lifecycle gets the input cc back
|
||||
// unchanged. Future allocators add their own branch; the default
|
||||
// is pass-through.
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
in := &plugin_pb.ClusterContext{Metadata: map[string]string{"unrelated": "v"}}
|
||||
out := r.decorateClusterContextForJob(in, "some_other_job", adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100), 1)
|
||||
assert.Same(t, in, out, "non-allocator job type must return the same pointer")
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_RpsZeroSkipsAllocation(t *testing.T) {
|
||||
// rps=0 means "operator hasn't configured a cap"; the worker
|
||||
// treats missing keys as unlimited. We must NOT inject any
|
||||
// metadata (in particular, not "0") because that would force the
|
||||
// worker into a no-throughput state on a misconfigured cluster.
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType, adminConfig(s3LifecycleClusterDeletesPerSecondKey, 0), 1)
|
||||
if out.Metadata != nil {
|
||||
_, hasRps := out.Metadata[s3LifecycleMetadataDeletesPerSecond]
|
||||
assert.False(t, hasRps, "rps=0 must not write a deletes_per_second key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_NoExecutorsSkipsAllocation(t *testing.T) {
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 0)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType, adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100), 1)
|
||||
if out.Metadata != nil {
|
||||
_, hasRps := out.Metadata[s3LifecycleMetadataDeletesPerSecond]
|
||||
assert.False(t, hasRps, "0 executors must not write share metadata (would divide by zero)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_SingletonJobGetsFullBudget(t *testing.T) {
|
||||
// s3_lifecycle has MaxJobsPerDetection=1: only ONE worker runs the
|
||||
// job at a time. The cluster budget must go to that worker undivided
|
||||
// — dividing by N capable executors would starve the active worker
|
||||
// to 1/N of the configured rps. Pin the singleton behavior.
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
|
||||
adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100), 1)
|
||||
require.NotNil(t, out.Metadata)
|
||||
assert.Equal(t, "100", out.Metadata[s3LifecycleMetadataDeletesPerSecond], "singleton job: full budget to the single active worker")
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_SharedEvenlyWhenParallelLimited(t *testing.T) {
|
||||
// Hypothetical parallel-dispatch job type (maxJobs=4): budget
|
||||
// divides across the running-set, which equals min(executors,
|
||||
// maxJobs)=4. 100/4=25.
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
|
||||
adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100), 4)
|
||||
require.NotNil(t, out.Metadata)
|
||||
assert.Equal(t, "25", out.Metadata[s3LifecycleMetadataDeletesPerSecond], "maxJobs=4 across 4 executors = 25/worker")
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_MaxJobsExceedsExecutors(t *testing.T) {
|
||||
// maxJobs=10 but only 4 executors exist — the divisor is the
|
||||
// smaller value (executors) since you can't run more jobs in
|
||||
// parallel than there are workers to run them.
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
|
||||
adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100), 10)
|
||||
require.NotNil(t, out.Metadata)
|
||||
assert.Equal(t, "25", out.Metadata[s3LifecycleMetadataDeletesPerSecond])
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_BurstSharedWhenParallel(t *testing.T) {
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 2)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
|
||||
adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100, s3LifecycleClusterDeletesBurstKey, 20), 2)
|
||||
require.NotNil(t, out.Metadata)
|
||||
assert.Equal(t, "50", out.Metadata[s3LifecycleMetadataDeletesPerSecond])
|
||||
assert.Equal(t, "10", out.Metadata[s3LifecycleMetadataDeletesBurst])
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_BurstZeroOmitsKey(t *testing.T) {
|
||||
// burst=0 means "let the worker default it." Don't write the key —
|
||||
// the worker's parsePositiveInt would then take the unset path
|
||||
// and compute 2 × rps automatically.
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
|
||||
adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100, s3LifecycleClusterDeletesBurstKey, 0), 4)
|
||||
_, hasBurst := out.Metadata[s3LifecycleMetadataDeletesBurst]
|
||||
assert.False(t, hasBurst, "burst=0 must NOT write the burst key (worker default kicks in)")
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_BurstFloorIsOneWhenDividesBelowOne(t *testing.T) {
|
||||
// burst=1 across 4 active workers would round to 0; clamp to 1 so
|
||||
// the limiter doesn't become "single-token bucket that never refills."
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
in := &plugin_pb.ClusterContext{}
|
||||
out := r.decorateClusterContextForJob(in, s3LifecycleJobType,
|
||||
adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100, s3LifecycleClusterDeletesBurstKey, 1), 4)
|
||||
assert.Equal(t, "1", out.Metadata[s3LifecycleMetadataDeletesBurst])
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_DoesNotMutateInput(t *testing.T) {
|
||||
// The same base ClusterContext is shared across many parallel
|
||||
// ExecuteJob calls. The decorator must produce a fresh map so it
|
||||
// can't race / leak per-job metadata into the base.
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
baseMeta := map[string]string{"existing": "value"}
|
||||
in := &plugin_pb.ClusterContext{Metadata: baseMeta}
|
||||
_ = r.decorateClusterContextForJob(in, s3LifecycleJobType,
|
||||
adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100), 1)
|
||||
_, leaked := baseMeta[s3LifecycleMetadataDeletesPerSecond]
|
||||
assert.False(t, leaked, "decorator must not mutate the input metadata map")
|
||||
assert.Equal(t, "value", baseMeta["existing"])
|
||||
}
|
||||
|
||||
func TestDecorateClusterContext_NilInputPassesThrough(t *testing.T) {
|
||||
r := pluginWithExecutors(t, s3LifecycleJobType, 4)
|
||||
out := r.decorateClusterContextForJob(nil, s3LifecycleJobType, adminConfig(s3LifecycleClusterDeletesPerSecondKey, 100), 1)
|
||||
assert.Nil(t, out)
|
||||
}
|
||||
@@ -649,14 +649,6 @@ func (r *Plugin) executeJobWithExecutor(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Apply per-job-type cluster-allocation decoration (e.g. s3_lifecycle
|
||||
// divides cluster_deletes_per_second by min(workers, maxJobsPerDetection)
|
||||
// and ships the share via ClusterContext.Metadata). No-op for job
|
||||
// types without an allocator registered. MaxJobsPerDetection caps
|
||||
// the divisor so a singleton job (maxJobs=1) gets the full budget on
|
||||
// the single active worker, not 1/N of it.
|
||||
clusterContext = r.decorateClusterContextForJob(clusterContext, job.JobType, adminConfigValues, int(adminRuntime.GetMaxJobsPerDetection()))
|
||||
|
||||
completedCh := make(chan *plugin_pb.JobCompleted, 1)
|
||||
r.pendingExecutionMu.Lock()
|
||||
r.pendingExecution[requestID] = completedCh
|
||||
@@ -1044,7 +1036,7 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
|
||||
defaults := descriptor.AdminRuntimeDefaults
|
||||
adminRuntime = &plugin_pb.AdminRuntimeConfig{
|
||||
Enabled: defaults.Enabled,
|
||||
DetectionIntervalMinutes: defaults.DetectionIntervalMinutes,
|
||||
DetectionIntervalSeconds: defaults.DetectionIntervalSeconds,
|
||||
DetectionTimeoutSeconds: defaults.DetectionTimeoutSeconds,
|
||||
MaxJobsPerDetection: defaults.MaxJobsPerDetection,
|
||||
GlobalExecutionConcurrency: defaults.GlobalExecutionConcurrency,
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestEnsureJobTypeConfigFromDescriptorBootstrapsDefaults(t *testing.T) {
|
||||
},
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 60,
|
||||
DetectionIntervalSeconds: 60,
|
||||
DetectionTimeoutSeconds: 20,
|
||||
MaxJobsPerDetection: 30,
|
||||
GlobalExecutionConcurrency: 4,
|
||||
|
||||
@@ -466,7 +466,7 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
|
||||
}
|
||||
|
||||
policy := schedulerPolicy{
|
||||
DetectionInterval: durationFromMinutes(adminRuntime.DetectionIntervalMinutes, defaultScheduledDetectionInterval),
|
||||
DetectionInterval: durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval),
|
||||
DetectionTimeout: durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout),
|
||||
ExecutionTimeout: durationFromSeconds(adminRuntime.ExecutionTimeoutSeconds, defaultScheduledExecutionTimeout),
|
||||
JobTypeMaxRuntime: durationFromSeconds(adminRuntime.JobTypeMaxRuntimeSeconds, defaultScheduledJobTypeMaxRuntime),
|
||||
@@ -548,7 +548,7 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
|
||||
} else {
|
||||
state.Enabled = enabled
|
||||
if enabled {
|
||||
state.DetectionIntervalMinutes = minutesFromDuration(policy.DetectionInterval)
|
||||
state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval)
|
||||
state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout)
|
||||
state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout)
|
||||
state.JobTypeMaxRuntimeSeconds = secondsFromDuration(policy.JobTypeMaxRuntime)
|
||||
@@ -613,8 +613,8 @@ func deriveSchedulerAdminRuntime(
|
||||
// default instead of the handler's declared baseline.
|
||||
if descriptor != nil && descriptor.AdminRuntimeDefaults != nil {
|
||||
defaults := descriptor.AdminRuntimeDefaults
|
||||
if adminConfig.DetectionIntervalMinutes <= 0 {
|
||||
adminConfig.DetectionIntervalMinutes = defaults.DetectionIntervalMinutes
|
||||
if adminConfig.DetectionIntervalSeconds <= 0 {
|
||||
adminConfig.DetectionIntervalSeconds = defaults.DetectionIntervalSeconds
|
||||
}
|
||||
if adminConfig.DetectionTimeoutSeconds <= 0 {
|
||||
adminConfig.DetectionTimeoutSeconds = defaults.DetectionTimeoutSeconds
|
||||
@@ -648,7 +648,7 @@ func deriveSchedulerAdminRuntime(
|
||||
defaults := descriptor.AdminRuntimeDefaults
|
||||
return &plugin_pb.AdminRuntimeConfig{
|
||||
Enabled: defaults.Enabled,
|
||||
DetectionIntervalMinutes: defaults.DetectionIntervalMinutes,
|
||||
DetectionIntervalSeconds: defaults.DetectionIntervalSeconds,
|
||||
DetectionTimeoutSeconds: defaults.DetectionTimeoutSeconds,
|
||||
MaxJobsPerDetection: defaults.MaxJobsPerDetection,
|
||||
GlobalExecutionConcurrency: defaults.GlobalExecutionConcurrency,
|
||||
@@ -1366,13 +1366,6 @@ func durationFromSeconds(seconds int32, defaultValue time.Duration) time.Duratio
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
func durationFromMinutes(minutes int32, defaultValue time.Duration) time.Duration {
|
||||
if minutes <= 0 {
|
||||
return defaultValue
|
||||
}
|
||||
return time.Duration(minutes) * time.Minute
|
||||
}
|
||||
|
||||
func secondsFromDuration(duration time.Duration) int32 {
|
||||
if duration <= 0 {
|
||||
return 0
|
||||
@@ -1380,13 +1373,6 @@ func secondsFromDuration(duration time.Duration) int32 {
|
||||
return int32(duration / time.Second)
|
||||
}
|
||||
|
||||
func minutesFromDuration(duration time.Duration) int32 {
|
||||
if duration <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int32(duration / time.Minute)
|
||||
}
|
||||
|
||||
func waitForShutdownOrTimerWithContext(shutdown <-chan struct{}, ctx context.Context, duration time.Duration) bool {
|
||||
if duration <= 0 {
|
||||
return true
|
||||
|
||||
@@ -23,7 +23,7 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
|
||||
JobType: "vacuum",
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 30,
|
||||
DetectionIntervalSeconds: 30,
|
||||
DetectionTimeoutSeconds: 20,
|
||||
MaxJobsPerDetection: 123,
|
||||
GlobalExecutionConcurrency: 5,
|
||||
@@ -74,7 +74,7 @@ func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T
|
||||
JobType: "ec",
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 60,
|
||||
DetectionIntervalSeconds: 60,
|
||||
DetectionTimeoutSeconds: 25,
|
||||
MaxJobsPerDetection: 30,
|
||||
GlobalExecutionConcurrency: 4,
|
||||
@@ -397,7 +397,7 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
|
||||
JobType: jobType,
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 45,
|
||||
DetectionIntervalSeconds: 45,
|
||||
DetectionTimeoutSeconds: 30,
|
||||
MaxJobsPerDetection: 80,
|
||||
GlobalExecutionConcurrency: 3,
|
||||
@@ -448,8 +448,8 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
|
||||
if state.NextDetectionAt.Unix() != nextDetectionAt.Unix() {
|
||||
t.Fatalf("unexpected next detection time: got=%v want=%v", state.NextDetectionAt, nextDetectionAt)
|
||||
}
|
||||
if state.DetectionIntervalMinutes != 45 {
|
||||
t.Fatalf("unexpected detection interval: got=%d", state.DetectionIntervalMinutes)
|
||||
if state.DetectionIntervalSeconds != 45 {
|
||||
t.Fatalf("unexpected detection interval: got=%d", state.DetectionIntervalSeconds)
|
||||
}
|
||||
if state.DetectionTimeoutSeconds != 30 {
|
||||
t.Fatalf("unexpected detection timeout: got=%d", state.DetectionTimeoutSeconds)
|
||||
@@ -657,7 +657,7 @@ func TestRunLaneSchedulerIterationLockBehavior(t *testing.T) {
|
||||
JobType: tt.jobType,
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 1,
|
||||
DetectionIntervalSeconds: 1,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -142,31 +142,6 @@ func (r *Registry) HasCapableWorker(jobType string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// CountCapableExecutors returns the number of non-stale workers that
|
||||
// can EXECUTE the given job type. Used by per-job-type cluster
|
||||
// allocators (e.g. the s3_lifecycle delete-rate divider) to compute a
|
||||
// per-worker share at dispatch time. Returns 0 when no executor is
|
||||
// available — callers should treat that as "skip allocation" rather
|
||||
// than dividing by zero.
|
||||
func (r *Registry) CountCapableExecutors(jobType string) int {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
now := time.Now()
|
||||
n := 0
|
||||
for _, session := range r.sessions {
|
||||
if r.isSessionStaleLocked(session, now) {
|
||||
continue
|
||||
}
|
||||
capability := session.Capabilities[jobType]
|
||||
if capability == nil || !capability.CanExecute {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// DetectableJobTypes returns sorted job types that currently have at least one detect-capable worker.
|
||||
func (r *Registry) DetectableJobTypes() []string {
|
||||
r.mu.RLock()
|
||||
|
||||
@@ -47,7 +47,7 @@ type SchedulerJobTypeStatus struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DetectionInFlight bool `json:"detection_in_flight"`
|
||||
NextDetectionAt *time.Time `json:"next_detection_at,omitempty"`
|
||||
DetectionIntervalMinutes int32 `json:"detection_interval_minutes,omitempty"`
|
||||
DetectionIntervalSeconds int32 `json:"detection_interval_seconds,omitempty"`
|
||||
LastDetectedAt *time.Time `json:"last_detected_at,omitempty"`
|
||||
LastDetectedCount int `json:"last_detected_count,omitempty"`
|
||||
LastDetectionError string `json:"last_detection_error,omitempty"`
|
||||
@@ -347,7 +347,7 @@ func (r *Plugin) GetLaneSchedulerStatus(lane SchedulerLane) SchedulerStatus {
|
||||
Enabled: state.Enabled,
|
||||
DetectionInFlight: state.DetectionInFlight,
|
||||
NextDetectionAt: state.NextDetectionAt,
|
||||
DetectionIntervalMinutes: state.DetectionIntervalMinutes,
|
||||
DetectionIntervalSeconds: state.DetectionIntervalSeconds,
|
||||
}
|
||||
if !info.lastDetectedAt.IsZero() {
|
||||
jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
|
||||
@@ -430,7 +430,7 @@ func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
|
||||
Enabled: state.Enabled,
|
||||
DetectionInFlight: state.DetectionInFlight,
|
||||
NextDetectionAt: state.NextDetectionAt,
|
||||
DetectionIntervalMinutes: state.DetectionIntervalMinutes,
|
||||
DetectionIntervalSeconds: state.DetectionIntervalSeconds,
|
||||
}
|
||||
if !info.lastDetectedAt.IsZero() {
|
||||
jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
|
||||
|
||||
@@ -90,7 +90,7 @@ func TestGetLaneSchedulerStatusShowsActiveConcurrentLaneWork(t *testing.T) {
|
||||
JobType: jobType,
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 30,
|
||||
DetectionIntervalSeconds: 30,
|
||||
DetectionTimeoutSeconds: 15,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -88,7 +88,7 @@ type SchedulerJobTypeState struct {
|
||||
PolicyError string `json:"policy_error,omitempty"`
|
||||
DetectionInFlight bool `json:"detection_in_flight"`
|
||||
NextDetectionAt *time.Time `json:"next_detection_at,omitempty"`
|
||||
DetectionIntervalMinutes int32 `json:"detection_interval_minutes,omitempty"`
|
||||
DetectionIntervalSeconds int32 `json:"detection_interval_seconds,omitempty"`
|
||||
DetectionTimeoutSeconds int32 `json:"detection_timeout_seconds,omitempty"`
|
||||
ExecutionTimeoutSeconds int32 `json:"execution_timeout_seconds,omitempty"`
|
||||
JobTypeMaxRuntimeSeconds int32 `json:"job_type_max_runtime_seconds,omitempty"`
|
||||
|
||||
@@ -408,57 +408,6 @@ func TestECPlanningNotBlockedByUnrelatedBalance(t *testing.T) {
|
||||
"EC must still see all 4 disks even with an unrelated in-flight balance")
|
||||
}
|
||||
|
||||
// #9369: same-type physical disks collapse into one DiskInfo at the master;
|
||||
// the active topology must still expose one entry per physical disk_id.
|
||||
func TestECPlannerSeesEachPhysicalDisk(t *testing.T) {
|
||||
topology := NewActiveTopology(10)
|
||||
|
||||
const numServers = 7
|
||||
const disksPerServer = 2
|
||||
nodes := make([]*master_pb.DataNodeInfo, 0, numServers)
|
||||
for i := 1; i <= numServers; i++ {
|
||||
volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, disksPerServer)
|
||||
for d := uint32(0); d < disksPerServer; d++ {
|
||||
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
|
||||
Id: uint32(i*10 + int(d)),
|
||||
DiskId: d,
|
||||
DiskType: "hdd",
|
||||
})
|
||||
}
|
||||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||||
Id: fmt.Sprintf("127.0.0.1:%d", 8080+i),
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd": {
|
||||
DiskId: 0,
|
||||
VolumeCount: int64(disksPerServer),
|
||||
MaxVolumeCount: 200,
|
||||
VolumeInfos: volumeInfos,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
require.NoError(t, topology.UpdateTopology(&master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: nodes,
|
||||
}},
|
||||
}},
|
||||
}))
|
||||
|
||||
candidates := topology.GetDisksWithEffectiveCapacity(TaskTypeErasureCoding, "", 0)
|
||||
assert.Equal(t, numServers*disksPerServer, len(candidates))
|
||||
|
||||
seen := make(map[string]bool, len(candidates))
|
||||
for _, c := range candidates {
|
||||
key := fmt.Sprintf("%s:%d", c.NodeID, c.DiskID)
|
||||
assert.False(t, seen[key], "duplicate placement target %s", key)
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublicInterfaces tests the public interface methods
|
||||
func TestPublicInterfaces(t *testing.T) {
|
||||
topology := NewActiveTopology(10)
|
||||
|
||||
@@ -8,74 +8,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
// splitDiskInfoByPhysicalDisk returns one master_pb.DiskInfo per physical
|
||||
// disk_id observed in VolumeInfos / EcShardInfos. Multiple same-type physical
|
||||
// disks collapse to one DiskInfo at the master; per-volume/per-shard records
|
||||
// keep the original disk_id and are the authoritative signal here. Capacity
|
||||
// is split evenly — the wire format doesn't carry per-disk capacity yet.
|
||||
func splitDiskInfoByPhysicalDisk(diskInfo *master_pb.DiskInfo) []*master_pb.DiskInfo {
|
||||
if diskInfo == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Records with DiskId=0 and a non-zero outer DiskId belong to the outer
|
||||
// disk — handles older payloads / fixtures that omit the per-record id.
|
||||
normalize := func(id uint32) uint32 {
|
||||
if id == 0 && diskInfo.DiskId != 0 {
|
||||
return diskInfo.DiskId
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
diskIDs := make(map[uint32]struct{})
|
||||
for _, vi := range diskInfo.VolumeInfos {
|
||||
diskIDs[normalize(vi.DiskId)] = struct{}{}
|
||||
}
|
||||
for _, eci := range diskInfo.EcShardInfos {
|
||||
diskIDs[normalize(eci.DiskId)] = struct{}{}
|
||||
}
|
||||
if len(diskIDs) == 0 {
|
||||
diskIDs[diskInfo.DiskId] = struct{}{}
|
||||
}
|
||||
|
||||
if len(diskIDs) == 1 {
|
||||
for diskID := range diskIDs {
|
||||
if diskID == diskInfo.DiskId {
|
||||
return []*master_pb.DiskInfo{diskInfo}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
perDiskVolumes := make(map[uint32][]*master_pb.VolumeInformationMessage)
|
||||
for _, vi := range diskInfo.VolumeInfos {
|
||||
perDiskVolumes[normalize(vi.DiskId)] = append(perDiskVolumes[normalize(vi.DiskId)], vi)
|
||||
}
|
||||
perDiskShards := make(map[uint32][]*master_pb.VolumeEcShardInformationMessage)
|
||||
for _, eci := range diskInfo.EcShardInfos {
|
||||
perDiskShards[normalize(eci.DiskId)] = append(perDiskShards[normalize(eci.DiskId)], eci)
|
||||
}
|
||||
|
||||
count := int64(len(diskIDs))
|
||||
share := func(total int64) int64 { return total / count }
|
||||
|
||||
result := make([]*master_pb.DiskInfo, 0, len(diskIDs))
|
||||
for diskID := range diskIDs {
|
||||
result = append(result, &master_pb.DiskInfo{
|
||||
Type: diskInfo.Type,
|
||||
MaxVolumeCount: share(diskInfo.MaxVolumeCount),
|
||||
VolumeCount: int64(len(perDiskVolumes[diskID])),
|
||||
FreeVolumeCount: share(diskInfo.FreeVolumeCount),
|
||||
ActiveVolumeCount: share(diskInfo.ActiveVolumeCount),
|
||||
RemoteVolumeCount: share(diskInfo.RemoteVolumeCount),
|
||||
VolumeInfos: perDiskVolumes[diskID],
|
||||
EcShardInfos: perDiskShards[diskID],
|
||||
DiskId: diskID,
|
||||
Tags: append([]string(nil), diskInfo.Tags...),
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// CountTopologyResources counts datacenters, nodes, and disks in topology info
|
||||
func CountTopologyResources(topologyInfo *master_pb.TopologyInfo) (dcCount, nodeCount, diskCount int) {
|
||||
if topologyInfo == nil {
|
||||
@@ -141,28 +73,24 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
|
||||
disks: make(map[uint32]*activeDisk),
|
||||
}
|
||||
|
||||
// One activeDisk per physical disk_id (#9369): the master keys
|
||||
// DiskInfos by disk type, so same-type disks must be split out.
|
||||
// Add disks for this node
|
||||
for diskType, diskInfo := range nodeInfo.DiskInfos {
|
||||
perDiskInfos := splitDiskInfoByPhysicalDisk(diskInfo)
|
||||
for _, perDisk := range perDiskInfos {
|
||||
disk := &activeDisk{
|
||||
DiskInfo: &DiskInfo{
|
||||
NodeID: nodeInfo.Id,
|
||||
DiskID: perDisk.DiskId,
|
||||
DiskType: diskType,
|
||||
DataCenter: dc.Id,
|
||||
Rack: rack.Id,
|
||||
DiskInfo: perDisk,
|
||||
},
|
||||
}
|
||||
|
||||
diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, perDisk.DiskId)
|
||||
glog.V(3).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
|
||||
diskKey, nodeInfo.Id, perDisk.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, perDisk.VolumeCount, perDisk.MaxVolumeCount)
|
||||
node.disks[perDisk.DiskId] = disk
|
||||
at.disks[diskKey] = disk
|
||||
disk := &activeDisk{
|
||||
DiskInfo: &DiskInfo{
|
||||
NodeID: nodeInfo.Id,
|
||||
DiskID: diskInfo.DiskId,
|
||||
DiskType: diskType,
|
||||
DataCenter: dc.Id,
|
||||
Rack: rack.Id,
|
||||
DiskInfo: diskInfo,
|
||||
},
|
||||
}
|
||||
|
||||
diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
|
||||
glog.V(3).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
|
||||
diskKey, nodeInfo.Id, diskInfo.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, diskInfo.VolumeCount, diskInfo.MaxVolumeCount)
|
||||
node.disks[diskInfo.DiskId] = disk
|
||||
at.disks[diskKey] = disk
|
||||
}
|
||||
|
||||
at.nodes[nodeInfo.Id] = node
|
||||
@@ -277,27 +205,23 @@ func (at *ActiveTopology) rebuildIndexes() {
|
||||
at.volumeIndex = make(map[uint32][]string)
|
||||
at.ecShardIndex = make(map[uint32][]string)
|
||||
|
||||
// Index by the per-record DiskId (not the outer DiskInfo.DiskId) so the
|
||||
// keys match the per-physical-disk activeDisk entries — see #9369.
|
||||
// Rebuild indexes from current topology
|
||||
for _, dc := range at.topologyInfo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
for _, nodeInfo := range rack.DataNodeInfos {
|
||||
for _, diskInfo := range nodeInfo.DiskInfos {
|
||||
diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
|
||||
|
||||
// Index volumes
|
||||
for _, volumeInfo := range diskInfo.VolumeInfos {
|
||||
diskID := volumeInfo.DiskId
|
||||
if diskID == 0 && diskInfo.DiskId != 0 {
|
||||
diskID = diskInfo.DiskId
|
||||
}
|
||||
diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskID)
|
||||
at.volumeIndex[volumeInfo.Id] = append(at.volumeIndex[volumeInfo.Id], diskKey)
|
||||
volumeID := volumeInfo.Id
|
||||
at.volumeIndex[volumeID] = append(at.volumeIndex[volumeID], diskKey)
|
||||
}
|
||||
|
||||
// Index EC shards
|
||||
for _, ecShardInfo := range diskInfo.EcShardInfos {
|
||||
diskID := ecShardInfo.DiskId
|
||||
if diskID == 0 && diskInfo.DiskId != 0 {
|
||||
diskID = diskInfo.DiskId
|
||||
}
|
||||
diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskID)
|
||||
at.ecShardIndex[ecShardInfo.Id] = append(at.ecShardIndex[ecShardInfo.Id], diskKey)
|
||||
volumeID := ecShardInfo.Id
|
||||
at.ecShardIndex[volumeID] = append(at.ecShardIndex[volumeID], diskKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,13 +59,13 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
for i, crumb := range data.Breadcrumbs {
|
||||
if i == len(data.Breadcrumbs)-1 {
|
||||
<li class="breadcrumb-item active" aria-current="page">
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPathURL(crumb.Path)) } class="text-decoration-none">
|
||||
<a href={ dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", crumb.Path)) } class="text-decoration-none">
|
||||
{ crumb.Name }
|
||||
</a>
|
||||
</li>
|
||||
} else {
|
||||
<li class="breadcrumb-item">
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPathURL(crumb.Path)) } class="text-decoration-none">
|
||||
<a href={ dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", crumb.Path)) } class="text-decoration-none">
|
||||
if crumb.Name == "Root" {
|
||||
<i class="fas fa-home me-1"></i>
|
||||
}
|
||||
@@ -83,14 +83,14 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="fas fa-folder-open me-2"></i>
|
||||
if data.CurrentPath == "/" {
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPathURL("/")) } class="text-decoration-none text-primary">Root Directory</a>
|
||||
<a href={ dash.PUrl(ctx, "/files?path=/") } class="text-decoration-none text-primary">Root Directory</a>
|
||||
} else if data.CurrentPath == "/buckets" {
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPathURL("/buckets")) } class="text-decoration-none text-primary">Object Store Buckets Directory</a>
|
||||
<a href={ dash.PUrl(ctx, "/files?path=/buckets") } class="text-decoration-none text-primary">Object Store Buckets Directory</a>
|
||||
<a href={ dash.PUrl(ctx, "/object-store/buckets") } class="btn btn-sm btn-outline-primary ms-2">
|
||||
<i class="fas fa-cube me-1"></i>Manage Buckets
|
||||
</a>
|
||||
} else {
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPathURL(data.CurrentPath)) } class="text-decoration-none text-primary">{ filepath.Base(data.CurrentPath) }</a>
|
||||
<a href={ dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", data.CurrentPath)) } class="text-decoration-none text-primary">{ filepath.Base(data.CurrentPath) }</a>
|
||||
}
|
||||
</h6>
|
||||
|
||||
@@ -107,7 +107,7 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
</div>
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
if data.HasNextPage {
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPageURL(data.CurrentPath, data.LastFileName, data.PageSize)) } class="btn btn-outline-primary" title="Next page">
|
||||
<a href={ dash.PUrl(ctx, fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)) } class="btn btn-outline-primary" title="Next page">
|
||||
Next <i class="fas fa-angle-right"></i>
|
||||
</a>
|
||||
} else {
|
||||
@@ -116,7 +116,7 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
</button>
|
||||
}
|
||||
if data.ParentPath != data.CurrentPath {
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPathURL(data.ParentPath)) } class="btn btn-outline-secondary" title="Go up one directory">
|
||||
<a href={ dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", data.ParentPath)) } class="btn btn-outline-secondary" title="Go up one directory">
|
||||
<i class="fas fa-arrow-up"></i> Up
|
||||
</a>
|
||||
}
|
||||
@@ -150,7 +150,7 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<div class="d-flex align-items-center">
|
||||
if entry.IsDirectory {
|
||||
<i class="fas fa-folder text-warning me-2"></i>
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPathURL(entry.FullPath)) } class="text-decoration-none">
|
||||
<a href={ dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", entry.FullPath)) } class="text-decoration-none">
|
||||
{ entry.Name }
|
||||
</a>
|
||||
} else {
|
||||
@@ -238,7 +238,7 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
|
||||
if data.HasNextPage {
|
||||
<a href={ dash.PUrl(ctx, fileBrowserPageURL(data.CurrentPath, data.LastFileName, data.PageSize)) } class="btn btn-outline-primary">
|
||||
<a href={ dash.PUrl(ctx, fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)) } class="btn btn-outline-primary">
|
||||
Next <i class="fas fa-angle-right ms-1"></i>
|
||||
</a>
|
||||
} else {
|
||||
|
||||
@@ -140,7 +140,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var6 templ.SafeURL
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPathURL(crumb.Path)))
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", crumb.Path)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 62, Col: 73}
|
||||
}
|
||||
@@ -171,7 +171,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var8 templ.SafeURL
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPathURL(crumb.Path)))
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", crumb.Path)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 68, Col: 73}
|
||||
}
|
||||
@@ -214,7 +214,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var10 templ.SafeURL
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPathURL("/")))
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, "/files?path=/"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 86, Col: 45}
|
||||
}
|
||||
@@ -232,7 +232,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var11 templ.SafeURL
|
||||
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPathURL("/buckets")))
|
||||
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, "/files?path=/buckets"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 88, Col: 58}
|
||||
}
|
||||
@@ -263,7 +263,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var13 templ.SafeURL
|
||||
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPathURL(data.CurrentPath)))
|
||||
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", data.CurrentPath)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 93, Col: 77}
|
||||
}
|
||||
@@ -356,7 +356,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var16 templ.SafeURL
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPageURL(data.CurrentPath, data.LastFileName, data.PageSize)))
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 110, Col: 138}
|
||||
}
|
||||
@@ -380,7 +380,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var17 templ.SafeURL
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPathURL(data.ParentPath)))
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", data.ParentPath)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 119, Col: 78}
|
||||
}
|
||||
@@ -426,7 +426,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var19 templ.SafeURL
|
||||
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPathURL(entry.FullPath)))
|
||||
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=%s", entry.FullPath)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 153, Col: 83}
|
||||
}
|
||||
@@ -731,7 +731,7 @@ func FileBrowser(data dash.FileBrowserData) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var35 templ.SafeURL
|
||||
templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fileBrowserPageURL(data.CurrentPath, data.LastFileName, data.PageSize)))
|
||||
templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinURLErrs(dash.PUrl(ctx, fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/file_browser.templ`, Line: 241, Col: 138}
|
||||
}
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
func fileBrowserPathURL(path string) string {
|
||||
return "/files?path=" + url.QueryEscape(path)
|
||||
}
|
||||
|
||||
func fileBrowserPageURL(path, lastFileName string, limit int) string {
|
||||
return fmt.Sprintf("%s&lastFileName=%s&limit=%d",
|
||||
fileBrowserPathURL(path),
|
||||
url.QueryEscape(lastFileName),
|
||||
limit,
|
||||
)
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package app
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFileBrowserPathURLEscapesQueryPath(t *testing.T) {
|
||||
path := "/buckets/airflow-logs/logs/run_id=backfill__2025-01-31T15:00:00+00:00"
|
||||
want := "/files?path=%2Fbuckets%2Fairflow-logs%2Flogs%2Frun_id%3Dbackfill__2025-01-31T15%3A00%3A00%2B00%3A00"
|
||||
|
||||
if got := fileBrowserPathURL(path); got != want {
|
||||
t.Fatalf("fileBrowserPathURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileBrowserPageURLEscapesCursor(t *testing.T) {
|
||||
got := fileBrowserPageURL("/buckets/airflow-logs/logs/run_id=a+b", "task:1+next", 50)
|
||||
want := "/files?path=%2Fbuckets%2Fairflow-logs%2Flogs%2Frun_id%3Da%2Bb&lastFileName=task%3A1%2Bnext&limit=50"
|
||||
|
||||
if got != want {
|
||||
t.Fatalf("fileBrowserPageURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -258,7 +258,7 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<label class="form-label" for="plugin-admin-detection-interval">Detection Interval (min)</label>
|
||||
<label class="form-label" for="plugin-admin-detection-interval">Detection Interval (s)</label>
|
||||
<input type="number" class="form-control" id="plugin-admin-detection-interval" min="0"/>
|
||||
<div class="form-text">How often to check for new work.</div>
|
||||
</div>
|
||||
@@ -758,7 +758,7 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
}
|
||||
|
||||
function updateURL(replace) {
|
||||
var url = basePath(buildPluginURL(state.activeTopTab, state.activeSubTab, state.selectedJobType));
|
||||
var url = buildPluginURL(state.activeTopTab, state.activeSubTab, state.selectedJobType);
|
||||
if (replace) {
|
||||
history.replaceState({ pluginTopTab: state.activeTopTab, pluginSubTab: state.activeSubTab, pluginJob: state.selectedJobType }, '', url);
|
||||
} else {
|
||||
@@ -2526,7 +2526,7 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
}
|
||||
|
||||
document.getElementById('plugin-admin-enabled').checked = pickBool('enabled');
|
||||
document.getElementById('plugin-admin-detection-interval').value = String(pickNumber('detection_interval_minutes'));
|
||||
document.getElementById('plugin-admin-detection-interval').value = String(pickNumber('detection_interval_seconds'));
|
||||
document.getElementById('plugin-admin-detection-timeout').value = String(pickNumber('detection_timeout_seconds'));
|
||||
document.getElementById('plugin-admin-execution-timeout').value = String(pickNumber('execution_timeout_seconds'));
|
||||
document.getElementById('plugin-admin-max-runtime').value = String(pickNumber('job_type_max_runtime_seconds'));
|
||||
@@ -2552,7 +2552,7 @@ templ Plugin(page string, initialJob string, lane string) {
|
||||
|
||||
return {
|
||||
enabled: !!document.getElementById('plugin-admin-enabled').checked,
|
||||
detection_interval_minutes: getInt('plugin-admin-detection-interval'),
|
||||
detection_interval_seconds: getInt('plugin-admin-detection-interval'),
|
||||
detection_timeout_seconds: getInt('plugin-admin-detection-timeout'),
|
||||
execution_timeout_seconds: getInt('plugin-admin-execution-timeout'),
|
||||
job_type_max_runtime_seconds: getInt('plugin-admin-max-runtime'),
|
||||
|
||||
@@ -112,7 +112,7 @@ templ PluginLane(page string, lane string) {
|
||||
html += '<tr>';
|
||||
html += '<td>' + jt.job_type + '</td>';
|
||||
html += '<td>' + (jt.enabled ? '<span class="badge bg-success">Yes</span>' : '<span class="badge bg-secondary">No</span>') + '</td>';
|
||||
html += '<td>' + (jt.detection_interval_minutes || '-') + 'm</td>';
|
||||
html += '<td>' + (jt.detection_interval_seconds || '-') + 's</td>';
|
||||
html += '<td>' + (jt.detection_in_flight ? '<span class="badge bg-warning">Yes</span>' : 'No') + '</td>';
|
||||
html += '<td>' + (jt.next_detection_at ? new Date(jt.next_detection_at).toLocaleTimeString() : '-') + '</td>';
|
||||
html += '</tr>';
|
||||
@@ -132,7 +132,7 @@ templ PluginLane(page string, lane string) {
|
||||
html += '<td><a href="' + basePath('/plugin/configuration?job=' + s.job_type) + '">' + s.job_type + '</a></td>';
|
||||
html += '<td>' + (s.enabled ? '<span class="badge bg-success">Yes</span>' : '<span class="badge bg-secondary">No</span>') + '</td>';
|
||||
html += '<td>' + (s.global_execution_concurrency || 1) + '</td>';
|
||||
html += '<td>' + (s.detection_interval_minutes || '-') + 'm</td>';
|
||||
html += '<td>' + (s.detection_interval_seconds || '-') + 's</td>';
|
||||
html += '<td>' + (s.last_run_status || '-') + '</td>';
|
||||
html += '</tr>';
|
||||
});
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -138,38 +138,6 @@ func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType stri
|
||||
return
|
||||
}
|
||||
|
||||
// IsKnownNode reports whether address is currently registered under nodeType
|
||||
// in any filer group. The lookup is intentionally group-agnostic because callers
|
||||
// (e.g. Ping admission) only know the target address, not the group it joined.
|
||||
func (cluster *Cluster) IsKnownNode(nodeType string, address pb.ServerAddress) bool {
|
||||
var groups *ClusterNodeGroups
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
groups = cluster.filerGroups
|
||||
case BrokerType:
|
||||
groups = cluster.brokerGroups
|
||||
case S3Type:
|
||||
groups = cluster.s3Groups
|
||||
default:
|
||||
return false
|
||||
}
|
||||
groups.RLock()
|
||||
defer groups.RUnlock()
|
||||
for _, members := range groups.groupMembers {
|
||||
if _, found := members.members[address]; found {
|
||||
return true
|
||||
}
|
||||
// fall back to a port-tolerant comparison so callers that omit the
|
||||
// grpc-port suffix still match a registered peer
|
||||
for stored := range members.members {
|
||||
if stored.Equals(address) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func buildClusterNodeUpdateMessage(isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
|
||||
@@ -39,28 +39,3 @@ func TestConcurrentAddRemoveNodes(t *testing.T) {
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestIsKnownNode(t *testing.T) {
|
||||
c := NewCluster()
|
||||
filer := pb.ServerAddress("10.0.0.20:8888")
|
||||
c.AddClusterNode("", FilerType, "dc1", "rack1", filer, "test")
|
||||
|
||||
if !c.IsKnownNode(FilerType, filer) {
|
||||
t.Fatalf("registered filer %s should be known", filer)
|
||||
}
|
||||
if c.IsKnownNode(VolumeServerType, filer) {
|
||||
t.Fatalf("filer address must not be accepted as a volume server target")
|
||||
}
|
||||
if c.IsKnownNode(FilerType, pb.ServerAddress("127.0.0.1:1")) {
|
||||
t.Fatalf("unregistered low-port target must be rejected")
|
||||
}
|
||||
if c.IsKnownNode(FilerType, pb.ServerAddress("127.0.0.1:65000")) {
|
||||
t.Fatalf("unregistered high-port target must be rejected")
|
||||
}
|
||||
if c.IsKnownNode(FilerType, pb.ServerAddress("example.com:443")) {
|
||||
t.Fatalf("unrelated host must be rejected")
|
||||
}
|
||||
if c.IsKnownNode("garbage", filer) {
|
||||
t.Fatalf("unknown node type must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,7 +156,7 @@ func backupFromLocation(volumeServer pb.ServerAddress, grpcDialOption grpc.DialO
|
||||
|
||||
// If local volume is larger than remote, recreate it
|
||||
if datSize > stats.TailOffset {
|
||||
if err := v.Destroy(false, false); err != nil {
|
||||
if err := v.Destroy(false); err != nil {
|
||||
v.Close()
|
||||
return fmt.Errorf("destroying volume: %w", err), false
|
||||
}
|
||||
|
||||
+4
-15
@@ -431,22 +431,11 @@ func (fo *FilerOptions) startFiler() {
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
|
||||
filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
|
||||
|
||||
// Register IAM gRPC service only when both a credential manager and an
|
||||
// admin signing key are configured. The IAM RPCs can create users and
|
||||
// mint access keys; mounting them on an unauthenticated listener would
|
||||
// hand any caller that can reach the gRPC port S3-admin equivalent power.
|
||||
// Operators who relied on the unauthenticated path must now set
|
||||
// jwt.filer_signing.key in security.toml and attach a Bearer token signed
|
||||
// with that key on every IAM call.
|
||||
// Register IAM gRPC service if credential manager is available
|
||||
if credentialManager != nil {
|
||||
adminSigningKey := security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key"))
|
||||
if len(adminSigningKey) == 0 {
|
||||
glog.Warningf("IAM gRPC service NOT registered on filer: jwt.filer_signing.key is empty in security.toml; configure it to enable IAM administration")
|
||||
} else {
|
||||
iamGrpcServer := weed_server.NewIamGrpcServer(credentialManager, adminSigningKey)
|
||||
iam_pb.RegisterSeaweedIdentityAccessManagementServer(grpcS, iamGrpcServer)
|
||||
glog.V(0).Info("Registered IAM gRPC service on filer (admin Bearer token required)")
|
||||
}
|
||||
iamGrpcServer := weed_server.NewIamGrpcServer(credentialManager)
|
||||
iam_pb.RegisterSeaweedIdentityAccessManagementServer(grpcS, iamGrpcServer)
|
||||
glog.V(0).Info("Registered IAM gRPC service on filer")
|
||||
}
|
||||
|
||||
reflection.Register(grpcS)
|
||||
|
||||
+5
-30
@@ -346,7 +346,6 @@ func initMiniVolumeFlags() {
|
||||
miniOptions.v.inflightDownloadDataTimeout = cmdMini.Flag.Duration("volume.inflightDownloadDataTimeout", 60*time.Second, "inflight download data wait timeout")
|
||||
miniOptions.v.hasSlowRead = cmdMini.Flag.Bool("volume.hasSlowRead", true, "if true, prevents slow reads from blocking other requests")
|
||||
miniOptions.v.readBufferSizeMB = cmdMini.Flag.Int("volume.readBufferSizeMB", 4, "read buffer size in MB")
|
||||
miniOptions.v.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.")
|
||||
miniOptions.v.preStopSeconds = cmdMini.Flag.Int("volume.preStopSeconds", 1, "number of seconds between stop send heartbeats and stop volume server (default: 1 for mini)")
|
||||
}
|
||||
|
||||
@@ -1254,15 +1253,7 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
|
||||
// Wait for admin server's HTTP port to be ready before launching worker
|
||||
adminAddr := fmt.Sprintf("http://%s:%d", bindIp, *miniAdminOptions.port)
|
||||
glog.V(1).Infof("Waiting for admin server to be ready at %s...", adminAddr)
|
||||
if err := waitForAdminServerReady(ctx, adminAddr); err != nil {
|
||||
// If the parent context was cancelled (e.g. a previous in-process
|
||||
// mini run is being torn down), bail out gracefully instead of
|
||||
// fataling — the test harness uses `cmd.Run` directly so a Fatalf
|
||||
// would terminate the entire test binary.
|
||||
if ctx.Err() != nil {
|
||||
glog.Warningf("Admin server readiness wait aborted: %v", err)
|
||||
return
|
||||
}
|
||||
if err := waitForAdminServerReady(adminAddr); err != nil {
|
||||
glog.Fatalf("Admin server readiness check failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -1281,28 +1272,16 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
|
||||
waitForWorkerReady(workerGrpcAddr)
|
||||
}
|
||||
|
||||
// waitForAdminServerReady pings the admin server HTTP endpoint to check if it's ready.
|
||||
// Returns ctx.Err() (typically context.Canceled) if the parent context is
|
||||
// cancelled mid-poll so a torn-down mini doesn't keep spinning for the full
|
||||
// timeout — relevant when tests embed mini in-process and reuse the same
|
||||
// goroutine pool across subtests.
|
||||
func waitForAdminServerReady(ctx context.Context, adminAddr string) error {
|
||||
// waitForAdminServerReady pings the admin server HTTP endpoint to check if it's ready
|
||||
func waitForAdminServerReady(adminAddr string) error {
|
||||
healthAddr := getHealthCheckAddr(fmt.Sprintf("%s/health", adminAddr))
|
||||
// 240 * 500ms = 120 seconds max wait. The previous 30-second ceiling was
|
||||
// too tight on busy CI runners where master + filer + volume + admin all
|
||||
// initialise on a shared worker — the S3 Policy Shell Integration Tests
|
||||
// flaked regularly even though the admin server still came up within a
|
||||
// minute or two. Two minutes leaves headroom without being absurd locally.
|
||||
maxAttempts := 240
|
||||
maxAttempts := 60 // 60 * 500ms = 30 seconds max wait
|
||||
attempt := 0
|
||||
client := &http.Client{
|
||||
Timeout: 1 * time.Second,
|
||||
}
|
||||
|
||||
for attempt < maxAttempts {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := client.Get(healthAddr)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
@@ -1310,11 +1289,7 @@ func waitForAdminServerReady(ctx context.Context, adminAddr string) error {
|
||||
return nil
|
||||
}
|
||||
attempt++
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
return fmt.Errorf("admin server did not become ready at %s after %d attempts", adminAddr, maxAttempts)
|
||||
|
||||
@@ -15,7 +15,7 @@ func TestMiniDefaultPluginJobTypes(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("buildPluginWorkerHandlers(mini default) err = %v", err)
|
||||
}
|
||||
if len(handlers) != 7 {
|
||||
t.Fatalf("expected mini default job types to include 7 handlers, got %d", len(handlers))
|
||||
if len(handlers) != 6 {
|
||||
t.Fatalf("expected mini default job types to include 6 handlers, got %d", len(handlers))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/iceberg"
|
||||
@@ -407,7 +406,6 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.s3"))
|
||||
s3_pb.RegisterSeaweedS3IamCacheServer(grpcS, s3ApiServer)
|
||||
s3_lifecycle_pb.RegisterSeaweedS3LifecycleInternalServer(grpcS, s3ApiServer)
|
||||
reflection.Register(grpcS)
|
||||
if grpcLocalL != nil {
|
||||
go grpcS.Serve(grpcLocalL)
|
||||
|
||||
@@ -44,11 +44,6 @@ expires_after_seconds = 10 # seconds
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# NOTE: This key is ALSO used as a fallback signing key for S3 STS if s3.iam.config does not specify a signingKey.
|
||||
# NOTE: This key is ALSO required to mount the IAM gRPC service (CreateUser,
|
||||
# PutPolicy, CreateAccessKey, ...) on the filer. The filer refuses to
|
||||
# register that service when the key is empty, and every IAM RPC must
|
||||
# carry a Bearer token signed with this key in its "authorization"
|
||||
# gRPC metadata. Mint such a token with security.GenJwtForFilerAdmin.
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing]
|
||||
key = ""
|
||||
|
||||
@@ -155,7 +155,6 @@ func init() {
|
||||
|
||||
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally")
|
||||
serverOptions.v.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.")
|
||||
|
||||
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
|
||||
s3Options.portHttps = cmdServer.Flag.Int("s3.port.https", 0, "s3 server https listen port")
|
||||
|
||||
+7
-10
@@ -70,14 +70,13 @@ type VolumeServerOptions struct {
|
||||
metricsHttpPort *int
|
||||
metricsHttpIp *string
|
||||
// pulseSeconds *int
|
||||
inflightUploadDataTimeout *time.Duration
|
||||
inflightDownloadDataTimeout *time.Duration
|
||||
hasSlowRead *bool
|
||||
readBufferSizeMB *int
|
||||
ldbTimeout *int64
|
||||
allowUntrustedRemoteEndpoints *bool
|
||||
debug *bool
|
||||
debugPort *int
|
||||
inflightUploadDataTimeout *time.Duration
|
||||
inflightDownloadDataTimeout *time.Duration
|
||||
hasSlowRead *bool
|
||||
readBufferSizeMB *int
|
||||
ldbTimeout *int64
|
||||
debug *bool
|
||||
debugPort *int
|
||||
// shutdownCtx, when non-nil, tells startVolumeServer to shut down once the
|
||||
// ctx is cancelled. Used by integration tests and by weed mini; nil for
|
||||
// standalone weed volume.
|
||||
@@ -121,7 +120,6 @@ func init() {
|
||||
v.inflightDownloadDataTimeout = cmdVolume.Flag.Duration("inflightDownloadDataTimeout", 60*time.Second, "inflight download data wait timeout of volume servers")
|
||||
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
|
||||
v.allowUntrustedRemoteEndpoints = cmdVolume.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.")
|
||||
v.debug = cmdVolume.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
|
||||
v.debugPort = cmdVolume.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
}
|
||||
@@ -304,7 +302,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
*v.hasSlowRead,
|
||||
*v.readBufferSizeMB,
|
||||
*v.ldbTimeout,
|
||||
*v.allowUntrustedRemoteEndpoints,
|
||||
)
|
||||
// starting grpc server
|
||||
grpcS := v.startGrpcService(volumeServer)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
|
||||
@@ -15,26 +14,8 @@ func TestWorkerDefaultJobTypes(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("buildPluginWorkerHandlers(default worker flag) err = %v", err)
|
||||
}
|
||||
want := []string{
|
||||
"admin_script",
|
||||
"ec_balance",
|
||||
"erasure_coding",
|
||||
"iceberg_maintenance",
|
||||
"s3_lifecycle",
|
||||
"vacuum",
|
||||
"volume_balance",
|
||||
}
|
||||
got := make([]string, 0, len(handlers))
|
||||
for _, h := range handlers {
|
||||
got = append(got, h.Capability().JobType)
|
||||
}
|
||||
sort.Strings(got)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("default worker job types: got %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("default worker job types: got %v, want %v", got, want)
|
||||
}
|
||||
// Expected: vacuum, volume_balance, admin_script, erasure_coding, iceberg_maintenance, ec_balance
|
||||
if len(handlers) != 6 {
|
||||
t.Fatalf("expected default worker job types to include 6 handlers, got %d", len(handlers))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ func (store *FilerEtcStore) saveGroup(ctx context.Context, group *iam_pb.Group)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filer.SaveInsideFiler(context.Background(), client, filer.IamConfigDirectory+"/"+IamGroupsDirectory, group.Name+".json", data)
|
||||
return filer.SaveInsideFiler(client, filer.IamConfigDirectory+"/"+IamGroupsDirectory, group.Name+".json", data)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -484,7 +484,7 @@ func (store *FilerEtcStore) saveIdentity(ctx context.Context, identity *iam_pb.I
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filer.SaveInsideFiler(context.Background(), client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, identity.Name+".json", data)
|
||||
return filer.SaveInsideFiler(client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, identity.Name+".json", data)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +64,7 @@ func (store *FilerEtcStore) saveLegacyPoliciesCollection(ctx context.Context, po
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filer.SaveInsideFiler(context.Background(), client, filer.IamConfigDirectory, filer.IamPoliciesFile, content)
|
||||
return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile, content)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -216,7 +216,7 @@ func (store *FilerEtcStore) savePolicy(ctx context.Context, name string, documen
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filer.SaveInsideFiler(context.Background(), client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json", data)
|
||||
return filer.SaveInsideFiler(client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json", data)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ func (store *FilerEtcStore) saveServiceAccount(ctx context.Context, sa *iam_pb.S
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filer.SaveInsideFiler(context.Background(), client, filer.IamConfigDirectory+"/"+IamServiceAccountsDirectory, sa.Id+".json", data)
|
||||
return filer.SaveInsideFiler(client, filer.IamConfigDirectory+"/"+IamServiceAccountsDirectory, sa.Id+".json", data)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -87,23 +87,6 @@ func (ma *MetaAggregator) HasRemotePeers() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// HasPeer reports whether address is currently a tracked filer peer (or this
|
||||
// filer's own address). Callers use this to gate operations on known cluster
|
||||
// members.
|
||||
func (ma *MetaAggregator) HasPeer(address pb.ServerAddress) bool {
|
||||
if address == ma.self || address.Equals(ma.self) {
|
||||
return true
|
||||
}
|
||||
ma.peerChansLock.Lock()
|
||||
defer ma.peerChansLock.Unlock()
|
||||
for peer := range ma.peerChans {
|
||||
if peer == address || peer.Equals(address) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time, stopChan chan struct{}) {
|
||||
lastTsNs := startFrom.UnixNano()
|
||||
for {
|
||||
|
||||
@@ -41,15 +41,15 @@ func ReadInsideFiler(ctx context.Context, filerClient filer_pb.SeaweedFilerClien
|
||||
return
|
||||
}
|
||||
|
||||
func SaveInsideFiler(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, name string, content []byte) error {
|
||||
func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, content []byte) error {
|
||||
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
|
||||
if err == filer_pb.ErrNotFound {
|
||||
err = filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
|
||||
err = filer_pb.CreateEntry(context.Background(), client, &filer_pb.CreateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: name,
|
||||
@@ -69,7 +69,7 @@ func SaveInsideFiler(ctx context.Context, client filer_pb.SeaweedFilerClient, di
|
||||
entry.Content = content
|
||||
entry.Attributes.Mtime = time.Now().Unix()
|
||||
entry.Attributes.FileSize = uint64(len(content))
|
||||
err = filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
|
||||
err = filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: entry,
|
||||
})
|
||||
|
||||
@@ -52,7 +52,7 @@ func InsertMountMapping(filerClient filer_pb.FilerClient, dir string, remoteStor
|
||||
|
||||
// save back
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return SaveInsideFiler(context.Background(), client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
|
||||
return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("save mapping: %w", err)
|
||||
@@ -83,7 +83,7 @@ func DeleteMountMapping(filerClient filer_pb.FilerClient, dir string) (err error
|
||||
|
||||
// save back
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return SaveInsideFiler(context.Background(), client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
|
||||
return SaveInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE, newContent)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("save mapping: %w", err)
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetProviderByIssuerAndAccount(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := NewMemoryOIDCProviderStore()
|
||||
|
||||
// Two providers for the same issuer, scoped to different accounts.
|
||||
must := func(rec *OIDCProviderRecord) {
|
||||
if err := store.StoreProvider(ctx, "", rec); err != nil {
|
||||
t.Fatalf("store: %v", err)
|
||||
}
|
||||
}
|
||||
must(&OIDCProviderRecord{
|
||||
AccountID: "111",
|
||||
ARN: "arn:aws:iam::111:oidc-provider/idp.example",
|
||||
URL: "https://idp.example",
|
||||
ClientIDs: []string{"a"},
|
||||
})
|
||||
must(&OIDCProviderRecord{
|
||||
AccountID: "222",
|
||||
ARN: "arn:aws:iam::222:oidc-provider/idp.example",
|
||||
URL: "https://idp.example",
|
||||
ClientIDs: []string{"b"},
|
||||
})
|
||||
|
||||
t.Run("matching account returns scoped record", func(t *testing.T) {
|
||||
rec, err := store.GetProviderByIssuerAndAccount(ctx, "", "https://idp.example", "111")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if rec.AccountID != "111" {
|
||||
t.Fatalf("unexpected record: %+v", rec)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unknown account is rejected", func(t *testing.T) {
|
||||
if _, err := store.GetProviderByIssuerAndAccount(ctx, "", "https://idp.example", "999"); err == nil {
|
||||
t.Fatal("expected error for cross-account use")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("global provider satisfies any account", func(t *testing.T) {
|
||||
must(&OIDCProviderRecord{
|
||||
AccountID: "",
|
||||
ARN: "arn:aws:iam:::oidc-provider/global.example",
|
||||
URL: "https://global.example",
|
||||
ClientIDs: []string{"x"},
|
||||
})
|
||||
rec, err := store.GetProviderByIssuerAndAccount(ctx, "", "https://global.example", "anything")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if rec.AccountID != "" {
|
||||
t.Fatalf("expected global record, got AccountID=%q", rec.AccountID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty caller account accepts any record", func(t *testing.T) {
|
||||
rec, err := store.GetProviderByIssuerAndAccount(ctx, "", "https://idp.example", "")
|
||||
if err != nil {
|
||||
t.Fatalf("err: %v", err)
|
||||
}
|
||||
if rec == nil {
|
||||
t.Fatal("expected a record")
|
||||
}
|
||||
})
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user