Compare commits

..
134 changed files with 3051 additions and 7294 deletions
+9 -102
View File
@@ -23,11 +23,6 @@ on:
- all
- standard
- large_disk
publish:
description: 'Publish images and manifests'
required: true
type: boolean
default: false
permissions:
contents: read
@@ -38,7 +33,6 @@ jobs:
runs-on: ubuntu-latest
outputs:
variants: ${{ steps.set-variants.outputs.variants }}
publish: ${{ steps.set-publish.outputs.publish }}
steps:
- name: Select variants for this run
id: set-variants
@@ -49,14 +43,6 @@ jobs:
variants='["standard","large_disk"]'
fi
echo "variants=$variants" >> "$GITHUB_OUTPUT"
- name: Select publish mode
id: set-publish
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "publish=${{ github.event.inputs.publish }}" >> "$GITHUB_OUTPUT"
else
echo "publish=true" >> "$GITHUB_OUTPUT"
fi
build:
needs: [setup]
@@ -126,13 +112,13 @@ jobs:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: needs.setup.outputs.publish == 'true'
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
with:
registry: ghcr.io
@@ -144,7 +130,7 @@ jobs:
DOCKER_BUILDKIT: 1
with:
context: ./docker
push: ${{ needs.setup.outputs.publish == 'true' }}
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.go_build
platforms: linux/${{ matrix.platform }}
# Push to GHCR only during build to avoid Docker Hub rate limits
@@ -180,112 +166,33 @@ jobs:
echo "tag_suffix=" >> $GITHUB_OUTPUT
fi
- name: Login to GHCR
if: needs.setup.outputs.publish == 'true'
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Checkout for local scan build
if: needs.setup.outputs.publish != 'true'
uses: actions/checkout@v6
with:
ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }}
- name: Create BuildKit config for local scan build
if: needs.setup.outputs.publish != 'true'
run: |
cat > /tmp/buildkitd.toml <<EOF
[registry."docker.io"]
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx for local scan build
if: needs.setup.outputs.publish != 'true'
uses: docker/setup-buildx-action@v4
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Build local scan image tarball
if: needs.setup.outputs.publish != 'true'
uses: docker/build-push-action@v7
env:
DOCKER_BUILDKIT: 1
with:
context: ./docker
file: ./docker/Dockerfile.go_build
platforms: linux/amd64
outputs: type=docker,dest=/tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
build-args: |
BUILDKIT_INLINE_CACHE=1
BRANCH=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.sha }}
${{ matrix.variant == 'large_disk' && 'TAGS=5BytesOffset' || '' }}
- name: Trivy report (published image)
if: needs.setup.outputs.publish == 'true'
# Pin to SHA - mutable tags were compromised (GHSA-69fq-xp46-6x23)
- name: Run Trivy vulnerability scanner
# Pin to SHA — mutable tags were compromised (GHSA-69fq-xp46-6x23)
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
scan-type: image
# Scan amd64 only - OS packages are identical across architectures
# Scan amd64 only — OS packages are identical across architectures
# since they all use the same alpine base, so a single-arch scan
# provides sufficient coverage without multiplying CI time.
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
- name: Trivy report (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
limit-severities-for-sarif: true
format: sarif
output: trivy-results.sarif
exit-code: '0'
exit-code: '1'
- name: Upload Trivy scan results to GitHub Security
uses: github/codeql-action/upload-sarif@v4
uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: trivy-results.sarif
- name: Trivy gate (published image)
if: needs.setup.outputs.publish == 'true'
# Gate only on fixable high/critical vulnerabilities. Non-fixable
# findings are still visible in the SARIF upload above.
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
scan-type: image
image-ref: ghcr.io/chrislusf/seaweedfs:${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }}${{ steps.config.outputs.tag_suffix }}-amd64
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
format: table
exit-code: '1'
skip-setup-trivy: true
- name: Trivy gate (local tarball)
if: needs.setup.outputs.publish != 'true'
uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
with:
input: /tmp/seaweedfs${{ steps.config.outputs.tag_suffix }}-amd64.tar
scanners: vuln
vuln-type: os,library
severity: HIGH,CRITICAL
ignore-unfixed: true
format: table
exit-code: '1'
skip-setup-trivy: true
create-manifest:
runs-on: ubuntu-latest
needs: [setup, build, trivy-scan]
if: needs.setup.outputs.publish == 'true' && github.event_name != 'pull_request'
if: github.event_name != 'pull_request'
strategy:
matrix:
variant: ${{ fromJSON(needs.setup.outputs.variants) }}
+1 -1
View File
@@ -21,7 +21,7 @@ jobs:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v5
uses: azure/setup-helm@v4
with:
version: v3.18.4
@@ -43,7 +43,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
@@ -80,7 +80,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
@@ -156,7 +156,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
+2 -2
View File
@@ -47,7 +47,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
@@ -118,7 +118,7 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
+3 -3
View File
@@ -43,7 +43,7 @@ jobs:
echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV"
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
@@ -112,7 +112,7 @@ jobs:
targets: ${{ matrix.target }}
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
@@ -170,7 +170,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable
- name: Cache cargo registry and target
uses: actions/cache@v5
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
+10 -8
View File
@@ -16,16 +16,15 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
&& export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \
&& CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}"
# Rust volume server builder. Alpine packages avoid depending on the
# upstream rust:alpine manifest list, which no longer includes linux/386.
FROM alpine:3.23 as rust_builder
# Rust volume server builder (amd64/arm64 only)
FROM rust:1-alpine as rust_builder
ARG TARGETARCH
RUN apk add musl-dev protobuf-dev git
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/proto /build/proto
WORKDIR /build/seaweed-volume
ARG TAGS
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \
if [ "$TAGS" = "5BytesOffset" ]; then \
cargo build --release; \
else \
@@ -50,8 +49,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh
# To disable: docker run -e GODEBUG=fips140=off ...
# Install dependencies and create non-root user
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse curl su-exec && \
RUN apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -81,5 +79,9 @@ RUN mkdir -p /data/filerldb2 && \
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
# Run as non-root by default (satisfies security scanners).
# Use `docker run --user root` if you need the entrypoint to fix
# /data volume ownership before dropping privileges.
USER seaweed
ENTRYPOINT ["/entrypoint.sh"]
+6 -3
View File
@@ -7,8 +7,7 @@ COPY ./filer.toml /etc/seaweedfs/filer.toml
COPY ./entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse curl su-exec && \
RUN apk add --no-cache fuse curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
@@ -38,5 +37,9 @@ RUN mkdir -p /data/filerldb2 && \
VOLUME /data
WORKDIR /data
# Entrypoint will handle permission fixes and user switching
# Run as non-root by default (satisfies security scanners).
# Use `docker run --user root` if you need the entrypoint to fix
# /data volume ownership before dropping privileges.
USER seaweed
ENTRYPOINT ["/entrypoint.sh"]
+1 -2
View File
@@ -34,8 +34,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse snappy gflags curl su-exec && \
RUN apk add --no-cache fuse snappy gflags curl su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
+1 -2
View File
@@ -17,8 +17,7 @@ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_rocksdb.
COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
# Install dependencies and create non-root user
RUN apk upgrade --no-cache zlib && \
apk add --no-cache fuse snappy gflags curl tmux su-exec && \
RUN apk add --no-cache fuse snappy gflags curl tmux su-exec && \
addgroup -g 1000 seaweed && \
adduser -D -u 1000 -G seaweed seaweed
+19 -19
View File
@@ -83,10 +83,10 @@ require (
github.com/valyala/bytebufferpool v1.0.0
github.com/viant/ptrie v1.0.1
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.2.0
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.etcd.io/etcd/client/v3 v3.6.9
go.etcd.io/etcd/client/v3 v3.6.7
go.mongodb.org/mongo-driver v1.17.9
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.45.0
@@ -94,7 +94,7 @@ require (
gocloud.dev/pubsub/rabbitpubsub v0.45.0
golang.org/x/crypto v0.49.0
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa
golang.org/x/image v0.38.0
golang.org/x/image v0.36.0
golang.org/x/net v0.51.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sys v0.42.0
@@ -114,7 +114,7 @@ require (
)
require (
cloud.google.com/go/kms v1.26.0
cloud.google.com/go/kms v1.25.0
github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0
github.com/Jille/raft-grpc-transport v1.6.1
github.com/ThreeDotsLabs/watermill v1.5.1
@@ -124,13 +124,13 @@ require (
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
github.com/arangodb/go-driver v1.6.9
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.41.5
github.com/aws/aws-sdk-go-v2/config v1.32.13
github.com/aws/aws-sdk-go-v2/credentials v1.19.13
github.com/aws/aws-sdk-go-v2 v1.41.4
github.com/aws/aws-sdk-go-v2/config v1.32.9
github.com/aws/aws-sdk-go-v2/credentials v1.19.12
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.44.1
github.com/getsentry/sentry-go v0.43.0
github.com/go-ldap/ldap/v3 v3.4.13
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -157,7 +157,7 @@ require (
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.3
go.etcd.io/etcd/client/pkg/v3 v3.6.9
go.etcd.io/etcd/client/pkg/v3 v3.6.7
go.uber.org/atomic v1.11.0
golang.org/x/sync v0.20.0
golang.org/x/tools/godoc v0.1.0-deprecated
@@ -181,7 +181,7 @@ require (
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apache/arrow-go/v18 v18.5.2-0.20260220015023-a886a5722b87 // indirect
github.com/apache/thrift v0.22.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/bazelbuild/rules_go v0.46.0 // indirect
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
@@ -324,21 +324,21 @@ require (
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9
github.com/aws/smithy-go v1.24.2
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
@@ -486,7 +486,7 @@ require (
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zeebo/errs v1.4.0 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.etcd.io/etcd/api/v3 v3.6.9 // indirect
go.etcd.io/etcd/api/v3 v3.6.7 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
+40 -38
View File
@@ -298,8 +298,8 @@ cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxs
cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0=
cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
cloud.google.com/go/kms v1.26.0 h1:cK9mN2cf+9V63D3H1f6koxTatWy39aTI/hCjz1I+adU=
cloud.google.com/go/kms v1.26.0/go.mod h1:pHKOdFJm63hxBsiPkYtowZPltu9dW0MWvBa6IA4HM58=
cloud.google.com/go/kms v1.25.0 h1:gVqvGGUmz0nYCmtoxWmdc1wli2L1apgP8U4fghPGSbQ=
cloud.google.com/go/kms v1.25.0/go.mod h1:XIdHkzfj0bUO3E+LvwPg+oc7s58/Ns8Nd8Sdtljihbk=
cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -714,48 +714,48 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg=
github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI=
github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk=
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
@@ -1051,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getsentry/sentry-go v0.44.1 h1:/cPtrA5qB7uMRrhgSn9TYtcEF36auGP3Y6+ThvD/yaI=
github.com/getsentry/sentry-go v0.44.1/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
@@ -1838,6 +1838,8 @@ github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1Ivohy
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564 h1:TgxPraf1NmF6XTcUG53ULpLQrKvhtUJxQ3hyekxSDNQ=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20260225204133-2f342c5ea564/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
github.com/seaweedfs/go-fuse/v2 v2.9.1 h1:gnKmfrKreCRGJmekGz5WMnNZqXEf9s9+V2hdWQdvx88=
github.com/seaweedfs/go-fuse/v2 v2.9.1/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
github.com/seaweedfs/go-fuse/v2 v2.9.2 h1:IfP/yFjLGO4rALcJY2Gb39PlebHxLnj7dkIiQAjFres=
github.com/seaweedfs/go-fuse/v2 v2.9.2/go.mod h1:zABdmWEa6A0bwaBeEOBUeUkGIZlxUhcdv+V1Dcc/U/I=
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
@@ -2036,8 +2038,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -2101,12 +2103,12 @@ go.einride.tech/aip v0.73.0 h1:bPo4oqBo2ZQeBKo4ZzLb1kxYXTY1ysJhpvQyfuGzvps=
go.einride.tech/aip v0.73.0/go.mod h1:Mj7rFbmXEgw0dq1dqJ7JGMvYCZZVxmGOR3S4ZcV5LvQ=
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
go.etcd.io/etcd/api/v3 v3.6.7 h1:7BNJ2gQmc3DNM+9cRkv7KkGQDayElg8x3X+tFDYS+E0=
go.etcd.io/etcd/api/v3 v3.6.7/go.mod h1:xJ81TLj9hxrYYEDmXTeKURMeY3qEDN24hqe+q7KhbnI=
go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/AhBVs=
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2240,8 +2242,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.18"
appVersion: "4.17"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.18.0
version: 4.17.0
@@ -84,8 +84,6 @@ service SeaweedFiler {
// distributed lock management internal use only
rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
}
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
}
}
//////////////////////////////////////////////////
@@ -524,7 +522,6 @@ message LockResponse {
string lock_owner = 2;
string lock_host_moved_to = 3;
string error = 4;
int64 generation = 5;
}
message UnlockRequest {
string name = 1;
@@ -547,26 +544,12 @@ message Lock {
string renew_token = 2;
int64 expired_at_ns = 3;
string owner = 4;
int64 generation = 5;
bool is_backup = 6;
int64 seq = 7;
}
message TransferLocksRequest {
repeated Lock locks = 1;
}
message TransferLocksResponse {
}
message ReplicateLockRequest {
string name = 1;
string renew_token = 2;
int64 expired_at_ns = 3;
string owner = 4;
int64 generation = 5;
bool is_unlock = 6;
int64 seq = 7;
}
message ReplicateLockResponse {
}
//////////////////////////////////////////////////
// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
@@ -442,20 +442,6 @@ func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, c
}
continue
}
// The kernel dcache can drop the FUSE entry moments after a
// successful check. Wait briefly and re-verify to confirm the
// directory has stabilised before returning to the caller.
time.Sleep(1 * time.Second)
refreshDirEntry(t, mountClone)
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
if attempt < maxAttempts {
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
os.RemoveAll(mountClone)
time.Sleep(2 * time.Second)
}
continue
}
return
}
require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
@@ -483,10 +469,6 @@ func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit
}
refreshDirEntry(t, cloneDir)
// Let the dcache stabilise before proceeding.
time.Sleep(1 * time.Second)
refreshDirEntry(t, cloneDir)
head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
if err != nil {
return fmt.Errorf("rev-parse after reset: %w", err)
+20 -20
View File
@@ -14,7 +14,7 @@ require (
replace github.com/seaweedfs/seaweedfs => ../../
require (
cloud.google.com/go/auth v0.18.1 // indirect
cloud.google.com/go/auth v0.17.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
@@ -45,19 +45,19 @@ require (
github.com/aws/aws-sdk-go v1.55.8 // indirect
github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.9 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
@@ -124,8 +124,8 @@ require (
github.com/google/btree v1.1.3 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.11 // indirect
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/gorilla/schema v1.4.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -147,9 +147,9 @@ require (
github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/karlseguin/ccache/v2 v2.0.8 // indirect
github.com/klauspost/compress v1.18.5 // indirect
github.com/klauspost/compress v1.18.4 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/klauspost/reedsolomon v1.13.3 // indirect
github.com/klauspost/reedsolomon v1.13.0 // indirect
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 // indirect
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
github.com/kr/fs v0.1.0 // indirect
@@ -237,7 +237,7 @@ require (
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/blake3 v0.2.4 // indirect
github.com/zeebo/errs v1.4.0 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.etcd.io/bbolt v1.4.3 // indirect
go.mongodb.org/mongo-driver v1.17.9 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
@@ -247,18 +247,18 @@ require (
go.opentelemetry.io/otel/trace v1.40.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.49.0 // indirect
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
golang.org/x/image v0.38.0 // indirect
golang.org/x/net v0.51.0 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
golang.org/x/image v0.36.0 // indirect
golang.org/x/net v0.49.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/term v0.41.0 // indirect
golang.org/x/text v0.35.0 // indirect
golang.org/x/term v0.40.0 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/time v0.14.0 // indirect
google.golang.org/api v0.267.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 // indirect
google.golang.org/api v0.258.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+48 -48
View File
@@ -13,8 +13,8 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
cloud.google.com/go/auth v0.18.1 h1:IwTEx92GFUo2pJ6Qea0EU3zYvKnTAeRCODxfA/G5UWs=
cloud.google.com/go/auth v0.18.1/go.mod h1:GfTYoS9G3CWpRA3Va9doKN9mjPGRS+v41jmZAhBzbrA=
cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@@ -116,8 +116,8 @@ github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1h
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.9 h1:ktda/mtAydeObvJXlHzyGpK1xcsLaP16zfUPDGoW90A=
github.com/aws/aws-sdk-go-v2/config v1.32.9/go.mod h1:U+fCQ+9QKsLW786BCfEjYRj34VVTbPdsLP3CHSYXMOI=
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo=
@@ -130,18 +130,18 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDk
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A=
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZifkk6vEGl/1Qor0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
@@ -385,12 +385,12 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.11 h1:vAe81Msw+8tKUxi2Dqh/NZMz7475yUvmRIkXr4oN2ao=
github.com/googleapis/enterprise-certificate-proxy v0.3.11/go.mod h1:RFV7MUdlb7AgEq2v7FmMCfeSMCllAzWxFgRdusoGks8=
github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.17.0 h1:RksgfBpxqff0EZkDWYuz9q/uWsTVz+kf43LsZ1J6SMc=
github.com/googleapis/gax-go/v2 v2.17.0/go.mod h1:mzaqghpQp4JDh3HvADwrat+6M3MOIDp5YKHhb9PAgDY=
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
@@ -467,12 +467,12 @@ github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRt
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/reedsolomon v1.13.3 h1:01GwnO2xoCSaM0ShP4qwl+FsHg3csFShC6Tu/RS1ji0=
github.com/klauspost/reedsolomon v1.13.3/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
github.com/klauspost/reedsolomon v1.13.0 h1:E0Cmgf2kMuhZTj6eefnvpKC4/Q4jhCi9YIjcZjK4arc=
github.com/klauspost/reedsolomon v1.13.0/go.mod h1:ggJT9lc71Vu+cSOPBlxGvBN6TfAS77qB4fp8vJ05NSA=
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988 h1:CjEMN21Xkr9+zwPmZPaJJw+apzVbjGL5uK/6g9Q2jGU=
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988/go.mod h1:/agobYum3uo/8V6yPVnq+R82pyVGCeuWW5arT4Txn8A=
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U=
@@ -739,8 +739,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
@@ -787,8 +787,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -799,12 +799,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -869,8 +869,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -894,8 +894,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -959,8 +959,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -976,8 +976,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1030,14 +1030,14 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1054,8 +1054,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/api v0.267.0 h1:w+vfWPMPYeRs8qH1aYYsFX68jMls5acWl/jocfLomwE=
google.golang.org/api v0.267.0/go.mod h1:Jzc0+ZfLnyvXma3UtaTl023TdhZu6OMBP9tJ+0EmFD0=
google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1091,12 +1091,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 h1:VQZ/yAbAtjkHgH80teYd2em3xtIkkHd7ZhqfH2N9CsM=
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409/go.mod h1:rxKD3IEILWEu3P44seeNOAwZN4SaoKaQ/2eTg4mM6EM=
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20 h1:7ei4lp52gK1uSejlA8AZl5AJjeLUOHBQscRQZUgAcu0=
google.golang.org/genproto/googleapis/api v0.0.0-20260203192932-546029d2fa20/go.mod h1:ZdbssH/1SOVnjnDlXzxDHK2MCidiqXtbYccJNzNYPEE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20 h1:Jr5R2J6F6qWyzINc+4AM8t5pfUz6beZpHp678GNrMbE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls=
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+5 -92
View File
@@ -11,11 +11,9 @@ import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// VolumeServer provides a minimal volume server for erasure coding tests.
@@ -198,25 +196,12 @@ func (v *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream vo
defer file.Close()
buf := make([]byte, 64*1024)
remaining := int64(req.GetStopOffset())
for {
if remaining == 0 {
break
}
readBuf := buf
if remaining > 0 && remaining < int64(len(buf)) {
readBuf = buf[:remaining]
}
n, readErr := file.Read(readBuf)
n, readErr := file.Read(buf)
if n > 0 {
if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: readBuf[:n]}); err != nil {
if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: buf[:n]}); err != nil {
return err
}
if remaining > 0 {
remaining -= int64(n)
}
}
if readErr == io.EOF {
break
@@ -322,21 +307,10 @@ func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_ser
v.mu.Lock()
v.readFileStatusCalls++
v.mu.Unlock()
datInfo, err := os.Stat(v.filePath(req.VolumeId, ".dat"))
if err != nil {
return nil, err
}
idxInfo, err := os.Stat(v.filePath(req.VolumeId, ".idx"))
if err != nil {
return nil, err
}
return &volume_server_pb.ReadVolumeFileStatusResponse{
VolumeId: req.VolumeId,
DatFileSize: uint64(datInfo.Size()),
IdxFileSize: uint64(idxInfo.Size()),
DatFileSize: 1024,
IdxFileSize: 16,
FileCount: 1,
}, nil
}
@@ -375,27 +349,7 @@ func (v *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, strea
v.volumeCopyCalls++
v.mu.Unlock()
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
var statusResp *volume_server_pb.ReadVolumeFileStatusResponse
if err := operation.WithVolumeServerClient(false, pb.ServerAddress(req.SourceDataNode), dialOption,
func(client volume_server_pb.VolumeServerClient) error {
var readErr error
statusResp, readErr = client.ReadVolumeFileStatus(stream.Context(), &volume_server_pb.ReadVolumeFileStatusRequest{
VolumeId: req.VolumeId,
})
return readErr
}); err != nil {
return err
}
if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".dat", statusResp.DatFileSize, dialOption); err != nil {
return err
}
if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".idx", statusResp.IdxFileSize, dialOption); err != nil {
return err
}
if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: int64(statusResp.DatFileSize + statusResp.IdxFileSize)}); err != nil {
if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: 1024}); err != nil {
return err
}
return stream.Send(&volume_server_pb.VolumeCopyResponse{LastAppendAtNs: uint64(time.Now().UnixNano())})
@@ -414,44 +368,3 @@ func (v *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serve
v.mu.Unlock()
return &volume_server_pb.VolumeTailReceiverResponse{}, nil
}
func (v *VolumeServer) copyRemoteFile(ctx context.Context, sourceDataNode string, volumeID uint32, ext string, fileSize uint64, dialOption grpc.DialOption) error {
path := v.filePath(volumeID, ext)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
return operation.WithVolumeServerClient(true, pb.ServerAddress(sourceDataNode), dialOption,
func(client volume_server_pb.VolumeServerClient) error {
stream, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ext,
StopOffset: fileSize,
})
if err != nil {
return err
}
for {
resp, recvErr := stream.Recv()
if recvErr == io.EOF {
return nil
}
if recvErr != nil {
return recvErr
}
if len(resp.FileContent) == 0 {
continue
}
if _, err := file.Write(resp.FileContent); err != nil {
return err
}
}
})
}
@@ -16,8 +16,6 @@ import (
"google.golang.org/protobuf/proto"
)
const testVolumeDatSize = 1 * 1024 * 1024
func TestVolumeBalanceExecutionIntegration(t *testing.T) {
volumeID := uint32(303)
@@ -33,7 +31,6 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
source := pluginworkers.NewVolumeServer(t, "")
target := pluginworkers.NewVolumeServer(t, "")
pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), volumeID, testVolumeDatSize)
job := &plugin_pb.JobSpec{
JobId: fmt.Sprintf("balance-job-%d", volumeID),
@@ -87,9 +84,6 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
// Build a batch job with 3 volume moves from source → target.
volumeIDs := []uint32{401, 402, 403}
for _, vid := range volumeIDs {
pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), vid, testVolumeDatSize)
}
moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
for i, vid := range volumeIDs {
moves[i] = &worker_pb.BalanceMoveSpec{
@@ -145,11 +139,10 @@ func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
}
// Each move reads source status once before copy and once inside the
// target's fake VolumeCopy implementation, then reads target status once
// before deleting the source.
require.Equal(t, len(volumeIDs)*2, source.ReadFileStatusCount(),
"each move should read source volume status before copy and during target copy")
// Pre-delete verification should have called ReadVolumeFileStatus on both
// source and target for each volume.
require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
"each move should read source volume status before delete")
require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
"each move should read target volume status before delete")
+49 -7
View File
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
"io"
"math/rand"
"net"
"os"
"os/exec"
"path/filepath"
@@ -94,9 +96,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Fatalf("failed to create temp directory: %v", err)
}
env.masterPort = testutil.MustFreeMiniPort(t, "Master")
env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
env.s3Port = testutil.MustFreeMiniPort(t, "S3")
env.masterPort = mustFreePort(t, "Master")
env.filerPort = mustFreePort(t, "Filer")
env.s3Port = mustFreePort(t, "S3")
bindIP := testutil.FindBindIP()
iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
@@ -133,14 +135,14 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
}
registerMiniProcess(env.masterProcess)
if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
if !waitForPort(env.masterPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
}
if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
if !waitForPort(env.filerPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
}
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
if !waitForPort(env.s3Port, 15*time.Second) {
t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
}
}
@@ -206,6 +208,46 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
}
}
func mustFreePort(t *testing.T, name string) int {
t.Helper()
for i := 0; i < 200; i++ {
port := 20000 + rand.Intn(30000)
listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
if err != nil {
continue
}
_ = listener.Close()
grpcPort := port + 10000
if grpcPort > 65535 {
continue
}
grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
if err != nil {
continue
}
_ = grpcListener.Close()
return port
}
t.Fatalf("failed to get free port for %s", name)
return 0
}
func waitForPort(port int, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
if err == nil {
_ = conn.Close()
return true
}
time.Sleep(100 * time.Millisecond)
}
return false
}
func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
t.Helper()
+55 -12
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"math/rand"
"net"
"net/http"
"os"
"os/exec"
@@ -114,11 +115,11 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Fatalf("failed to create temp directory: %v", err)
}
env.masterPort = testutil.MustFreeMiniPort(t, "Master")
env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
env.s3Port = testutil.MustFreeMiniPort(t, "S3")
env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
env.risingwavePort = testutil.MustFreeMiniPort(t, "RisingWave")
env.masterPort = mustFreePort(t, "Master")
env.filerPort = mustFreePort(t, "Filer")
env.s3Port = mustFreePort(t, "S3")
env.icebergRestPort = mustFreePort(t, "Iceberg")
env.risingwavePort = mustFreePort(t, "RisingWave")
env.bindIP = testutil.FindBindIP()
@@ -160,20 +161,62 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
registerMiniProcess(env.masterProcess)
// Wait for all services to be ready
if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
if !waitForPort(env.masterPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
}
if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
if !waitForPort(env.filerPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
}
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
if !waitForPort(env.s3Port, 15*time.Second) {
t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
}
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
if !waitForPort(env.icebergRestPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
}
}
func mustFreePort(t *testing.T, name string) int {
t.Helper()
minPort := 10000
maxPort := 55000 // Ensure port+10000 < 65535
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for i := 0; i < 1000; i++ {
port := minPort + r.Intn(maxPort-minPort)
// Check http port
ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
if err != nil {
continue
}
ln.Close()
// Check grpc port (weed mini uses port+10000)
ln2, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port+10000))
if err != nil {
continue
}
ln2.Close()
return port
}
t.Fatalf("failed to find a free port < %d for %s after 1000 attempts", maxPort, name)
return 0
}
func waitForPort(port int, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
if err == nil {
conn.Close()
return true
}
time.Sleep(100 * time.Millisecond)
}
return false
}
func (env *TestEnvironment) StartRisingWave(t *testing.T) {
t.Helper()
@@ -210,7 +253,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) {
}
// Wait for RisingWave port to be open on host
if !testutil.WaitForPort(env.risingwavePort, 120*time.Second) {
if !waitForPort(env.risingwavePort, 120*time.Second) {
t.Fatalf("timed out waiting for RisingWave port %d to be open", env.risingwavePort)
}
+49 -10
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"math/rand"
"net"
"os"
"os/exec"
"path/filepath"
@@ -90,10 +91,10 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Fatalf("failed to create temp directory: %v", err)
}
env.masterPort = testutil.MustFreeMiniPort(t, "Master")
env.filerPort = testutil.MustFreeMiniPort(t, "Filer")
env.s3Port = testutil.MustFreeMiniPort(t, "S3")
env.icebergRestPort = testutil.MustFreeMiniPort(t, "Iceberg")
env.masterPort = mustFreePort(t, "Master")
env.filerPort = mustFreePort(t, "Filer")
env.s3Port = mustFreePort(t, "S3")
env.icebergRestPort = mustFreePort(t, "Iceberg")
bindIP := testutil.FindBindIP()
@@ -126,20 +127,58 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
registerMiniProcess(env.masterProcess)
// Wait for all services to be ready
if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
if !waitForPort(env.masterPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
}
if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
if !waitForPort(env.filerPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
}
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
if !waitForPort(env.s3Port, 15*time.Second) {
t.Fatalf("weed mini failed to start - s3 port %d not listening", env.s3Port)
}
if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
if !waitForPort(env.icebergRestPort, 15*time.Second) {
t.Fatalf("weed mini failed to start - iceberg rest port %d not listening", env.icebergRestPort)
}
}
func mustFreePort(t *testing.T, name string) int {
t.Helper()
for i := 0; i < 200; i++ {
port := 20000 + rand.Intn(30000)
listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", port))
if err != nil {
continue
}
listener.Close()
grpcPort := port + 10000
if grpcPort > 65535 {
continue
}
grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", grpcPort))
if err != nil {
continue
}
grpcListener.Close()
return port
}
t.Fatalf("failed to get free port for %s", name)
return 0
}
func waitForPort(port int, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", port), 500*time.Millisecond)
if err == nil {
conn.Close()
return true
}
time.Sleep(100 * time.Millisecond)
}
return false
}
func (env *TestEnvironment) writeSparkConfig(t *testing.T, catalogBucket string) string {
t.Helper()
-15
View File
@@ -2,7 +2,6 @@ package testutil
import (
"context"
"fmt"
"net"
"net/http"
"os/exec"
@@ -65,17 +64,3 @@ func WaitForService(url string, timeout time.Duration) bool {
}
}
}
func WaitForPort(port int, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
address := fmt.Sprintf("127.0.0.1:%d", port)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
if err == nil {
_ = conn.Close()
return true
}
time.Sleep(100 * time.Millisecond)
}
return false
}
-37
View File
@@ -2,16 +2,11 @@ package testutil
import (
"fmt"
"math/rand"
"net"
"os"
"path/filepath"
"testing"
"time"
)
const SeaweedMiniStartupTimeout = 45 * time.Second
func FindBindIP() string {
addrs, err := net.InterfaceAddrs()
if err != nil {
@@ -59,35 +54,3 @@ func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
}
return iamConfigPath, nil
}
func MustFreeMiniPort(t *testing.T, name string) int {
t.Helper()
const (
minPort = 10000
maxPort = 55000
)
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for i := 0; i < 1000; i++ {
port := minPort + r.Intn(maxPort-minPort)
listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
if err != nil {
continue
}
_ = listener.Close()
grpcPort := port + 10000
grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort))
if err != nil {
continue
}
_ = grpcListener.Close()
return port
}
t.Fatalf("failed to get free weed mini port for %s", name)
return 0
}
@@ -1,294 +0,0 @@
package volume_server_grpc_test
import (
"context"
"net/http"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
)
// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
// decoded volume produced by VolumeEcShardsToVolume.
func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartVolumeCluster(t, matrix.P1())
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
defer conn.Close()
const (
volumeID = uint32(140)
keyA = uint64(990020)
cookieA = uint32(0xDA001122)
keyB = uint64(990021)
cookieB = uint32(0xDA003344)
)
framework.AllocateVolume(t, client, volumeID, "")
httpClient := framework.NewHTTPClient()
fidA := framework.NewFileID(volumeID, keyA, cookieA)
fidB := framework.NewFileID(volumeID, keyB, cookieB)
payloadA := []byte("needle-A-should-be-deleted-after-decode")
payloadB := []byte("needle-B-should-survive-decode")
// Upload two needles.
for _, tc := range []struct {
fid string
payload []byte
}{
{fidA, payloadA},
{fidB, payloadB},
} {
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// EC encode.
_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsGenerate: %v", err)
}
// Mount all data shards so the EC volume is usable.
_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: "",
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
})
if err != nil {
t.Fatalf("VolumeEcShardsMount: %v", err)
}
// Delete needle A via EC path (writes to .ecj).
_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
VolumeId: volumeID, Collection: "",
FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
})
if err != nil {
t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
}
// Unmount the normal volume so decode writes fresh files.
_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeUnmount: %v", err)
}
// Decode EC shards back to a normal volume.
_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsToVolume: %v", err)
}
// Re-mount the decoded volume.
_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeMount: %v", err)
}
// Needle A should be gone (deleted via .ecj before decode).
respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
bodyA := framework.ReadAllAndClose(t, respA)
if respA.StatusCode >= 500 {
t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
}
if respA.StatusCode != http.StatusNotFound {
t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
}
// Needle B should still be readable.
respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
bodyB := framework.ReadAllAndClose(t, respB)
if respB.StatusCode != http.StatusOK {
t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
}
if string(bodyB) != string(payloadB) {
t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
}
}
// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
// peer server that contributes no new data shards are still collected during
// decode. This is the regression test for the fix in collectEcShards that
// always copies .ecj from every shard location.
//
// Scenario:
// - Server 0 holds all 10 data shards (decode target).
// - Server 1 holds a copy of shard 0 (no new shards for server 0).
// - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
// - During decode on server 0, server 1's .ecj must be collected and applied.
func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer conn0.Close()
conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer conn1.Close()
const (
volumeID = uint32(141)
keyA = uint64(990030)
cookieA = uint32(0xDB001122)
keyB = uint64(990031)
cookieB = uint32(0xDB003344)
)
// Allocate and upload on server 0.
framework.AllocateVolume(t, client0, volumeID, "")
httpClient := framework.NewHTTPClient()
fidA := framework.NewFileID(volumeID, keyA, cookieA)
fidB := framework.NewFileID(volumeID, keyB, cookieB)
payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
}
resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != http.StatusCreated {
t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// EC encode on server 0.
_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
}
// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
// Copy shard 0 + ecx + ecj from server 0 → server 1.
_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: volumeID,
Collection: "",
SourceDataNode: sourceDataNode,
ShardIds: []uint32{0},
CopyEcxFile: true,
CopyEcjFile: true,
CopyVifFile: true,
})
if err != nil {
t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
}
// Mount shard 0 on server 1 so the EC volume can accept deletions.
_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: "",
ShardIds: []uint32{0},
})
if err != nil {
t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
}
// Delete needle A on server 1 only (creates .ecj entry on server 1).
_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
VolumeId: volumeID, Collection: "",
FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
})
if err != nil {
t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
}
// Mount all data shards on server 0 (the decode target).
_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: "",
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
})
if err != nil {
t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
}
// Collect .ecj from server 1 → server 0 with NO new shard IDs.
// This is the critical path: server 1 has shard 0 which server 0 already
// has, so needToCopyShardsInfo would be empty. Before the fix in
// collectEcShards, this copy would be skipped entirely, losing server 1's
// deletion entries.
server1DataNode := cluster.VolumeAdminAddress(1) + "." +
strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
VolumeId: volumeID,
Collection: "",
SourceDataNode: server1DataNode,
ShardIds: []uint32{}, // No new shards — just .ecj.
CopyEcxFile: false,
CopyEcjFile: true,
CopyVifFile: false,
})
if err != nil {
t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
}
// Unmount the normal volume before decode.
_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeUnmount on server 0: %v", err)
}
// Decode on server 0. RebuildEcxFile should see needle A's deletion from
// the .ecj that was collected from server 1.
_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
VolumeId: volumeID, Collection: "",
})
if err != nil {
t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
}
// Re-mount the decoded normal volume.
_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("VolumeMount on server 0: %v", err)
}
// Needle A should be gone — its deletion was in server 1's .ecj.
respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
bodyA := framework.ReadAllAndClose(t, respA)
if respA.StatusCode >= 500 {
t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
}
if respA.StatusCode != http.StatusNotFound {
t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
}
// Needle B should still be readable.
respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
bodyB := framework.ReadAllAndClose(t, respB)
if respB.StatusCode != http.StatusOK {
t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
}
if string(bodyB) != string(payloadB) {
t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
}
}
+158 -5
View File
@@ -943,7 +943,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
leaderCount++
}
masterMap[masterInfo.Address] = masterInfo
masterMap[master.Address] = masterInfo
}
// Then, get additional master information from Raft cluster
@@ -955,11 +955,11 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
// Process each raft server
for _, server := range resp.ClusterServers {
// Raft stores gRPC addresses, convert to HTTP address
httpAddress := pb.GrpcAddressToServerAddress(server.Address)
address := server.Address
httpAddress := pb.ServerAddress(address).ToHttpAddress()
// Update existing master info or create new one
if masterInfo, exists := masterMap[httpAddress]; exists {
if masterInfo, exists := masterMap[address]; exists {
// Update existing master with raft data
masterInfo.IsLeader = server.IsLeader
masterInfo.Suffrage = server.Suffrage
@@ -970,7 +970,7 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
IsLeader: server.IsLeader,
Suffrage: server.Suffrage,
}
masterMap[httpAddress] = masterInfo
masterMap[address] = masterInfo
}
if server.IsLeader {
@@ -1646,6 +1646,159 @@ func (as *AdminServer) GetConfigPersistence() *ConfigPersistence {
return as.configPersistence
}
// convertJSONToMaintenanceConfig converts JSON map to protobuf MaintenanceConfig
func convertJSONToMaintenanceConfig(jsonConfig map[string]interface{}) (*maintenance.MaintenanceConfig, error) {
config := &maintenance.MaintenanceConfig{}
// Helper function to get int32 from interface{}
getInt32 := func(key string) (int32, error) {
if val, ok := jsonConfig[key]; ok {
switch v := val.(type) {
case int:
return int32(v), nil
case int32:
return v, nil
case int64:
return int32(v), nil
case float64:
return int32(v), nil
default:
return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
}
}
return 0, nil
}
// Helper function to get bool from interface{}
getBool := func(key string) bool {
if val, ok := jsonConfig[key]; ok {
if b, ok := val.(bool); ok {
return b
}
}
return false
}
var err error
// Convert basic fields
config.Enabled = getBool("enabled")
if config.ScanIntervalSeconds, err = getInt32("scan_interval_seconds"); err != nil {
return nil, err
}
if config.WorkerTimeoutSeconds, err = getInt32("worker_timeout_seconds"); err != nil {
return nil, err
}
if config.TaskTimeoutSeconds, err = getInt32("task_timeout_seconds"); err != nil {
return nil, err
}
if config.RetryDelaySeconds, err = getInt32("retry_delay_seconds"); err != nil {
return nil, err
}
if config.MaxRetries, err = getInt32("max_retries"); err != nil {
return nil, err
}
if config.CleanupIntervalSeconds, err = getInt32("cleanup_interval_seconds"); err != nil {
return nil, err
}
if config.TaskRetentionSeconds, err = getInt32("task_retention_seconds"); err != nil {
return nil, err
}
// Convert policy if present
if policyData, ok := jsonConfig["policy"]; ok {
if policyMap, ok := policyData.(map[string]interface{}); ok {
policy := &maintenance.MaintenancePolicy{}
if globalMaxConcurrent, err := getInt32FromMap(policyMap, "global_max_concurrent"); err != nil {
return nil, err
} else {
policy.GlobalMaxConcurrent = globalMaxConcurrent
}
if defaultRepeatIntervalSeconds, err := getInt32FromMap(policyMap, "default_repeat_interval_seconds"); err != nil {
return nil, err
} else {
policy.DefaultRepeatIntervalSeconds = defaultRepeatIntervalSeconds
}
if defaultCheckIntervalSeconds, err := getInt32FromMap(policyMap, "default_check_interval_seconds"); err != nil {
return nil, err
} else {
policy.DefaultCheckIntervalSeconds = defaultCheckIntervalSeconds
}
// Convert task policies if present
if taskPoliciesData, ok := policyMap["task_policies"]; ok {
if taskPoliciesMap, ok := taskPoliciesData.(map[string]interface{}); ok {
policy.TaskPolicies = make(map[string]*maintenance.TaskPolicy)
for taskType, taskPolicyData := range taskPoliciesMap {
if taskPolicyMap, ok := taskPolicyData.(map[string]interface{}); ok {
taskPolicy := &maintenance.TaskPolicy{}
taskPolicy.Enabled = getBoolFromMap(taskPolicyMap, "enabled")
if maxConcurrent, err := getInt32FromMap(taskPolicyMap, "max_concurrent"); err != nil {
return nil, err
} else {
taskPolicy.MaxConcurrent = maxConcurrent
}
if repeatIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "repeat_interval_seconds"); err != nil {
return nil, err
} else {
taskPolicy.RepeatIntervalSeconds = repeatIntervalSeconds
}
if checkIntervalSeconds, err := getInt32FromMap(taskPolicyMap, "check_interval_seconds"); err != nil {
return nil, err
} else {
taskPolicy.CheckIntervalSeconds = checkIntervalSeconds
}
policy.TaskPolicies[taskType] = taskPolicy
}
}
}
}
config.Policy = policy
}
}
return config, nil
}
// Helper functions for map conversion
func getInt32FromMap(m map[string]interface{}, key string) (int32, error) {
if val, ok := m[key]; ok {
switch v := val.(type) {
case int:
return int32(v), nil
case int32:
return v, nil
case int64:
return int32(v), nil
case float64:
return int32(v), nil
default:
return 0, fmt.Errorf("invalid type for %s: expected number, got %T", key, v)
}
}
return 0, nil
}
func getBoolFromMap(m map[string]interface{}, key string) bool {
if val, ok := m[key]; ok {
if b, ok := val.(bool); ok {
return b
}
}
return false
}
type collectionStats struct {
PhysicalSize int64
LogicalSize int64
+20
View File
@@ -361,6 +361,26 @@ func normalizeQuotaUnit(unit string) (string, error) {
}
}
// Helper function to convert bytes to appropriate unit and size
func convertBytesToQuota(bytes int64) (int64, string) {
if bytes == 0 {
return 0, "MB"
}
// Convert to TB if >= 1TB
if bytes >= 1024*1024*1024*1024 && bytes%(1024*1024*1024*1024) == 0 {
return bytes / (1024 * 1024 * 1024 * 1024), "TB"
}
// Convert to GB if >= 1GB
if bytes >= 1024*1024*1024 && bytes%(1024*1024*1024) == 0 {
return bytes / (1024 * 1024 * 1024), "GB"
}
// Convert to MB (default)
return bytes / (1024 * 1024), "MB"
}
// SetBucketQuota sets the quota for a bucket
func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+12
View File
@@ -506,6 +506,18 @@ func getShardCount(ecIndexBits uint32) int {
return count
}
// getMissingShards returns a slice of missing shard IDs for a volume
// Assumes default 10+4 EC configuration (14 total shards)
func getMissingShards(ecIndexBits uint32) []int {
var missing []int
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
if (ecIndexBits & (1 << uint(i))) == 0 {
missing = append(missing, i)
}
}
return missing
}
// sortEcShards sorts EC shards based on the specified field and order
func sortEcShards(shards []EcShardWithInfo, sortBy string, sortOrder string) {
sort.Slice(shards, func(i, j int) bool {
+61
View File
@@ -430,6 +430,67 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
return offsets, nil
}
// convertRecordTypeToSchemaFields converts a protobuf RecordType to SchemaFieldInfo slice
func convertRecordTypeToSchemaFields(recordType *schema_pb.RecordType) []SchemaFieldInfo {
var schemaFields []SchemaFieldInfo
if recordType == nil || recordType.Fields == nil {
return schemaFields
}
for _, field := range recordType.Fields {
schemaField := SchemaFieldInfo{
Name: field.Name,
Type: getFieldTypeString(field.Type),
Required: field.IsRequired,
}
schemaFields = append(schemaFields, schemaField)
}
return schemaFields
}
// getFieldTypeString converts a protobuf Type to a human-readable string
func getFieldTypeString(fieldType *schema_pb.Type) string {
if fieldType == nil {
return "unknown"
}
switch kind := fieldType.Kind.(type) {
case *schema_pb.Type_ScalarType:
return getScalarTypeString(kind.ScalarType)
case *schema_pb.Type_RecordType:
return "record"
case *schema_pb.Type_ListType:
elementType := getFieldTypeString(kind.ListType.ElementType)
return fmt.Sprintf("list<%s>", elementType)
default:
return "unknown"
}
}
// getScalarTypeString converts a protobuf ScalarType to a string
func getScalarTypeString(scalarType schema_pb.ScalarType) string {
switch scalarType {
case schema_pb.ScalarType_BOOL:
return "bool"
case schema_pb.ScalarType_INT32:
return "int32"
case schema_pb.ScalarType_INT64:
return "int64"
case schema_pb.ScalarType_FLOAT:
return "float"
case schema_pb.ScalarType_DOUBLE:
return "double"
case schema_pb.ScalarType_BYTES:
return "bytes"
case schema_pb.ScalarType_STRING:
return "string"
default:
return "unknown"
}
}
// convertTopicPublishers converts protobuf TopicPublisher slice to PublisherInfo slice
func convertTopicPublishers(publishers []*mq_pb.TopicPublisher) []PublisherInfo {
publisherInfos := make([]PublisherInfo, 0, len(publishers))
+39
View File
@@ -2,6 +2,8 @@ package dash
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -849,6 +851,43 @@ func normalizeTimeout(timeoutSeconds int, defaultTimeout, maxTimeout time.Durati
return timeout
}
func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, index int) *plugin_pb.JobSpec {
now := timestamppb.Now()
suffix := make([]byte, 4)
if _, err := rand.Read(suffix); err != nil {
// Fallback to simpler ID if rand fails
suffix = []byte(fmt.Sprintf("%d", index))
}
jobID := fmt.Sprintf("%s-%d-%s", jobType, now.AsTime().UnixNano(), hex.EncodeToString(suffix))
jobSpec := &plugin_pb.JobSpec{
JobId: jobID,
JobType: jobType,
Priority: plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
CreatedAt: now,
Labels: make(map[string]string),
Parameters: make(map[string]*plugin_pb.ConfigValue),
DedupeKey: "",
}
if proposal != nil {
jobSpec.Summary = proposal.Summary
jobSpec.Detail = proposal.Detail
if proposal.Priority != plugin_pb.JobPriority_JOB_PRIORITY_UNSPECIFIED {
jobSpec.Priority = proposal.Priority
}
jobSpec.DedupeKey = proposal.DedupeKey
jobSpec.Parameters = plugin.CloneConfigValueMap(proposal.Parameters)
if proposal.Labels != nil {
for k, v := range proposal.Labels {
jobSpec.Labels[k] = v
}
}
}
return jobSpec
}
func applyDescriptorDefaultsToPersistedConfig(
config *plugin_pb.PersistedJobTypeConfig,
descriptor *plugin_pb.JobTypeDescriptor,
+26
View File
@@ -115,6 +115,32 @@ func TestExpirePluginJobAPI(t *testing.T) {
})
}
func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
t.Parallel()
proposal := &plugin_pb.JobProposal{
ProposalId: "vacuum-2",
DedupeKey: "vacuum:2",
JobType: "vacuum",
}
jobA := buildJobSpecFromProposal("vacuum", proposal, 0)
jobB := buildJobSpecFromProposal("vacuum", proposal, 1)
if jobA.JobId == proposal.ProposalId {
t.Fatalf("job id must not reuse proposal id: %s", jobA.JobId)
}
if jobB.JobId == proposal.ProposalId {
t.Fatalf("job id must not reuse proposal id: %s", jobB.JobId)
}
if jobA.JobId == jobB.JobId {
t.Fatalf("job ids must be unique across jobs: %s", jobA.JobId)
}
if jobA.DedupeKey != proposal.DedupeKey {
t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
}
}
func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
t.Parallel()
+9 -2
View File
@@ -108,8 +108,6 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
go s.cleanupRoutine()
go s.activeLogFetchLoop()
pb.ServeGrpcOnLocalSocket(grpcServer, port)
// Start serving in a goroutine
go func() {
if err := s.grpcServer.Serve(listener); err != nil {
@@ -787,6 +785,15 @@ func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntri
return results, nil
}
// convertTaskParameters converts task parameters to protobuf format
func convertTaskParameters(params map[string]interface{}) map[string]string {
result := make(map[string]string)
for key, value := range params {
result[key] = fmt.Sprintf("%v", value)
}
return result
}
func findClientAddress(ctx context.Context) string {
// fmt.Printf("FromContext %+v\n", ctx)
pr, ok := peer.FromContext(ctx)
+1 -11
View File
@@ -43,8 +43,7 @@ type LiveLock struct {
lc *LockClient
owner string
lockTTL time.Duration
consecutiveFailures int // Track connection failures to trigger fallback
generation int64 // fencing token from the lock server
consecutiveFailures int // Track connection failures to trigger fallback
}
// NewShortLivedLock creates a lock with a 5-second duration
@@ -215,9 +214,6 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
glog.V(4).Infof("LOCK: DistributedLock response - key=%s err=%v", lock.key, err)
if err == nil && resp != nil {
lock.renewToken = resp.RenewToken
if resp.Generation > 0 {
atomic.StoreInt64(&lock.generation, resp.Generation)
}
lock.consecutiveFailures = 0 // Reset failure counter on success
glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
} else {
@@ -268,12 +264,6 @@ func (lock *LiveLock) LockOwner() string {
return lock.owner
}
// Generation returns the fencing token for this lock.
// It increments on each fresh acquisition and stays the same on renewal.
func (lock *LiveLock) Generation() int64 {
return atomic.LoadInt64(&lock.generation)
}
// IsLocked returns true if this instance currently holds the lock
func (lock *LiveLock) IsLocked() bool {
return atomic.LoadInt32(&lock.isLocked) == 1
@@ -13,17 +13,10 @@ const LiveLockTTL = time.Second * 7
var NoLockServerError = fmt.Errorf("no lock server found")
// ReplicateFunc is called to replicate a lock operation to a backup server.
// The caller (filer server) provides this to avoid a circular dependency.
// seq is a per-lock monotonic sequence number for causal ordering — the backup
// rejects mutations with seq <= its current seq for that key.
type ReplicateFunc func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool)
type DistributedLockManager struct {
lockManager *LockManager
LockRing *LockRing
Host pb.ServerAddress
ReplicateFn ReplicateFunc // set by filer server after creation
lockManager *LockManager
LockRing *LockRing
Host pb.ServerAddress
}
func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
@@ -34,51 +27,35 @@ func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
}
}
func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, generation int64, movedTo pb.ServerAddress, err error) {
primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
if primary == "" {
func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, movedTo pb.ServerAddress, err error) {
movedTo, err = dlm.findLockOwningFiler(key)
if err != nil {
return
}
if movedTo != dlm.Host {
return
}
lockOwner, renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
return
}
func (dlm *DistributedLockManager) findLockOwningFiler(key string) (movedTo pb.ServerAddress, err error) {
servers := dlm.LockRing.GetSnapshot()
if servers == nil {
err = NoLockServerError
return
}
if primary != dlm.Host {
// If this is a renewal (non-empty token) and we still hold the lock locally,
// serve it here rather than redirecting. This handles the window between
// ring update and lock transfer completion — the old primary remains
// authoritative for locks it still holds.
if token != "" {
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
var seq int64
lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
if err == nil && renewToken != "" {
dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
}
return
}
}
movedTo = primary
return
}
var seq int64
lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
if err == nil && renewToken != "" {
dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
}
movedTo = hashKeyToServer(key, servers)
return
}
func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, movedTo pb.ServerAddress, err error) {
primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
if primary == "" {
err = NoLockServerError
movedTo, err = dlm.findLockOwningFiler(key)
if err != nil {
return
}
if primary != dlm.Host {
// If we still hold this lock locally, serve it here
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup {
owner = lock.Owner
return
}
movedTo = primary
if movedTo != dlm.Host {
servers := dlm.LockRing.GetSnapshot()
glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
return
@@ -88,60 +65,26 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
}
func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.ServerAddress, err error) {
primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
if primary == "" {
servers := dlm.LockRing.GetSnapshot()
if servers == nil {
err = NoLockServerError
return
}
if primary != dlm.Host {
// If we still hold this lock locally, serve the unlock here
if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
var isUnlocked bool
var generation int64
var seq int64
isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
if isUnlocked {
dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
}
return
}
movedTo = primary
server := hashKeyToServer(key, servers)
if server != dlm.Host {
movedTo = server
return
}
var isUnlocked bool
var generation int64
var seq int64
isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
if isUnlocked {
dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
}
_, err = dlm.lockManager.Unlock(key, token)
return
}
// InsertLock is used to insert a lock to a server unconditionally.
// It is used when a server is down and the lock is moved to another server.
// After inserting, it replicates to the backup for this key.
func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
if dlm.lockManager.InsertLock(key, expiredAtNs, token, owner, generation, seq) {
dlm.replicateToBackup(key, expiredAtNs, token, owner, generation, seq, false)
}
// InsertLock is used to insert a lock to a server unconditionally
// It is used when a server is down and the lock is moved to another server
func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string) {
dlm.lockManager.InsertLock(key, expiredAtNs, token, owner)
}
// InsertBackupLock inserts a lock as a backup copy, rejecting stale seq
func (dlm *DistributedLockManager) InsertBackupLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
dlm.lockManager.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
}
// RemoveBackupLock removes a backup lock unconditionally
func (dlm *DistributedLockManager) RemoveBackupLock(key string) {
dlm.lockManager.RemoveLock(key)
}
// RemoveBackupLockIfSeq removes a local copy only if the incoming mutation is not older.
func (dlm *DistributedLockManager) RemoveBackupLockIfSeq(key string, generation int64, seq int64) {
dlm.lockManager.RemoveBackupLockIfSeq(key, generation, seq)
}
func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
return dlm.lockManager.SelectLocks(func(key string) bool {
server := hashKeyToServer(key, servers)
@@ -153,40 +96,9 @@ func (dlm *DistributedLockManager) CalculateTargetServer(key string, servers []p
}
func (dlm *DistributedLockManager) IsLocal(key string) bool {
primary := dlm.LockRing.GetPrimary(key)
if primary == "" {
servers := dlm.LockRing.GetSnapshot()
if len(servers) <= 1 {
return true
}
return primary == dlm.Host
}
// AllLocks returns all non-expired locks on this node
func (dlm *DistributedLockManager) AllLocks() []*Lock {
return dlm.lockManager.AllLocks()
}
// PromoteLock promotes a backup lock to primary
func (dlm *DistributedLockManager) PromoteLock(key string) bool {
return dlm.lockManager.PromoteLock(key)
}
// DemoteLock demotes a primary lock to backup
func (dlm *DistributedLockManager) DemoteLock(key string) bool {
return dlm.lockManager.DemoteLock(key)
}
// GetLock returns a copy of a lock if it exists
func (dlm *DistributedLockManager) GetLock(key string) (*Lock, bool) {
return dlm.lockManager.GetLock(key)
}
// replicateToBackup asynchronously replicates a lock operation to the backup server
func (dlm *DistributedLockManager) replicateToBackup(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
_, backup := dlm.LockRing.GetPrimaryAndBackup(key)
if backup == "" {
return // single-server deployment, no backup
}
if dlm.ReplicateFn != nil {
go dlm.ReplicateFn(backup, key, expiredAtNs, token, owner, generation, seq, isUnlock)
}
return hashKeyToServer(key, servers) == dlm.Host
}
@@ -1,710 +0,0 @@
package lock_manager
import (
"fmt"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// testCluster simulates a cluster of filer nodes with DLMs.
// It wires up ReplicateFn so that replication calls arrive at the
// correct peer's DLM, enabling end-to-end backup testing without gRPC.
type testCluster struct {
mu sync.Mutex
nodes map[pb.ServerAddress]*DistributedLockManager
}
func newTestCluster(hosts ...pb.ServerAddress) *testCluster {
c := &testCluster{nodes: make(map[pb.ServerAddress]*DistributedLockManager)}
servers := make([]pb.ServerAddress, len(hosts))
copy(servers, hosts)
for _, host := range hosts {
dlm := NewDistributedLockManager(host)
dlm.LockRing.SetSnapshot(servers, 0)
c.nodes[host] = dlm
}
// Wire up replication: each node's ReplicateFn calls the backup's DLM directly
for _, dlm := range c.nodes {
d := dlm // capture
d.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
c.mu.Lock()
target, ok := c.nodes[server]
c.mu.Unlock()
if !ok {
return // server is down
}
if isUnlock {
target.RemoveBackupLockIfSeq(key, generation, seq)
} else {
target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
}
}
}
return c
}
func (c *testCluster) removeNode(host pb.ServerAddress) {
c.mu.Lock()
delete(c.nodes, host)
c.mu.Unlock()
// Update all remaining nodes' rings
remaining := c.getServers()
for _, dlm := range c.getNodes() {
dlm.LockRing.SetSnapshot(remaining, 0)
}
}
func (c *testCluster) addNode(host pb.ServerAddress) {
c.mu.Lock()
dlm := NewDistributedLockManager(host)
c.nodes[host] = dlm
c.mu.Unlock()
// Wire up replication
dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
c.mu.Lock()
target, ok := c.nodes[server]
c.mu.Unlock()
if !ok {
return
}
if isUnlock {
target.RemoveBackupLockIfSeq(key, generation, seq)
} else {
target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
}
}
servers := c.getServers()
for _, n := range c.getNodes() {
n.LockRing.SetSnapshot(servers, 0)
}
}
func (c *testCluster) getNodes() map[pb.ServerAddress]*DistributedLockManager {
c.mu.Lock()
defer c.mu.Unlock()
cp := make(map[pb.ServerAddress]*DistributedLockManager, len(c.nodes))
for k, v := range c.nodes {
cp[k] = v
}
return cp
}
func (c *testCluster) getServers() []pb.ServerAddress {
c.mu.Lock()
defer c.mu.Unlock()
var servers []pb.ServerAddress
for s := range c.nodes {
servers = append(servers, s)
}
return servers
}
func (c *testCluster) get(host pb.ServerAddress) *DistributedLockManager {
c.mu.Lock()
defer c.mu.Unlock()
return c.nodes[host]
}
// acquireLock tries to acquire a lock on the correct primary node.
// It follows redirects (movedTo) like a real client would.
func (c *testCluster) acquireLock(key, owner string, ttl time.Duration) (renewToken string, generation int64, primaryHost pb.ServerAddress, err error) {
// Try any node first (simulates client connecting to seed filer)
for _, dlm := range c.getNodes() {
expiry := time.Now().Add(ttl).UnixNano()
var movedTo pb.ServerAddress
var lockErr error
_, renewToken, generation, movedTo, lockErr = dlm.LockWithTimeout(key, expiry, "", owner)
if movedTo != "" && movedTo != dlm.Host {
// Follow redirect
target := c.get(movedTo)
if target == nil {
err = fmt.Errorf("primary %s is down", movedTo)
return
}
_, renewToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, "", owner)
if lockErr != nil {
err = lockErr
return
}
primaryHost = movedTo
// Wait briefly for async replication to complete
time.Sleep(10 * time.Millisecond)
return
}
if lockErr != nil {
err = lockErr
return
}
primaryHost = dlm.Host
time.Sleep(10 * time.Millisecond)
return
}
err = fmt.Errorf("no nodes available")
return
}
// renewLock renews a lock on the primary node
func (c *testCluster) renewLock(key, owner, token string, ttl time.Duration, primaryHost pb.ServerAddress) (newToken string, generation int64, err error) {
target := c.get(primaryHost)
if target == nil {
err = fmt.Errorf("primary %s is down", primaryHost)
return
}
expiry := time.Now().Add(ttl).UnixNano()
var movedTo pb.ServerAddress
var lockErr error
_, newToken, generation, movedTo, lockErr = target.LockWithTimeout(key, expiry, token, owner)
if movedTo != "" && movedTo != primaryHost {
target = c.get(movedTo)
if target == nil {
err = fmt.Errorf("new primary %s is down", movedTo)
return
}
// Pass the existing token so the redirected renewal can match
// if the lock was already transferred to the new primary.
_, newToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, token, owner)
}
err = lockErr
time.Sleep(10 * time.Millisecond)
return
}
// --- Test Cases ---
func TestDLM_PrimaryCrash_BackupPromotes(t *testing.T) {
// Scenario: Lock is acquired, primary crashes, backup should have the lock
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
key := "test-lock-primary-crash"
renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
require.NoError(t, err)
require.NotEmpty(t, renewToken)
// Find the backup for this key
_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
require.NotEmpty(t, backup, "should have a backup server")
// Verify backup has the lock
backupDlm := cluster.get(backup)
backupLock, found := backupDlm.GetLock(key)
require.True(t, found, "backup should have the lock")
assert.True(t, backupLock.IsBackup, "lock on backup should be marked as backup")
assert.Equal(t, renewToken, backupLock.Token, "backup should have the same token")
// Crash the primary
cluster.removeNode(primaryHost)
// Simulate topology change: promote backup locks
for _, dlm := range cluster.getNodes() {
locks := dlm.AllLocks()
for _, lock := range locks {
newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
if newPrimary == dlm.Host && lock.IsBackup {
dlm.PromoteLock(lock.Key)
}
}
}
// The backup should now be the primary
newPrimary := backupDlm.LockRing.GetPrimary(key)
assert.Equal(t, backup, newPrimary, "backup should be the new primary")
// The promoted lock should work — verify it's no longer a backup
promotedLock, found := backupDlm.GetLock(key)
require.True(t, found, "lock should still exist after promotion")
assert.False(t, promotedLock.IsBackup, "lock should be promoted to primary")
// Client should be able to renew with the same token on the new primary
newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, backup)
require.NoError(t, err)
assert.NotEmpty(t, newToken, "renewal on new primary should succeed")
}
func TestDLM_BackupCrash_PrimaryContinues(t *testing.T) {
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
key := "test-lock-backup-crash"
renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
require.NoError(t, err)
_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
// Crash the backup
cluster.removeNode(backup)
// Primary should still work — renew the lock
newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
require.NoError(t, err)
assert.NotEmpty(t, newToken, "primary should continue working after backup crash")
// Verify primary is still the primary for this key
newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
assert.Equal(t, primaryHost, newPrimary)
}
func TestDLM_BothPrimaryAndBackupCrash(t *testing.T) {
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
key := "test-lock-both-crash"
_, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
require.NoError(t, err)
_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
// Crash both
cluster.removeNode(primaryHost)
cluster.removeNode(backup)
// The lock is lost — the surviving node should be able to acquire it fresh
newToken, _, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
require.NoError(t, err)
assert.NotEmpty(t, newToken, "new owner should acquire lock after both crash")
}
func TestDLM_RollingRestart(t *testing.T) {
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
// Acquire multiple locks
type lockState struct {
key, owner, token string
generation int64
primary pb.ServerAddress
}
locks := make([]lockState, 5)
for i := range locks {
key := fmt.Sprintf("rolling-lock-%d", i)
token, gen, primary, err := cluster.acquireLock(key, fmt.Sprintf("owner-%d", i), 30*time.Second)
require.NoError(t, err)
locks[i] = lockState{key: key, owner: fmt.Sprintf("owner-%d", i), token: token, generation: gen, primary: primary}
}
// Rolling restart: remove and re-add each node one at a time.
// After removing a node, promote backups and re-replicate to new backups
// to maintain the invariant that each lock has a backup copy.
for _, host := range hosts {
cluster.removeNode(host)
// Simulate full OnDlmChangeSnapshot: promote backups and re-replicate
for _, dlm := range cluster.getNodes() {
for _, lock := range dlm.AllLocks() {
newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
if newPrimary == dlm.Host && lock.IsBackup {
dlm.PromoteLock(lock.Key)
}
}
// Re-replicate all primary locks to their new backups
for _, lock := range dlm.AllLocks() {
newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
if newPrimary == dlm.Host && !lock.IsBackup {
dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
}
}
}
time.Sleep(10 * time.Millisecond)
// Re-add the node
cluster.addNode(host)
time.Sleep(10 * time.Millisecond)
}
// After rolling restart, locks should survive via backup promotion
survivedCount := 0
for _, ls := range locks {
for _, dlm := range cluster.getNodes() {
lock, found := dlm.GetLock(ls.key)
if found && !lock.IsBackup {
survivedCount++
break
}
}
}
t.Logf("Locks survived rolling restart: %d / %d", survivedCount, len(locks))
require.Greater(t, survivedCount, 0, "at least some locks should survive a rolling restart via backup promotion")
}
func TestDLM_GenerationIncrementsOnNewAcquisition(t *testing.T) {
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
cluster := newTestCluster(hosts...)
key := "gen-test-lock"
// Acquire lock — generation should be > 0
token1, gen1, primary, err := cluster.acquireLock(key, "owner1", 2*time.Second)
require.NoError(t, err)
assert.Greater(t, gen1, int64(0))
// Renew — generation should stay the same
token2, gen2, err := cluster.renewLock(key, "owner1", token1, 2*time.Second, primary)
require.NoError(t, err)
assert.Equal(t, gen1, gen2, "generation should not change on renewal")
// Let lock expire
time.Sleep(3 * time.Second)
// Re-acquire — generation should increment
_, gen3, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
require.NoError(t, err)
assert.Greater(t, gen3, gen1, "generation should increment on new acquisition")
_ = token2
}
func TestDLM_ReplicationFailure_PrimaryStillWorks(t *testing.T) {
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
// Break replication by setting a no-op ReplicateFn on all nodes
for _, dlm := range cluster.getNodes() {
dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
// Simulate replication failure: do nothing
}
}
key := "repl-fail-lock"
renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
require.NoError(t, err)
// Primary should have the lock
primaryDlm := cluster.get(primaryHost)
lock, found := primaryDlm.GetLock(key)
require.True(t, found, "primary should have the lock")
assert.False(t, lock.IsBackup)
// Backup should NOT have it (replication failed)
_, backup := primaryDlm.LockRing.GetPrimaryAndBackup(key)
backupDlm := cluster.get(backup)
_, found = backupDlm.GetLock(key)
assert.False(t, found, "backup should not have the lock when replication fails")
// Primary should still be able to renew
newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
require.NoError(t, err)
assert.NotEmpty(t, newToken)
}
func TestDLM_UnlockReplicatesToBackup(t *testing.T) {
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
cluster := newTestCluster(hosts...)
key := "unlock-repl-lock"
renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
require.NoError(t, err)
_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
// Verify backup has the lock
_, found := cluster.get(backup).GetLock(key)
require.True(t, found, "backup should have the lock")
// Unlock on primary
primaryDlm := cluster.get(primaryHost)
movedTo, err := primaryDlm.Unlock(key, renewToken)
require.NoError(t, err)
assert.Empty(t, movedTo)
// Wait for async replication
time.Sleep(20 * time.Millisecond)
// Backup should also have removed the lock
_, found = cluster.get(backup).GetLock(key)
assert.False(t, found, "backup should remove lock after unlock replication")
}
func TestDLM_TopologyChange_LockSurvivesServerAddition(t *testing.T) {
// Start with 2 servers, acquire lock, add a 3rd server
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
cluster := newTestCluster(hosts...)
key := "topo-add-lock"
renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
require.NoError(t, err)
// Add a new server
cluster.addNode("filer3:8888")
time.Sleep(20 * time.Millisecond)
// The lock should still be accessible — either the same primary or on a new one
// Try to renew on the original primary first
newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
if newPrimary == primaryHost {
// Still on same primary
newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
require.NoError(t, err)
assert.NotEmpty(t, newToken)
}
// If primary changed, the lock may need transfer — that's handled by OnDlmChangeSnapshot
// which is tested at the server level
}
func TestDLM_ConsistentHashing_MinimalDisruption(t *testing.T) {
// Verify that removing a server only affects locks on that server
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
// Acquire 50 locks
type lockInfo struct {
key, token string
primary pb.ServerAddress
}
locks := make([]lockInfo, 50)
for i := range locks {
key := fmt.Sprintf("min-disrupt-%d", i)
token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
require.NoError(t, err)
locks[i] = lockInfo{key: key, token: token, primary: primary}
}
// Count locks per server before removal
countBefore := make(map[pb.ServerAddress]int)
for _, l := range locks {
countBefore[l.primary]++
}
t.Logf("Lock distribution before: %v", countBefore)
// Remove filer2
cluster.removeNode("filer2:8888")
// Count how many locks changed primary
changed := 0
for _, l := range locks {
// Check where the lock should be now
for _, dlm := range cluster.getNodes() {
newPrimary := dlm.LockRing.GetPrimary(l.key)
if newPrimary != l.primary {
changed++
}
break
}
}
// Only locks from filer2 should have changed
assert.Equal(t, countBefore["filer2:8888"], changed,
"only locks from removed server should change primary")
}
func TestDLM_NodeDropAndJoin_OwnershipDisruption(t *testing.T) {
// Scenario: 3 nodes, acquire locks, one drops and a NEW node joins quickly.
// The new node steals hash ranges from surviving nodes, not just from the
// departed node. This test measures the disruption.
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
// Acquire many locks
numLocks := 100
type lockInfo struct {
key, token string
primary pb.ServerAddress
}
locks := make([]lockInfo, numLocks)
for i := range locks {
key := fmt.Sprintf("churn-lock-%d", i)
token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
require.NoError(t, err)
locks[i] = lockInfo{key: key, token: token, primary: primary}
}
// Record primary for each lock before the change
beforePrimary := make(map[string]pb.ServerAddress)
for _, l := range locks {
beforePrimary[l.key] = l.primary
}
// Drop filer3 and immediately add filer4
cluster.removeNode("filer3:8888")
// Promote backups on remaining nodes (simulates OnDlmChangeSnapshot)
for _, dlm := range cluster.getNodes() {
for _, lock := range dlm.AllLocks() {
p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
if p == dlm.Host && lock.IsBackup {
dlm.PromoteLock(lock.Key)
}
}
// Re-replicate primary locks to new backups
for _, lock := range dlm.AllLocks() {
p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
if p == dlm.Host && !lock.IsBackup {
dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
}
}
}
time.Sleep(10 * time.Millisecond)
// Now add filer4 (new node, empty)
cluster.addNode("filer4:8888")
time.Sleep(10 * time.Millisecond)
// Simulate OnDlmChangeSnapshot on all nodes after filer4 joins:
// transfer locks that now belong to filer4
for host, dlm := range cluster.getNodes() {
for _, lock := range dlm.AllLocks() {
p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
if p != host && !lock.IsBackup {
// This lock should move to the new primary
target := cluster.get(p)
if target != nil {
target.InsertLock(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq)
dlm.DemoteLock(lock.Key)
}
}
}
}
time.Sleep(10 * time.Millisecond)
// Count disruptions: locks whose primary changed to a node other than filer3's successor
disruptedFromSurvivors := 0
disruptedFromDeparted := 0
movedToFiler4 := 0
for _, l := range locks {
// What's the new primary?
var newPrimary pb.ServerAddress
for _, dlm := range cluster.getNodes() {
newPrimary = dlm.LockRing.GetPrimary(l.key)
break
}
oldPrimary := beforePrimary[l.key]
if newPrimary != oldPrimary {
if oldPrimary == "filer3:8888" {
disruptedFromDeparted++
} else {
disruptedFromSurvivors++
}
}
if newPrimary == "filer4:8888" {
movedToFiler4++
}
}
t.Logf("Locks disrupted from departed filer3: %d / %d", disruptedFromDeparted, numLocks)
t.Logf("Locks disrupted from surviving filer1/filer2: %d / %d", disruptedFromSurvivors, numLocks)
t.Logf("Locks now on new filer4: %d / %d", movedToFiler4, numLocks)
// The key concern: filer4 joining disrupts locks on surviving nodes
// With consistent hashing, new node steals ~1/N of each surviving node's keys
// Verify that the transfer logic above moved those locks to filer4
for _, l := range locks {
var newPrimary pb.ServerAddress
for _, dlm := range cluster.getNodes() {
newPrimary = dlm.LockRing.GetPrimary(l.key)
break
}
target := cluster.get(newPrimary)
require.NotNil(t, target, "primary %s should exist", newPrimary)
lock, found := target.GetLock(l.key)
if !found {
// Lock may have only a backup copy if transfer happened but
// the lock was on the departed node and wasn't re-replicated.
// Check all nodes for any copy.
anyFound := false
for _, dlm := range cluster.getNodes() {
if _, f := dlm.GetLock(l.key); f {
anyFound = true
break
}
}
if !anyFound {
t.Errorf("lock %s completely lost (primary should be %s)", l.key, newPrimary)
}
continue
}
assert.False(t, lock.IsBackup, "lock %s on primary %s should not be a backup", l.key, newPrimary)
}
}
func TestDLM_RenewalDuringTransferWindow(t *testing.T) {
// When a new node joins and steals a key range from a surviving node,
// there's a window between ring update and lock transfer. During this
// window, a client renewal should still succeed on the old primary
// (because it still holds the lock locally).
hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
cluster := newTestCluster(hosts...)
// Find a key that will move primary when filer4 is added.
// Try candidate keys until we find one whose primary changes.
var key, renewToken string
var primaryHost pb.ServerAddress
for i := 0; i < 1000; i++ {
candidate := fmt.Sprintf("transfer-window-lock-%d", i)
token, _, primary, err := cluster.acquireLock(candidate, "owner1", 30*time.Second)
require.NoError(t, err)
// Check if adding filer4 would move this key's primary
tmpRing := NewHashRing(DefaultVnodeCount)
tmpRing.SetServers([]pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888", "filer4:8888"})
newPrimary := tmpRing.GetPrimary(candidate)
if newPrimary != primary {
key = candidate
renewToken = token
primaryHost = primary
break
}
}
require.NotEmpty(t, key, "should find a key that moves primary when filer4 joins")
// Add filer4 — this changes the primary for our key per the ring
cluster.addNode("filer4:8888")
time.Sleep(10 * time.Millisecond)
newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
require.NotEqual(t, primaryHost, newPrimary, "key should have moved to a different primary")
// Renewal on the OLD primary should still succeed because it holds the lock locally
newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
require.NoError(t, err, "renewal on old primary should succeed during transfer window")
assert.NotEmpty(t, newToken, "should get a new token from old primary")
t.Logf("Key %s: primary changed from %s to %s, but renewal on old primary succeeded", key, primaryHost, newPrimary)
}
func TestDLM_StaleReplicationRejected(t *testing.T) {
// Verify that a stale replication (lower seq) does not overwrite a newer one
lm := NewLockManager()
// Insert backup with seq=3
lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 1, 3)
lock, found := lm.GetLock("key1")
require.True(t, found)
assert.Equal(t, "token-new", lock.Token)
assert.Equal(t, int64(3), lock.Seq)
// Try to overwrite with stale seq=2 — should be rejected
lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 1, 2)
lock, found = lm.GetLock("key1")
require.True(t, found)
assert.Equal(t, "token-new", lock.Token, "stale replication should be rejected")
assert.Equal(t, int64(3), lock.Seq)
// Update with higher seq=4 — should succeed
lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-newer", "owner1", 1, 4)
lock, found = lm.GetLock("key1")
require.True(t, found)
assert.Equal(t, "token-newer", lock.Token, "newer replication should be accepted")
assert.Equal(t, int64(4), lock.Seq)
// Stale unlock (seq=2) should not delete the lock
removed := lm.RemoveBackupLockIfSeq("key1", 1, 2)
assert.False(t, removed, "stale unlock should be rejected")
_, found = lm.GetLock("key1")
assert.True(t, found, "lock should still exist after stale unlock")
// Valid unlock (seq=5) should delete
removed = lm.RemoveBackupLockIfSeq("key1", 1, 5)
assert.True(t, removed, "valid unlock should be accepted")
_, found = lm.GetLock("key1")
assert.False(t, found, "lock should be removed after valid unlock")
}
-198
View File
@@ -1,198 +0,0 @@
package lock_manager
import (
"hash/crc32"
"sort"
"sync"
"github.com/seaweedfs/seaweedfs/weed/pb"
)
const DefaultVnodeCount = 50
// HashRing implements consistent hashing with virtual nodes.
// When a server is removed, only the keys that hashed to that server
// are remapped (to the next server on the ring), leaving all other
// key-to-server mappings stable.
//
// UPGRADE NOTE: This replaces the previous modulo-based hashing
// (hash % len(servers)). The two schemes compute different primaries
// for the same key, so all filer nodes in the cluster must be upgraded
// together (or via a rolling restart that completes within the lock TTL
// window of ~7 seconds) to avoid routing disagreements.
type HashRing struct {
mu sync.RWMutex
vnodeCount int
sortedHashes []uint32 // sorted ring positions
vnodeToServer map[uint32]pb.ServerAddress // ring position → server
servers map[pb.ServerAddress]struct{} // set of all servers
}
func NewHashRing(vnodeCount int) *HashRing {
if vnodeCount <= 0 {
vnodeCount = DefaultVnodeCount
}
return &HashRing{
vnodeCount: vnodeCount,
vnodeToServer: make(map[uint32]pb.ServerAddress),
servers: make(map[pb.ServerAddress]struct{}),
}
}
// AddServer adds a server with virtual nodes to the ring.
func (hr *HashRing) AddServer(server pb.ServerAddress) {
hr.mu.Lock()
defer hr.mu.Unlock()
if _, exists := hr.servers[server]; exists {
return
}
hr.servers[server] = struct{}{}
hr.rebuildRing()
}
// RemoveServer removes a server and its virtual nodes from the ring.
func (hr *HashRing) RemoveServer(server pb.ServerAddress) {
hr.mu.Lock()
defer hr.mu.Unlock()
if _, exists := hr.servers[server]; !exists {
return
}
delete(hr.servers, server)
hr.rebuildRing()
}
// SetServers replaces the entire server set.
func (hr *HashRing) SetServers(servers []pb.ServerAddress) {
hr.mu.Lock()
defer hr.mu.Unlock()
hr.servers = make(map[pb.ServerAddress]struct{}, len(servers))
for _, s := range servers {
hr.servers[s] = struct{}{}
}
hr.rebuildRing()
}
// GetPrimaryAndBackup returns the primary server for a key and its backup
// (the next distinct server clockwise on the ring).
// If there is only one server, backup is empty.
func (hr *HashRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
hr.mu.RLock()
defer hr.mu.RUnlock()
if len(hr.sortedHashes) == 0 {
return "", ""
}
hash := hashKey(key)
idx := hr.search(hash)
primary = hr.vnodeToServer[hr.sortedHashes[idx]]
// Walk clockwise to find a different server for backup
ringLen := len(hr.sortedHashes)
for i := 1; i < ringLen; i++ {
candidate := hr.vnodeToServer[hr.sortedHashes[(idx+i)%ringLen]]
if candidate != primary {
backup = candidate
return
}
}
// Only one server — no backup
return primary, ""
}
// GetPrimary returns just the primary server for a key.
func (hr *HashRing) GetPrimary(key string) pb.ServerAddress {
hr.mu.RLock()
defer hr.mu.RUnlock()
if len(hr.sortedHashes) == 0 {
return ""
}
hash := hashKey(key)
idx := hr.search(hash)
return hr.vnodeToServer[hr.sortedHashes[idx]]
}
// GetServers returns a sorted copy of all servers in the ring.
func (hr *HashRing) GetServers() []pb.ServerAddress {
hr.mu.RLock()
defer hr.mu.RUnlock()
servers := make([]pb.ServerAddress, 0, len(hr.servers))
for s := range hr.servers {
servers = append(servers, s)
}
sort.Slice(servers, func(i, j int) bool {
return servers[i] < servers[j]
})
return servers
}
// ServerCount returns the number of servers in the ring.
func (hr *HashRing) ServerCount() int {
hr.mu.RLock()
defer hr.mu.RUnlock()
return len(hr.servers)
}
// rebuildRing rebuilds the sorted hash ring from the current server set.
// Caller must hold hr.mu write lock.
func (hr *HashRing) rebuildRing() {
hr.vnodeToServer = make(map[uint32]pb.ServerAddress, len(hr.servers)*hr.vnodeCount)
hr.sortedHashes = make([]uint32, 0, len(hr.servers)*hr.vnodeCount)
for server := range hr.servers {
for i := 0; i < hr.vnodeCount; i++ {
vnodeKey := vnodeKeyFor(server, i)
hash := hashKey(vnodeKey)
hr.vnodeToServer[hash] = server
hr.sortedHashes = append(hr.sortedHashes, hash)
}
}
sort.Slice(hr.sortedHashes, func(i, j int) bool {
return hr.sortedHashes[i] < hr.sortedHashes[j]
})
}
// search finds the first ring position >= hash.
func (hr *HashRing) search(hash uint32) int {
idx := sort.Search(len(hr.sortedHashes), func(i int) bool {
return hr.sortedHashes[i] >= hash
})
if idx >= len(hr.sortedHashes) {
idx = 0 // wrap around
}
return idx
}
func hashKey(key string) uint32 {
return crc32.ChecksumIEEE([]byte(key))
}
func vnodeKeyFor(server pb.ServerAddress, index int) string {
// Use a format that distributes well across the ring
buf := make([]byte, 0, len(server)+10)
buf = append(buf, []byte(server)...)
buf = append(buf, '#')
buf = appendInt(buf, index)
return string(buf)
}
func appendInt(buf []byte, n int) []byte {
if n == 0 {
return append(buf, '0')
}
// Simple int-to-string without importing strconv
digits := [20]byte{}
pos := len(digits)
for n > 0 {
pos--
digits[pos] = byte('0' + n%10)
n /= 10
}
return append(buf, digits[pos:]...)
}
-173
View File
@@ -1,173 +0,0 @@
package lock_manager
import (
"fmt"
"math"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/stretchr/testify/assert"
)
func TestHashRing_BasicOperations(t *testing.T) {
hr := NewHashRing(50)
// Empty ring
p, b := hr.GetPrimaryAndBackup("key1")
assert.Equal(t, pb.ServerAddress(""), p)
assert.Equal(t, pb.ServerAddress(""), b)
// Single server — no backup
hr.AddServer("filer1:8888")
p, b = hr.GetPrimaryAndBackup("key1")
assert.Equal(t, pb.ServerAddress("filer1:8888"), p)
assert.Equal(t, pb.ServerAddress(""), b)
// Two servers — backup is the other server
hr.AddServer("filer2:8888")
p, b = hr.GetPrimaryAndBackup("key1")
assert.NotEqual(t, p, b)
assert.NotEmpty(t, b)
// Three servers
hr.AddServer("filer3:8888")
p, b = hr.GetPrimaryAndBackup("key1")
assert.NotEqual(t, p, b)
assert.NotEmpty(t, b)
// Remove server
hr.RemoveServer("filer2:8888")
assert.Equal(t, 2, hr.ServerCount())
}
func TestHashRing_DuplicateAddRemove(t *testing.T) {
hr := NewHashRing(50)
hr.AddServer("filer1:8888")
hr.AddServer("filer1:8888") // duplicate
assert.Equal(t, 1, hr.ServerCount())
hr.RemoveServer("filer1:8888")
assert.Equal(t, 0, hr.ServerCount())
hr.RemoveServer("filer1:8888") // remove non-existent
assert.Equal(t, 0, hr.ServerCount())
}
func TestHashRing_SetServers(t *testing.T) {
hr := NewHashRing(50)
hr.SetServers([]pb.ServerAddress{"a:1", "b:2", "c:3"})
assert.Equal(t, 3, hr.ServerCount())
servers := hr.GetServers()
assert.Equal(t, 3, len(servers))
// SetServers replaces
hr.SetServers([]pb.ServerAddress{"x:1", "y:2"})
assert.Equal(t, 2, hr.ServerCount())
}
func TestHashRing_ConsistencyOnRemoval(t *testing.T) {
// The key property of consistent hashing: when a server is removed,
// only keys that mapped to the removed server change.
hr := NewHashRing(50)
servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
hr.SetServers(servers)
numKeys := 1000
// Record where each key maps before removal
before := make(map[string]pb.ServerAddress, numKeys)
for i := 0; i < numKeys; i++ {
key := fmt.Sprintf("lock-key-%d", i)
before[key] = hr.GetPrimary(key)
}
// Remove filer2
hr.RemoveServer("filer2:8888")
moved := 0
for i := 0; i < numKeys; i++ {
key := fmt.Sprintf("lock-key-%d", i)
after := hr.GetPrimary(key)
if before[key] != after {
// Only keys from filer2 should move
assert.Equal(t, pb.ServerAddress("filer2:8888"), before[key],
"key %s moved from %s to %s, but it wasn't on the removed server", key, before[key], after)
moved++
}
}
// Roughly 1/3 of keys should move (those that were on filer2)
t.Logf("Keys that moved: %d / %d", moved, numKeys)
assert.Greater(t, moved, 0, "some keys should have moved")
assert.Less(t, moved, numKeys, "not all keys should move")
}
func TestHashRing_BackupIsSuccessor(t *testing.T) {
// After removing primary, the backup should become the new primary
hr := NewHashRing(50)
servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
hr.SetServers(servers)
// For each key, verify that removing the primary makes the backup the new primary
promoted := 0
total := 500
for i := 0; i < total; i++ {
key := fmt.Sprintf("test-lock-%d", i)
primary, backup := hr.GetPrimaryAndBackup(key)
assert.NotEqual(t, primary, backup)
// Temporarily remove primary
hr.RemoveServer(primary)
newPrimary := hr.GetPrimary(key)
if newPrimary == backup {
promoted++
}
// Restore
hr.AddServer(primary)
}
// The backup should become new primary for all keys
assert.Equal(t, total, promoted,
"backup should become new primary for all keys when primary is removed")
}
func TestHashRing_Distribution(t *testing.T) {
hr := NewHashRing(50)
servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
hr.SetServers(servers)
counts := make(map[pb.ServerAddress]int)
numKeys := 3000
for i := 0; i < numKeys; i++ {
key := fmt.Sprintf("dist-key-%d", i)
p := hr.GetPrimary(key)
counts[p]++
}
expected := float64(numKeys) / float64(len(servers))
for server, count := range counts {
deviation := math.Abs(float64(count)-expected) / expected
t.Logf("Server %s: %d keys (%.1f%% deviation)", server, count, deviation*100)
// Allow up to 40% deviation with 50 vnodes and 3 servers
assert.Less(t, deviation, 0.40,
"server %s has too many or too few keys: %d (expected ~%d)", server, count, int(expected))
}
}
func TestHashRing_GetPrimary(t *testing.T) {
hr := NewHashRing(50)
// Empty ring
assert.Equal(t, pb.ServerAddress(""), hr.GetPrimary("key"))
hr.SetServers([]pb.ServerAddress{"a:1", "b:2"})
// Deterministic: same key always maps to same server
p1 := hr.GetPrimary("mykey")
p2 := hr.GetPrimary("mykey")
assert.Equal(t, p1, p2)
// GetPrimary matches the primary from GetPrimaryAndBackup
primary, _ := hr.GetPrimaryAndBackup("mykey")
assert.Equal(t, primary, hr.GetPrimary("mykey"))
}
+15 -185
View File
@@ -3,7 +3,6 @@ package lock_manager
import (
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
@@ -18,18 +17,14 @@ var LockNotFound = fmt.Errorf("lock not found")
// LockManager local lock manager, used by distributed lock manager
type LockManager struct {
locks map[string]*Lock
accessLock sync.RWMutex
nextGeneration atomic.Int64
locks map[string]*Lock
accessLock sync.RWMutex
}
type Lock struct {
Token string
ExpiredAtNs int64
Key string // only used for moving locks
Owner string
IsBackup bool // true if this node holds the lock as a backup
Generation int64 // monotonic fencing token, increments on fresh acquisition
Seq int64 // per-lock sequence number, increments on every mutation (acquire/renew/unlock)
}
func NewLockManager() *LockManager {
@@ -40,38 +35,7 @@ func NewLockManager() *LockManager {
return t
}
func (lm *LockManager) NextGeneration() int64 {
return lm.nextGeneration.Add(1)
}
func compareMutationVersion(generation, seq int64, existingGeneration, existingSeq int64) int {
switch {
case generation < existingGeneration:
return -1
case generation > existingGeneration:
return 1
case seq < existingSeq:
return -1
case seq > existingSeq:
return 1
default:
return 0
}
}
func (lm *LockManager) advanceGenerationFloor(generation int64) {
for {
current := lm.nextGeneration.Load()
if generation < current {
return
}
if lm.nextGeneration.CompareAndSwap(current, generation) {
return
}
}
}
func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, generation int64, seq int64, err error) {
func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, err error) {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
@@ -87,10 +51,8 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
} else {
// new lock
renewToken = uuid.New().String()
generation = lm.NextGeneration()
seq = 1
glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
glog.V(4).Infof("key %s new token %v owner %v", path, renewToken, owner)
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
return
}
}
@@ -99,10 +61,8 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
if oldValue.Token == token {
// token matches, renew the lock
renewToken = uuid.New().String()
generation = oldValue.Generation // keep same generation on renewal
seq = oldValue.Seq + 1
glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
return
} else {
if token == "" {
@@ -119,11 +79,9 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
glog.V(4).Infof("key %s no lock owner %v", path, owner)
if token == "" {
// new lock
glog.V(4).Infof("key %s new token %v owner %v", path, token, owner)
renewToken = uuid.New().String()
generation = lm.NextGeneration()
seq = 1
glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
return
} else {
glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
@@ -133,7 +91,7 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner
}
}
func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, generation int64, seq int64, err error) {
func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err error) {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
@@ -142,16 +100,12 @@ func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, gener
if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
// lock is expired, delete it
isUnlocked = true
generation = oldValue.Generation
seq = oldValue.Seq + 1
glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
delete(lm.locks, path)
return
}
if oldValue.Token == token {
isUnlocked = true
generation = oldValue.Generation
seq = oldValue.Seq + 1
glog.V(4).Infof("key %s unlocked with %v", path, token)
delete(lm.locks, path)
return
@@ -186,10 +140,10 @@ func (lm *LockManager) CleanUp() {
}
// SelectLocks takes out locks by key
// if selectFn returns true, the lock will be removed and returned
// if keyFn return true, the lock will be taken out
func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
lm.accessLock.RLock()
defer lm.accessLock.RUnlock()
now := time.Now().UnixNano()
@@ -209,90 +163,12 @@ func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Loc
return
}
// InsertLock inserts a lock from a transfer if it is newer than the current state.
func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) bool {
// InsertLock inserts a lock unconditionally
func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string) {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
if existing, found := lm.locks[path]; found {
if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
glog.V(4).Infof("lock %s: rejecting stale transfer gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
return false
}
}
lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
lm.advanceGenerationFloor(generation)
return true
}
// InsertBackupLock inserts or updates a lock as a backup copy.
// It rejects stale mutations by comparing (generation, seq). If a current primary
// already exists on this node, newer replicated state refreshes that primary copy
// without demoting it back to a backup.
func (lm *LockManager) InsertBackupLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
if existing, found := lm.locks[path]; found {
if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
glog.V(4).Infof("backup lock %s: rejecting stale gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
return
}
lm.locks[path] = &Lock{
Token: token,
ExpiredAtNs: expiredAtNs,
Owner: owner,
IsBackup: existing.IsBackup,
Generation: generation,
Seq: seq,
}
lm.advanceGenerationFloor(generation)
return
}
lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, IsBackup: true, Generation: generation, Seq: seq}
lm.advanceGenerationFloor(generation)
}
// RemoveLock removes a lock by key
func (lm *LockManager) RemoveLock(path string) {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
delete(lm.locks, path)
}
// RemoveBackupLockIfSeq removes the local copy only if the incoming mutation is
// not older than the current (generation, seq). This prevents a late unlock from
// deleting a newer reacquired lock whose seq has reset.
func (lm *LockManager) RemoveBackupLockIfSeq(path string, generation int64, seq int64) bool {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
if existing, found := lm.locks[path]; found {
if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) >= 0 {
delete(lm.locks, path)
return true
}
glog.V(4).Infof("backup lock %s: rejecting stale unlock gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
return false
}
return true
}
// GetLock returns a copy of the lock for a key, if it exists and is not expired
func (lm *LockManager) GetLock(key string) (*Lock, bool) {
lm.accessLock.RLock()
defer lm.accessLock.RUnlock()
lock, found := lm.locks[key]
if !found {
return nil, false
}
if time.Now().UnixNano() > lock.ExpiredAtNs {
return nil, false
}
// Return a copy
cp := *lock
return &cp, true
lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner}
}
func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
@@ -300,54 +176,8 @@ func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
defer lm.accessLock.RUnlock()
if lock, found := lm.locks[key]; found {
if time.Now().UnixNano() > lock.ExpiredAtNs {
err = LockNotFound
return
}
return lock.Owner, nil
}
err = LockNotFound
return
}
// AllLocks returns a copy of all non-expired locks
func (lm *LockManager) AllLocks() []*Lock {
lm.accessLock.RLock()
defer lm.accessLock.RUnlock()
now := time.Now().UnixNano()
var result []*Lock
for key, lock := range lm.locks {
if now > lock.ExpiredAtNs {
continue
}
cp := *lock
cp.Key = key
result = append(result, &cp)
}
return result
}
// PromoteLock changes a backup lock to a primary lock
func (lm *LockManager) PromoteLock(key string) bool {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
if lock, found := lm.locks[key]; found && lock.IsBackup {
lock.IsBackup = false
return true
}
return false
}
// DemoteLock changes a primary lock to a backup lock
func (lm *LockManager) DemoteLock(key string) bool {
lm.accessLock.Lock()
defer lm.accessLock.Unlock()
if lock, found := lm.locks[key]; found && !lock.IsBackup {
lock.IsBackup = true
return true
}
return false
}
@@ -1,86 +0,0 @@
package lock_manager
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestLockManager_GetLockOwnerIgnoresExpiredLock(t *testing.T) {
lm := NewLockManager()
lm.InsertLock("key1", time.Now().Add(-time.Second).UnixNano(), "token1", "owner1", 7, 3)
owner, err := lm.GetLockOwner("key1")
assert.Empty(t, owner)
assert.ErrorIs(t, err, LockNotFound)
}
func TestLockManager_InsertLockRejectsStaleTransfer(t *testing.T) {
lm := NewLockManager()
lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 4)
lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 3)
lock, found := lm.GetLock("key1")
assert.True(t, found)
assert.Equal(t, "token-new", lock.Token)
assert.Equal(t, int64(8), lock.Generation)
assert.Equal(t, int64(4), lock.Seq)
}
func TestLockManager_InsertLockAdvancesGenerationCounter(t *testing.T) {
lm := NewLockManager()
lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token1", "owner1", 12, 1)
_, renewToken, generation, _, err := lm.Lock("key2", time.Now().Add(30*time.Second).UnixNano(), "", "owner2")
assert.NoError(t, err)
assert.NotEmpty(t, renewToken)
assert.Greater(t, generation, int64(12))
}
func TestLockManager_InsertBackupLockRejectsOlderGeneration(t *testing.T) {
lm := NewLockManager()
lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 9)
lock, found := lm.GetLock("key1")
assert.True(t, found)
assert.Equal(t, "token-new", lock.Token)
assert.Equal(t, int64(8), lock.Generation)
assert.Equal(t, int64(1), lock.Seq)
}
func TestLockManager_InsertBackupLockKeepsPrimaryRole(t *testing.T) {
lm := NewLockManager()
ok := lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 8, 1)
assert.True(t, ok)
lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 2)
lock, found := lm.GetLock("key1")
assert.True(t, found)
assert.False(t, lock.IsBackup)
assert.Equal(t, "token-new", lock.Token)
assert.Equal(t, int64(8), lock.Generation)
assert.Equal(t, int64(2), lock.Seq)
}
func TestLockManager_RemoveBackupLockRejectsOlderGeneration(t *testing.T) {
lm := NewLockManager()
lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
removed := lm.RemoveBackupLockIfSeq("key1", 7, 9)
assert.False(t, removed)
lock, found := lm.GetLock("key1")
assert.True(t, found)
assert.Equal(t, "token-new", lock.Token)
assert.Equal(t, int64(8), lock.Generation)
assert.Equal(t, int64(1), lock.Seq)
}
+81 -37
View File
@@ -7,6 +7,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type LockRingSnapshot struct {
@@ -17,18 +18,18 @@ type LockRingSnapshot struct {
type LockRing struct {
sync.RWMutex
snapshots []*LockRingSnapshot
candidateServers map[pb.ServerAddress]struct{}
lastUpdateTime time.Time
lastCompactTime time.Time
snapshotInterval time.Duration
onTakeSnapshot func(snapshot []pb.ServerAddress)
cleanupWg sync.WaitGroup
Ring *HashRing // consistent hash ring
version int64 // monotonic version from master, rejects stale updates
}
func NewLockRing(snapshotInterval time.Duration) *LockRing {
return &LockRing{
snapshotInterval: snapshotInterval,
Ring: NewHashRing(DefaultVnodeCount),
candidateServers: make(map[pb.ServerAddress]struct{}),
}
}
@@ -38,26 +39,52 @@ func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.Ser
r.onTakeSnapshot = onTakeSnapshot
}
// SetSnapshot replaces the ring with a new server list from the master.
// The version must be >= the current version, otherwise the update is rejected
// (protects against reordered messages). Version 0 is always accepted (bootstrap).
func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
// AddServer adds a server to the ring
// if the previous snapshot passed the snapshot interval, create a new snapshot
func (r *LockRing) AddServer(server pb.ServerAddress) {
glog.V(0).Infof("add server %v", server)
r.Lock()
if _, found := r.candidateServers[server]; found {
glog.V(0).Infof("add server: already exists %v", server)
r.Unlock()
return
}
r.lastUpdateTime = time.Now()
r.candidateServers[server] = struct{}{}
r.Unlock()
r.takeSnapshotWithDelayedCompaction()
}
func (r *LockRing) RemoveServer(server pb.ServerAddress) {
glog.V(0).Infof("remove server %v", server)
r.Lock()
if _, found := r.candidateServers[server]; !found {
r.Unlock()
return
}
r.lastUpdateTime = time.Now()
delete(r.candidateServers, server)
r.Unlock()
r.takeSnapshotWithDelayedCompaction()
}
func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
sort.Slice(servers, func(i, j int) bool {
return servers[i] < servers[j]
})
r.Lock()
if version > 0 && version < r.version {
glog.V(0).Infof("LockRing: rejecting stale update v%d (current v%d)", version, r.version)
r.Unlock()
return false
r.lastUpdateTime = time.Now()
// init candidateServers
for _, server := range servers {
r.candidateServers[server] = struct{}{}
}
r.version = version
// Update the ring while holding the lock so version and ring state
// are always consistent — prevents a concurrent SetSnapshot from
// seeing the new version but applying its servers to the old ring.
r.Ring.SetServers(servers)
r.Unlock()
r.addOneSnapshot(servers)
@@ -68,14 +95,23 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
<-time.After(r.snapshotInterval)
r.compactSnapshots()
}()
return true
}
// Version returns the current ring version.
func (r *LockRing) Version() int64 {
r.RLock()
defer r.RUnlock()
return r.version
func (r *LockRing) takeSnapshotWithDelayedCompaction() {
r.doTakeSnapshot()
r.cleanupWg.Add(1)
go func() {
defer r.cleanupWg.Done()
<-time.After(r.snapshotInterval)
r.compactSnapshots()
}()
}
func (r *LockRing) doTakeSnapshot() {
servers := r.getSortedServers()
r.addOneSnapshot(servers)
}
func (r *LockRing) addOneSnapshot(servers []pb.ServerAddress) {
@@ -102,19 +138,35 @@ func (r *LockRing) compactSnapshots() {
r.Lock()
defer r.Unlock()
// Always attempt compaction when called, regardless of lastCompactTime
// This ensures proper cleanup even with multiple concurrent compaction requests
ts := time.Now()
// remove old snapshots
recentSnapshotIndex := 1
for ; recentSnapshotIndex < len(r.snapshots); recentSnapshotIndex++ {
if ts.Sub(r.snapshots[recentSnapshotIndex].ts) > r.snapshotInterval {
break
}
}
// keep the one that has been running for a while
if recentSnapshotIndex+1 <= len(r.snapshots) {
r.snapshots = r.snapshots[:recentSnapshotIndex+1]
}
r.lastCompactTime = ts
}
func (r *LockRing) getSortedServers() []pb.ServerAddress {
sortedServers := make([]pb.ServerAddress, 0, len(r.candidateServers))
for server := range r.candidateServers {
sortedServers = append(sortedServers, server)
}
sort.Slice(sortedServers, func(i, j int) bool {
return sortedServers[i] < sortedServers[j]
})
return sortedServers
}
func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
r.RLock()
defer r.RUnlock()
@@ -126,6 +178,7 @@ func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
}
// WaitForCleanup waits for all pending cleanup operations to complete
// This is useful for testing to ensure deterministic behavior
func (r *LockRing) WaitForCleanup() {
r.cleanupWg.Wait()
}
@@ -137,23 +190,14 @@ func (r *LockRing) GetSnapshotCount() int {
return len(r.snapshots)
}
// GetPrimaryAndBackup returns the primary and backup servers for a key
// using the consistent hash ring.
func (r *LockRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
return r.Ring.GetPrimaryAndBackup(key)
}
// GetPrimary returns the primary server for a key using the consistent hash ring.
func (r *LockRing) GetPrimary(key string) pb.ServerAddress {
return r.Ring.GetPrimary(key)
}
// hashKeyToServer uses a temporary consistent hash ring for the given server list.
func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
if len(servers) == 0 {
return ""
}
ring := NewHashRing(DefaultVnodeCount)
ring.SetServers(servers)
return ring.GetPrimary(key)
x := util.HashStringToLong(key)
if x < 0 {
x = -x
}
x = x % int64(len(servers))
return servers[x]
}
+46 -53
View File
@@ -8,91 +8,84 @@ import (
"github.com/stretchr/testify/assert"
)
func TestLockRing_SetSnapshot(t *testing.T) {
func TestAddServer(t *testing.T) {
r := NewLockRing(100 * time.Millisecond)
// Set 5 servers
r.SetSnapshot([]pb.ServerAddress{
"localhost:8080", "localhost:8081", "localhost:8082",
"localhost:8083", "localhost:8084",
}, 1)
// Add servers
r.AddServer("localhost:8080")
r.AddServer("localhost:8081")
r.AddServer("localhost:8082")
r.AddServer("localhost:8083")
r.AddServer("localhost:8084")
// Verify all servers are present
servers := r.GetSnapshot()
assert.Equal(t, 5, len(servers))
assert.Contains(t, servers, pb.ServerAddress("localhost:8080"))
assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
assert.Contains(t, servers, pb.ServerAddress("localhost:8082"))
assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
assert.Contains(t, servers, pb.ServerAddress("localhost:8084"))
// Replace with 2 servers
r.SetSnapshot([]pb.ServerAddress{"localhost:8081", "localhost:8083"}, 2)
// Remove servers
r.RemoveServer("localhost:8084")
r.RemoveServer("localhost:8082")
r.RemoveServer("localhost:8080")
// Wait for all cleanup operations to complete
r.WaitForCleanup()
// Verify only 2 servers remain (localhost:8081 and localhost:8083)
servers = r.GetSnapshot()
assert.Equal(t, 2, len(servers))
assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
// Verify compaction
// Verify cleanup has happened - wait for snapshot interval and check snapshots are compacted
time.Sleep(110 * time.Millisecond)
r.WaitForCleanup()
assert.LessOrEqual(t, r.GetSnapshotCount(), 2)
// Verify snapshot history is cleaned up properly (should have at most 2 snapshots after compaction)
snapshotCount := r.GetSnapshotCount()
assert.LessOrEqual(t, snapshotCount, 2, "Snapshot history should be compacted")
}
func TestLockRing_SnapshotCompaction(t *testing.T) {
func TestLockRing(t *testing.T) {
r := NewLockRing(100 * time.Millisecond)
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"}, 1)
// Test initial snapshot
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"})
assert.Equal(t, 1, r.GetSnapshotCount())
servers := r.GetSnapshot()
assert.Equal(t, 2, len(servers))
assert.Contains(t, servers, pb.ServerAddress("localhost:8080"))
assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"}, 2)
// Add another server
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"})
assert.Equal(t, 2, r.GetSnapshotCount())
servers = r.GetSnapshot()
assert.Equal(t, 3, len(servers))
assert.Contains(t, servers, pb.ServerAddress("localhost:8082"))
// Wait for compaction
// Wait for cleanup interval and add another server
time.Sleep(110 * time.Millisecond)
r.WaitForCleanup()
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"}, 3)
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"})
assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
servers := r.GetSnapshot()
servers = r.GetSnapshot()
assert.Equal(t, 4, len(servers))
assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
// Wait for cleanup and verify compaction
time.Sleep(110 * time.Millisecond)
r.WaitForCleanup()
assert.LessOrEqual(t, r.GetSnapshotCount(), 2, "Snapshots should be compacted")
r.SetSnapshot([]pb.ServerAddress{
"localhost:8080", "localhost:8081", "localhost:8082",
"localhost:8083", "localhost:8084",
}, 4)
// Add final server
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083", "localhost:8084"})
servers = r.GetSnapshot()
assert.Equal(t, 5, len(servers))
}
func TestLockRing_VersionRejectsStale(t *testing.T) {
r := NewLockRing(100 * time.Millisecond)
// Apply version 3
ok := r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2", "c:3"}, 3)
assert.True(t, ok)
assert.Equal(t, int64(3), r.Version())
assert.Equal(t, 3, len(r.GetSnapshot()))
// Stale version 2 — should be rejected
ok = r.SetSnapshot([]pb.ServerAddress{"x:1"}, 2)
assert.False(t, ok)
assert.Equal(t, int64(3), r.Version())
assert.Equal(t, 3, len(r.GetSnapshot()), "stale update should not change the ring")
// Same version 3 — accepted (SetSnapshot accepts version >= current, state-changing)
ok = r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 3)
assert.True(t, ok)
assert.Equal(t, 2, len(r.GetSnapshot()))
// Newer version 5 — should be accepted
ok = r.SetSnapshot([]pb.ServerAddress{"d:1", "e:2", "f:3", "g:4"}, 5)
assert.True(t, ok)
assert.Equal(t, int64(5), r.Version())
assert.Equal(t, 4, len(r.GetSnapshot()))
// Version 0 always accepted (bootstrap)
ok = r.SetSnapshot([]pb.ServerAddress{"z:1"}, 0)
assert.True(t, ok)
assert.Equal(t, 1, len(r.GetSnapshot()))
assert.Contains(t, servers, pb.ServerAddress("localhost:8084"))
assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
}
-164
View File
@@ -1,164 +0,0 @@
package cluster
import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
const LockRingStabilizationInterval = 1 * time.Second
// LockRingManager tracks filer membership for the distributed lock ring.
// It batches rapid topology changes (e.g., node drop + join) with a
// stabilization timer, then broadcasts the complete member list atomically
// so filers receive a single consistent ring update instead of multiple
// intermediate states.
type LockRingManager struct {
mu sync.Mutex
members map[FilerGroupName]map[pb.ServerAddress]struct{}
version map[FilerGroupName]int64
lastBroadcast map[FilerGroupName]*master_pb.LockRingUpdate
pendingTimer map[FilerGroupName]*time.Timer
broadcastFn func(resp *master_pb.KeepConnectedResponse)
stabilizeDelay time.Duration
}
func NewLockRingManager(broadcastFn func(resp *master_pb.KeepConnectedResponse)) *LockRingManager {
return &LockRingManager{
members: make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
version: make(map[FilerGroupName]int64),
lastBroadcast: make(map[FilerGroupName]*master_pb.LockRingUpdate),
pendingTimer: make(map[FilerGroupName]*time.Timer),
broadcastFn: broadcastFn,
stabilizeDelay: LockRingStabilizationInterval,
}
}
// AddServer records a filer joining and schedules a batched broadcast.
func (lrm *LockRingManager) AddServer(filerGroup FilerGroupName, address pb.ServerAddress) {
lrm.mu.Lock()
defer lrm.mu.Unlock()
if _, ok := lrm.members[filerGroup]; !ok {
lrm.members[filerGroup] = make(map[pb.ServerAddress]struct{})
}
lrm.members[filerGroup][address] = struct{}{}
lrm.scheduleBroadcast(filerGroup)
}
// RemoveServer records a filer leaving and schedules a batched broadcast.
func (lrm *LockRingManager) RemoveServer(filerGroup FilerGroupName, address pb.ServerAddress) {
lrm.mu.Lock()
defer lrm.mu.Unlock()
if members, ok := lrm.members[filerGroup]; ok {
delete(members, address)
}
lrm.scheduleBroadcast(filerGroup)
}
// GetServers returns the current member list for a filer group.
func (lrm *LockRingManager) GetServers(filerGroup FilerGroupName) []string {
lrm.mu.Lock()
defer lrm.mu.Unlock()
members, ok := lrm.members[filerGroup]
if !ok {
return nil
}
servers := make([]string, 0, len(members))
for addr := range members {
servers = append(servers, string(addr))
}
return servers
}
// GetVersion returns the current version for a filer group.
func (lrm *LockRingManager) GetVersion(filerGroup FilerGroupName) int64 {
lrm.mu.Lock()
defer lrm.mu.Unlock()
return lrm.version[filerGroup]
}
// GetLastUpdate returns a copy of the most recently broadcast lock-ring snapshot
// for the filer group. It intentionally does not expose pending, unstabilized changes.
func (lrm *LockRingManager) GetLastUpdate(filerGroup FilerGroupName) *master_pb.LockRingUpdate {
lrm.mu.Lock()
defer lrm.mu.Unlock()
update, ok := lrm.lastBroadcast[filerGroup]
if !ok || update == nil {
return nil
}
cp := *update
cp.Servers = append([]string(nil), update.Servers...)
return &cp
}
// scheduleBroadcast resets the stabilization timer. If another change arrives
// before the timer fires, the timer resets, batching the changes.
// Caller must hold lrm.mu.
func (lrm *LockRingManager) scheduleBroadcast(filerGroup FilerGroupName) {
if timer, ok := lrm.pendingTimer[filerGroup]; ok {
if !timer.Stop() {
// Timer already fired, callback is running or queued.
// It will pick up the latest state from lrm.members, so
// just schedule a new one for any further changes.
}
}
lrm.pendingTimer[filerGroup] = time.AfterFunc(lrm.stabilizeDelay, func() {
lrm.doBroadcast(filerGroup)
})
}
func (lrm *LockRingManager) doBroadcast(filerGroup FilerGroupName) {
lrm.mu.Lock()
// Use wall-clock nanoseconds so the version survives master restarts
// without persistence — a restarted master produces a version greater
// than any pre-restart value (assuming clocks don't jump backward).
version := time.Now().UnixNano()
lrm.version[filerGroup] = version
servers := make([]string, 0)
if members, ok := lrm.members[filerGroup]; ok {
for addr := range members {
servers = append(servers, string(addr))
}
}
update := &master_pb.LockRingUpdate{
FilerGroup: string(filerGroup),
Servers: append([]string(nil), servers...),
Version: version,
}
lrm.lastBroadcast[filerGroup] = update
delete(lrm.pendingTimer, filerGroup)
lrm.mu.Unlock()
glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, version, servers)
if lrm.broadcastFn != nil {
lrm.broadcastFn(&master_pb.KeepConnectedResponse{
LockRingUpdate: update,
})
}
}
// FlushPending fires any pending timer immediately (for testing or shutdown).
func (lrm *LockRingManager) FlushPending(filerGroup FilerGroupName) {
lrm.mu.Lock()
if timer, ok := lrm.pendingTimer[filerGroup]; ok {
if timer.Stop() {
// Timer was pending — we stopped it, so we broadcast now
delete(lrm.pendingTimer, filerGroup)
lrm.mu.Unlock()
lrm.doBroadcast(filerGroup)
} else {
// Timer already fired, callback is running — let it finish
lrm.mu.Unlock()
}
} else {
lrm.mu.Unlock()
}
}
-229
View File
@@ -1,229 +0,0 @@
package cluster
import (
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLockRingManager_BatchesRapidChanges(t *testing.T) {
var mu sync.Mutex
var broadcasts []*master_pb.LockRingUpdate
lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
mu.Lock()
if resp.LockRingUpdate != nil {
broadcasts = append(broadcasts, resp.LockRingUpdate)
}
mu.Unlock()
})
lrm.stabilizeDelay = 100 * time.Millisecond
group := FilerGroupName("default")
// Add 3 servers in rapid succession
lrm.AddServer(group, "filer1:8888")
lrm.AddServer(group, "filer2:8888")
lrm.AddServer(group, "filer3:8888")
// No broadcast should have happened yet (timer hasn't fired)
mu.Lock()
assert.Equal(t, 0, len(broadcasts), "should not broadcast before stabilization delay")
mu.Unlock()
// Wait for stabilization
time.Sleep(200 * time.Millisecond)
mu.Lock()
require.Equal(t, 1, len(broadcasts), "should batch into a single broadcast")
assert.Equal(t, 3, len(broadcasts[0].Servers), "should include all 3 servers")
assert.Greater(t, broadcasts[0].Version, int64(0))
mu.Unlock()
}
func TestLockRingManager_DropAndJoinBatched(t *testing.T) {
var mu sync.Mutex
var broadcasts []*master_pb.LockRingUpdate
lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
mu.Lock()
if resp.LockRingUpdate != nil {
broadcasts = append(broadcasts, resp.LockRingUpdate)
}
mu.Unlock()
})
lrm.stabilizeDelay = 100 * time.Millisecond
group := FilerGroupName("default")
// Set up initial state
lrm.AddServer(group, "filer1:8888")
lrm.AddServer(group, "filer2:8888")
lrm.AddServer(group, "filer3:8888")
lrm.FlushPending(group)
mu.Lock()
broadcasts = nil // reset
mu.Unlock()
// Simulate drop + join in rapid succession
lrm.RemoveServer(group, "filer3:8888")
lrm.AddServer(group, "filer4:8888")
// Should not have broadcast yet
mu.Lock()
assert.Equal(t, 0, len(broadcasts))
mu.Unlock()
// Wait for stabilization
time.Sleep(200 * time.Millisecond)
mu.Lock()
require.Equal(t, 1, len(broadcasts), "drop+join should be batched into single broadcast")
servers := broadcasts[0].Servers
assert.Equal(t, 3, len(servers), "should have filer1, filer2, filer4")
// filer3 should be gone, filer4 should be present
serverSet := make(map[string]bool)
for _, s := range servers {
serverSet[s] = true
}
assert.False(t, serverSet["filer3:8888"], "filer3 should be removed")
assert.True(t, serverSet["filer4:8888"], "filer4 should be added")
mu.Unlock()
}
func TestLockRingManager_VersionIncrements(t *testing.T) {
var mu sync.Mutex
var broadcasts []*master_pb.LockRingUpdate
lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
mu.Lock()
if resp.LockRingUpdate != nil {
broadcasts = append(broadcasts, resp.LockRingUpdate)
}
mu.Unlock()
})
lrm.stabilizeDelay = 50 * time.Millisecond
group := FilerGroupName("default")
lrm.AddServer(group, "filer1:8888")
time.Sleep(100 * time.Millisecond)
lrm.AddServer(group, "filer2:8888")
time.Sleep(100 * time.Millisecond)
mu.Lock()
require.Equal(t, 2, len(broadcasts))
assert.Greater(t, broadcasts[0].Version, int64(0), "version should be positive")
assert.Greater(t, broadcasts[1].Version, broadcasts[0].Version, "versions should be monotonically increasing")
mu.Unlock()
}
func TestLockRingManager_FlushPending(t *testing.T) {
var mu sync.Mutex
var broadcasts []*master_pb.LockRingUpdate
lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
mu.Lock()
if resp.LockRingUpdate != nil {
broadcasts = append(broadcasts, resp.LockRingUpdate)
}
mu.Unlock()
})
lrm.stabilizeDelay = 10 * time.Second // long delay
group := FilerGroupName("default")
lrm.AddServer(group, "filer1:8888")
lrm.AddServer(group, "filer2:8888")
// Flush immediately
lrm.FlushPending(group)
mu.Lock()
require.Equal(t, 1, len(broadcasts))
assert.Equal(t, 2, len(broadcasts[0].Servers))
mu.Unlock()
}
func TestLockRingManager_MultipleGroups(t *testing.T) {
var mu sync.Mutex
broadcastsByGroup := make(map[string][]*master_pb.LockRingUpdate)
lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
mu.Lock()
if resp.LockRingUpdate != nil {
broadcastsByGroup[resp.LockRingUpdate.FilerGroup] = append(
broadcastsByGroup[resp.LockRingUpdate.FilerGroup], resp.LockRingUpdate)
}
mu.Unlock()
})
lrm.stabilizeDelay = 50 * time.Millisecond
lrm.AddServer("group1", "filer1:8888")
lrm.AddServer("group2", "filer2:8888")
time.Sleep(100 * time.Millisecond)
mu.Lock()
assert.Equal(t, 1, len(broadcastsByGroup["group1"]))
assert.Equal(t, 1, len(broadcastsByGroup["group2"]))
assert.Equal(t, []string{"filer1:8888"}, broadcastsByGroup["group1"][0].Servers)
assert.Equal(t, []string{"filer2:8888"}, broadcastsByGroup["group2"][0].Servers)
mu.Unlock()
}
func TestLockRingManager_GetServers(t *testing.T) {
lrm := NewLockRingManager(nil)
group := FilerGroupName("default")
lrm.AddServer(group, "filer1:8888")
lrm.AddServer(group, "filer2:8888")
servers := lrm.GetServers(group)
assert.Equal(t, 2, len(servers))
// Contains both
serverSet := make(map[string]bool)
for _, s := range servers {
serverSet[s] = true
}
assert.True(t, serverSet["filer1:8888"])
assert.True(t, serverSet["filer2:8888"])
// Remove one
lrm.RemoveServer(group, "filer1:8888")
servers = lrm.GetServers(group)
assert.Equal(t, 1, len(servers))
assert.Equal(t, "filer2:8888", servers[0])
}
func TestLockRingManager_NoBroadcastWithoutFn(t *testing.T) {
// No panic when broadcastFn is nil
lrm := NewLockRingManager(nil)
lrm.stabilizeDelay = 10 * time.Millisecond
lrm.AddServer("default", pb.ServerAddress("filer1:8888"))
time.Sleep(50 * time.Millisecond) // should not panic
}
func TestLockRingManager_GetLastUpdateReturnsBroadcastState(t *testing.T) {
lrm := NewLockRingManager(nil)
group := FilerGroupName("default")
lrm.AddServer(group, "filer1:8888")
lrm.AddServer(group, "filer2:8888")
lrm.FlushPending(group)
update := lrm.GetLastUpdate(group)
require.NotNil(t, update)
assert.Equal(t, "default", update.FilerGroup)
assert.ElementsMatch(t, []string{"filer1:8888", "filer2:8888"}, update.Servers)
assert.Greater(t, update.Version, int64(0))
}
-1
View File
@@ -421,7 +421,6 @@ func (fo *FilerOptions) startFiler() {
go grpcS.Serve(grpcLocalL)
}
go grpcS.Serve(grpcL)
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
if runtime.GOOS != "windows" {
localSocket := *fo.localSocket
+3 -7
View File
@@ -164,7 +164,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
message := resp.EventNotification
if metadataEventUpdatesDirectory(resp, filer.DirectoryEtcRemote) {
if message.NewEntry != nil {
// update
if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
newMappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
@@ -180,11 +180,8 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
}
option.remoteConfs[conf.Name] = conf
}
} else if metadataEventRemovesFromDirectory(resp, filer.DirectoryEtcRemote) {
} else if message.OldEntry != nil {
// deletion
if message.OldEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
option.mappings = &remote_pb.RemoteStorageMapping{}
}
if strings.HasSuffix(message.OldEntry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
conf := &remote_pb.RemoteConf{}
if err := proto.Unmarshal(message.OldEntry.Content, conf); err != nil {
@@ -199,8 +196,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
message := resp.EventNotification
sourceInEtcRemote, targetInEtcRemote := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
if sourceInEtcRemote || targetInEtcRemote {
if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
return handleEtcRemoteChanges(resp)
}
+25 -30
View File
@@ -92,38 +92,34 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
handleEtcRemoteChanges := func(resp *filer_pb.SubscribeMetadataResponse) error {
message := resp.EventNotification
if metadataEventUpdatesDirectory(resp, filer.DirectoryEtcRemote) {
if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
if readErr != nil {
return fmt.Errorf("unmarshal mappings: %w", readErr)
}
if remoteLoc, found := mappings.Mappings[mountedDir]; found {
if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
}
} else {
glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
os.Exit(0)
}
if message.NewEntry == nil {
return nil
}
if message.NewEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
mappings, readErr := filer.UnmarshalRemoteStorageMappings(message.NewEntry.Content)
if readErr != nil {
return fmt.Errorf("unmarshal mappings: %w", readErr)
}
if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
conf := &remote_pb.RemoteConf{}
if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
}
remoteStorage = conf
if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
client = newClient
} else {
return err
if remoteLoc, found := mappings.Mappings[mountedDir]; found {
if remoteStorageMountLocation.Bucket != remoteLoc.Bucket || remoteStorageMountLocation.Path != remoteLoc.Path {
glog.Fatalf("Unexpected mount changes %+v => %+v", remoteStorageMountLocation, remoteLoc)
}
} else {
glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
os.Exit(0)
}
}
if metadataEventRemovesFromDirectory(resp, filer.DirectoryEtcRemote) &&
message.OldEntry.Name == filer.REMOTE_STORAGE_MOUNT_FILE {
glog.V(0).Infof("unmounted %s exiting ...", mountedDir)
os.Exit(0)
if message.NewEntry.Name == remoteStorage.Name+filer.REMOTE_STORAGE_CONF_SUFFIX {
conf := &remote_pb.RemoteConf{}
if err := proto.Unmarshal(message.NewEntry.Content, conf); err != nil {
return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, message.NewEntry.Name, err)
}
remoteStorage = conf
if newClient, err := remote_storage.GetRemoteStorage(remoteStorage); err == nil {
client = newClient
} else {
return err
}
}
return nil
@@ -131,8 +127,7 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
eachEntryFunc := func(resp *filer_pb.SubscribeMetadataResponse) error {
message := resp.EventNotification
sourceInEtcRemote, targetInEtcRemote := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
if sourceInEtcRemote || targetInEtcRemote {
if strings.HasPrefix(resp.Directory, filer.DirectoryEtcRemote) {
return handleEtcRemoteChanges(resp)
}
+59 -102
View File
@@ -26,39 +26,37 @@ import (
)
type SyncOptions struct {
isActivePassive *bool
filerA *string
filerB *string
aPath *string
aExcludePaths *string
bPath *string
bExcludePaths *string
aReplication *string
bReplication *string
aCollection *string
bCollection *string
aTtlSec *int
bTtlSec *int
aDiskType *string
bDiskType *string
aDebug *bool
bDebug *bool
aFromTsMs *int64
bFromTsMs *int64
aProxyByFiler *bool
bProxyByFiler *bool
metricsHttpIp *string
metricsHttpPort *int
isActivePassive *bool
filerA *string
filerB *string
aPath *string
aExcludePaths *string
bPath *string
bExcludePaths *string
aReplication *string
bReplication *string
aCollection *string
bCollection *string
aTtlSec *int
bTtlSec *int
aDiskType *string
bDiskType *string
aDebug *bool
bDebug *bool
aFromTsMs *int64
bFromTsMs *int64
aProxyByFiler *bool
bProxyByFiler *bool
metricsHttpIp *string
metricsHttpPort *int
concurrency *int
chunkConcurrency *int
aDoDeleteFiles *bool
bDoDeleteFiles *bool
aSecurity *string
bSecurity *string
clientId int32
clientEpoch atomic.Int32
debug *bool
debugPort *int
bDoDeleteFiles *bool
clientId int32
clientEpoch atomic.Int32
debug *bool
debugPort *int
}
const (
@@ -115,8 +113,6 @@ func init() {
syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
syncOptions.aSecurity = cmdFilerSynchronize.Flag.String("a.security", "", "security.toml file for filer A when clusters use different certificates")
syncOptions.bSecurity = cmdFilerSynchronize.Flag.String("b.security", "", "security.toml file for filer B when clusters use different certificates")
syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
syncOptions.clientId = util.RandomInt32()
@@ -148,22 +144,6 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
util.LoadSecurityConfiguration()
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
// per-filer TLS when clusters use different certificates
grpcDialOptionA := grpcDialOption
grpcDialOptionB := grpcDialOption
if *syncOptions.aSecurity != "" {
var err error
if grpcDialOptionA, err = security.LoadClientTLSFromFile(*syncOptions.aSecurity, "grpc.client"); err != nil {
glog.Fatalf("load security config for filer A: %v", err)
}
}
if *syncOptions.bSecurity != "" {
var err error
if grpcDialOptionB, err = security.LoadClientTLSFromFile(*syncOptions.bSecurity, "grpc.client"); err != nil {
glog.Fatalf("load security config for filer B: %v", err)
}
}
grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)
filerA := pb.ServerAddress(*syncOptions.filerA)
@@ -173,13 +153,13 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
go statsCollect.StartMetricsServer(*syncOptions.metricsHttpIp, *syncOptions.metricsHttpPort)
// read a filer signature
aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOptionA, filerA)
aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
if aFilerErr != nil {
glog.Errorf("get filer 'a' signature %d error from %s to %s: %v", aFilerSignature, *syncOptions.filerA, *syncOptions.filerB, aFilerErr)
return true
}
// read b filer signature
bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOptionB, filerB)
bFilerSignature, bFilerErr := replication.ReadFilerSignature(grpcDialOption, filerB)
if bFilerErr != nil {
glog.Errorf("get filer 'b' signature %d error from %s to %s: %v", bFilerSignature, *syncOptions.filerA, *syncOptions.filerB, bFilerErr)
return true
@@ -209,9 +189,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
go func() {
// a->b
// set synchronization start timestamp to offset
initOffsetError := initOffsetFromTsMs(grpcDialOptionB, filerB, aFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
initOffsetError := initOffsetFromTsMs(grpcDialOption, filerB, aFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.aPath))
if initOffsetError != nil {
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerA, *syncOptions.filerB, initOffsetError)
os.Exit(2)
}
for {
@@ -219,12 +199,11 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
err := doSubscribeFilerMetaChanges(
syncOptions.clientId,
syncOptions.clientEpoch.Load(),
grpcDialOptionA,
grpcDialOption,
filerA,
*syncOptions.aPath,
util.StringSplit(*syncOptions.aExcludePaths, ","),
*syncOptions.aProxyByFiler,
grpcDialOptionB,
filerB,
*syncOptions.bPath,
*syncOptions.bReplication,
@@ -249,9 +228,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
if !*syncOptions.isActivePassive {
// b->a
// set synchronization start timestamp to offset
initOffsetError := initOffsetFromTsMs(grpcDialOptionA, filerA, bFilerSignature, *syncOptions.bFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
initOffsetError := initOffsetFromTsMs(grpcDialOption, filerA, bFilerSignature, *syncOptions.aFromTsMs, getSignaturePrefixByPath(*syncOptions.bPath))
if initOffsetError != nil {
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.bFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
glog.Errorf("init offset from timestamp %d error from %s to %s: %v", *syncOptions.aFromTsMs, *syncOptions.filerB, *syncOptions.filerA, initOffsetError)
os.Exit(2)
}
go func() {
@@ -260,12 +239,11 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
err := doSubscribeFilerMetaChanges(
syncOptions.clientId,
syncOptions.clientEpoch.Load(),
grpcDialOptionB,
grpcDialOption,
filerB,
*syncOptions.bPath,
util.StringSplit(*syncOptions.bExcludePaths, ","),
*syncOptions.bProxyByFiler,
grpcDialOptionA,
filerA,
*syncOptions.aPath,
*syncOptions.aReplication,
@@ -307,12 +285,12 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
return nil
}
func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetGrpcDialOption grpc.DialOption, targetFiler pb.ServerAddress, targetPath string,
func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, chunkConcurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState]) error {
// if first time, start from now
// if has previously synced, resume from that point of time
sourceFilerOffsetTsNs, err := getOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
if err != nil {
return err
}
@@ -322,9 +300,8 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
// create filer sink
filerSource := &source.FilerSource{}
filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, sourceReadChunkFromFiler)
filerSource.SetGrpcDialOption(sourceGrpcDialOption)
filerSink := &filersink.FilerSink{}
filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, targetGrpcDialOption, sinkWriteChunkByFiler)
filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
filerSink.SetChunkConcurrency(chunkConcurrency)
filerSink.SetSourceFiler(filerSource)
@@ -351,7 +328,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
if statePtr != nil {
statePtr.Store(&syncState{
processor: processor,
grpcDialOption: targetGrpcDialOption,
grpcDialOption: grpcDialOption,
targetFiler: targetFiler,
sourcePath: sourcePath,
sourceFilerSignature: sourceFilerSignature,
@@ -374,7 +351,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
lastLogTsNs = now
// collect synchronous offset
statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
return setOffset(targetGrpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
})
prefix := sourcePath
@@ -395,7 +372,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi
EventErrorType: pb.RetryForeverOnError,
}
return pb.FollowMetadata(sourceFiler, sourceGrpcDialOption, metadataFollowOption, processEventFnWithOffset)
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
}
@@ -468,17 +445,12 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
message := resp.EventNotification
// Derive the target (new-side) directory once. MetadataEventTargetDirectory
// returns NewParentPath when set, falling back to resp.Directory for
// delete events or legacy events with an empty NewParentPath.
targetDir := filer_pb.MetadataEventTargetDirectory(resp)
var sourceOldKey, sourceNewKey util.FullPath
if message.OldEntry != nil {
sourceOldKey = util.FullPath(resp.Directory).Child(message.OldEntry.Name)
}
if message.NewEntry != nil {
sourceNewKey = util.FullPath(targetDir).Child(message.NewEntry.Name)
sourceNewKey = util.FullPath(message.NewParentPath).Child(message.NewEntry.Name)
}
if debug {
@@ -489,24 +461,19 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
return nil
}
// For rename events the key/directory is the old (source) path.
// Check both old and new directories so cross-boundary renames
// are not silently dropped. The downstream old/new key handling
// (lines below) already converts these to create or delete.
oldDirExcluded := matchesExcludePath(resp.Directory, excludePaths)
newDirExcluded := matchesExcludePath(targetDir, excludePaths)
oldDirInScope := util.IsEqualOrUnder(resp.Directory, sourcePath) && !oldDirExcluded
newDirInScope := message.NewEntry != nil &&
util.IsEqualOrUnder(targetDir, sourcePath) &&
!newDirExcluded
if !oldDirInScope && !newDirInScope {
if !strings.HasPrefix(resp.Directory+"/", sourcePath) {
return nil
}
for _, excludePath := range excludePaths {
if strings.HasPrefix(resp.Directory+"/", excludePath) {
return nil
}
}
// Compute per-side exclusion so that rename events crossing an
// exclude boundary are handled as delete + create rather than
// being entirely skipped.
oldExcluded := oldDirExcluded || isEntryExcluded(resp.Directory, message.OldEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
newExcluded := newDirExcluded || isEntryExcluded(targetDir, message.NewEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
oldExcluded := isEntryExcluded(resp.Directory, message.OldEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
newExcluded := isEntryExcluded(message.NewParentPath, message.NewEntry, reExcludeFileName, excludeFileNames, excludePathPatterns)
if oldExcluded && newExcluded {
return nil
@@ -528,7 +495,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
if !doDeleteFiles {
return nil
}
if !util.IsEqualOrUnder(string(sourceOldKey), sourcePath) {
if !strings.HasPrefix(string(sourceOldKey), sourcePath) {
return nil
}
key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
@@ -537,7 +504,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
// handle new entries
if filer_pb.IsCreate(resp) {
if !util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
if !strings.HasPrefix(string(sourceNewKey), sourcePath) {
return nil
}
key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
@@ -554,19 +521,18 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
}
// handle updates
if util.IsEqualOrUnder(string(sourceOldKey), sourcePath) {
if strings.HasPrefix(string(sourceOldKey), sourcePath) {
// old key is in the watched directory
if util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
if strings.HasPrefix(string(sourceNewKey), sourcePath) {
// new key is also in the watched directory
if doDeleteFiles {
oldKey := util.Join(targetPath, string(sourceOldKey)[len(sourcePath):])
var sinkNewParentPath string
if strings.HasSuffix(sourcePath, "/") {
sinkNewParentPath = util.Join(targetPath, targetDir[len(sourcePath)-1:])
message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath)-1:])
} else {
sinkNewParentPath = util.Join(targetPath, targetDir[len(sourcePath):])
message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath):])
}
foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, sinkNewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
if foundExisting {
return err
}
@@ -593,7 +559,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
}
} else {
// old key is outside the watched directory
if util.IsEqualOrUnder(string(sourceNewKey), sourcePath) {
if strings.HasPrefix(string(sourceNewKey), sourcePath) {
// new key is in the watched directory
key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
@@ -657,15 +623,6 @@ func isEntryExcluded(dir string, entry *filer_pb.Entry, reExcludeFileName *regex
return false
}
func matchesExcludePath(dir string, excludePaths []string) bool {
for _, excludePath := range excludePaths {
if util.IsEqualOrUnder(dir, excludePath) {
return true
}
}
return false
}
// compileExcludePattern compiles a regexp pattern string, returning nil if empty.
func compileExcludePattern(pattern string, label string) (*regexp.Regexp, error) {
if pattern == "" {
-121
View File
@@ -1,121 +0,0 @@
package command
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var _ sink.ReplicationSink = (*recordingSyncSink)(nil)
type recordingSyncSink struct {
deleteKeys []string
createKeys []string
updateKeys []string
}
func (s *recordingSyncSink) GetName() string { return "recording" }
func (s *recordingSyncSink) Initialize(util.Configuration, string) error {
return nil
}
func (s *recordingSyncSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
s.deleteKeys = append(s.deleteKeys, key)
return nil
}
func (s *recordingSyncSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
s.createKeys = append(s.createKeys, key)
return nil
}
func (s *recordingSyncSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
s.updateKeys = append(s.updateKeys, key)
return true, nil
}
func (s *recordingSyncSink) GetSinkToDirectory() string { return "/dest" }
func (s *recordingSyncSink) SetSourceFiler(*source.FilerSource) {}
func (s *recordingSyncSink) IsIncremental() bool { return false }
func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
tests := []struct {
name string
candidate string
other string
expected bool
}{
{name: "equal", candidate: "/foo", other: "/foo", expected: true},
{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
{name: "root", candidate: "/foo/bar", other: "/", expected: true},
{name: "empty", candidate: "", other: "/foo", expected: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
}
})
}
}
func TestMatchesExcludePathUsesDirectoryBoundaries(t *testing.T) {
if !matchesExcludePath("/tmp", []string{"/tmp"}) {
t.Fatal("expected exact directory match to be excluded")
}
if !matchesExcludePath("/tmp/sub", []string{"/tmp"}) {
t.Fatal("expected descendant directory to be excluded")
}
if matchesExcludePath("/tmp2/sub", []string{"/tmp"}) {
t.Fatal("did not expect sibling directory to be excluded")
}
}
func TestGenProcessFunctionRenameToSiblingPrefixBecomesDelete(t *testing.T) {
dataSink := &recordingSyncSink{}
processFn := genProcessFunction("/foo", "/dest", nil, nil, nil, nil, dataSink, true, false)
err := processFn(&filer_pb.SubscribeMetadataResponse{
Directory: "/foo/dir",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file.txt"},
NewEntry: &filer_pb.Entry{Name: "file.txt"},
NewParentPath: "/foobar/dir",
},
})
if err != nil {
t.Fatalf("processFn rename to sibling prefix: %v", err)
}
if len(dataSink.deleteKeys) != 1 || dataSink.deleteKeys[0] != "/dest/dir/file.txt" {
t.Fatalf("delete keys = %v, want [/dest/dir/file.txt]", dataSink.deleteKeys)
}
if len(dataSink.createKeys) != 0 || len(dataSink.updateKeys) != 0 {
t.Fatalf("unexpected create/update calls: creates=%v updates=%v", dataSink.createKeys, dataSink.updateKeys)
}
}
func TestGenProcessFunctionRenameFromExcludedDirBecomesCreate(t *testing.T) {
dataSink := &recordingSyncSink{}
processFn := genProcessFunction("/foo", "/dest", []string{"/foo/excluded"}, nil, nil, nil, dataSink, true, false)
err := processFn(&filer_pb.SubscribeMetadataResponse{
Directory: "/foo/excluded",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file.txt"},
NewEntry: &filer_pb.Entry{Name: "file.txt"},
NewParentPath: "/foo/live",
},
})
if err != nil {
t.Fatalf("processFn rename from excluded dir: %v", err)
}
if len(dataSink.createKeys) != 1 || dataSink.createKeys[0] != "/dest/live/file.txt" {
t.Fatalf("create keys = %v, want [/dest/live/file.txt]", dataSink.createKeys)
}
if len(dataSink.deleteKeys) != 0 || len(dataSink.updateKeys) != 0 {
t.Fatalf("unexpected delete/update calls: deletes=%v updates=%v", dataSink.deleteKeys, dataSink.updateKeys)
}
}
-1
View File
@@ -254,7 +254,6 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
go grpcS.Serve(grpcLocalL)
}
go grpcS.Serve(grpcL)
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
// For multi-master mode with non-Hashicorp raft, wait and check if we should join
if !*masterOption.raftHashicorp && !isSingleMaster {
-36
View File
@@ -1,36 +0,0 @@
package command
import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func metadataEventDirectoryMembership(resp *filer_pb.SubscribeMetadataResponse, dir string) (sourceInDir, targetInDir bool) {
if resp == nil || resp.EventNotification == nil {
return false, false
}
sourceInDir = util.IsEqualOrUnder(resp.Directory, dir)
targetInDir = resp.EventNotification.NewEntry != nil &&
util.IsEqualOrUnder(filer_pb.MetadataEventTargetDirectory(resp), dir)
return sourceInDir, targetInDir
}
func metadataEventUpdatesDirectory(resp *filer_pb.SubscribeMetadataResponse, dir string) bool {
if resp == nil || resp.EventNotification == nil || resp.EventNotification.NewEntry == nil {
return false
}
_, targetInDir := metadataEventDirectoryMembership(resp, dir)
return targetInDir
}
func metadataEventRemovesFromDirectory(resp *filer_pb.SubscribeMetadataResponse, dir string) bool {
if resp == nil || resp.EventNotification == nil || resp.EventNotification.OldEntry == nil {
return false
}
sourceInDir, targetInDir := metadataEventDirectoryMembership(resp, dir)
return sourceInDir && !targetInDir
}
-116
View File
@@ -1,116 +0,0 @@
package command
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestMetadataEventDirectoryMembershipUsesDirectoryBoundaries(t *testing.T) {
resp := &filer_pb.SubscribeMetadataResponse{
Directory: filer.DirectoryEtcRemote,
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "remote.conf"},
NewEntry: &filer_pb.Entry{Name: "remote.conf"},
NewParentPath: "/etc/remote-sibling",
},
}
sourceInDir, targetInDir := metadataEventDirectoryMembership(resp, filer.DirectoryEtcRemote)
if !sourceInDir {
t.Fatal("expected source directory to match")
}
if targetInDir {
t.Fatal("did not expect sibling target directory to match")
}
}
func TestMetadataEventUpdatesAndRemovesDirectory(t *testing.T) {
tests := []struct {
name string
resp *filer_pb.SubscribeMetadataResponse
wantUpdate bool
wantRemoval bool
}{
{
name: "nil response",
resp: nil,
wantUpdate: false,
wantRemoval: false,
},
{
name: "create event",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: filer.DirectoryEtcRemote,
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "new.conf"},
NewParentPath: filer.DirectoryEtcRemote,
},
},
wantUpdate: true,
wantRemoval: false,
},
{
name: "rename out",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: filer.DirectoryEtcRemote,
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "remote.conf"},
NewEntry: &filer_pb.Entry{Name: "remote.conf"},
NewParentPath: "/tmp",
},
},
wantUpdate: false,
wantRemoval: true,
},
{
name: "rename into",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: "/tmp",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "remote.conf"},
NewEntry: &filer_pb.Entry{Name: "remote.conf"},
NewParentPath: filer.DirectoryEtcRemote,
},
},
wantUpdate: true,
wantRemoval: false,
},
{
name: "rename within",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: filer.DirectoryEtcRemote,
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "remote.conf"},
NewEntry: &filer_pb.Entry{Name: "renamed.conf"},
NewParentPath: filer.DirectoryEtcRemote,
},
},
wantUpdate: true,
wantRemoval: false,
},
{
name: "delete",
resp: &filer_pb.SubscribeMetadataResponse{
Directory: filer.DirectoryEtcRemote,
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "remote.conf"},
},
},
wantUpdate: false,
wantRemoval: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := metadataEventUpdatesDirectory(tt.resp, filer.DirectoryEtcRemote); got != tt.wantUpdate {
t.Fatalf("metadataEventUpdatesDirectory() = %v, want %v", got, tt.wantUpdate)
}
if got := metadataEventRemovesFromDirectory(tt.resp, filer.DirectoryEtcRemote); got != tt.wantRemoval {
t.Fatalf("metadataEventRemovesFromDirectory() = %v, want %v", got, tt.wantRemoval)
}
})
}
}
-10
View File
@@ -823,16 +823,6 @@ func runMini(cmd *Command, args []string) bool {
miniS3Options.filer = &filerAddress
miniWebDavOptions.filer = &filerAddress
// Register Unix socket paths for gRPC services so local inter-service
// communication goes through Unix sockets instead of TCP.
pb.RegisterLocalGrpcSocket(*miniMasterOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-master-grpc-%d.sock", *miniMasterOptions.portGrpc))
pb.RegisterLocalGrpcSocket(*miniOptions.v.portGrpc, fmt.Sprintf("/tmp/seaweedfs-volume-grpc-%d.sock", *miniOptions.v.portGrpc))
pb.RegisterLocalGrpcSocket(*miniFilerOptions.portGrpc, fmt.Sprintf("/tmp/seaweedfs-filer-grpc-%d.sock", *miniFilerOptions.portGrpc))
if *miniS3Options.portGrpc > 0 {
pb.RegisterLocalGrpcSocket(*miniS3Options.portGrpc, fmt.Sprintf("/tmp/seaweedfs-s3-grpc-%d.sock", *miniS3Options.portGrpc))
}
pb.RegisterLocalGrpcSocket(*miniAdminOptions.grpcPort, fmt.Sprintf("/tmp/seaweedfs-admin-grpc-%d.sock", *miniAdminOptions.grpcPort))
go stats_collect.StartMetricsServer(*miniMetricsHttpIp, *miniMetricsHttpPort)
if *miniMasterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
-1
View File
@@ -376,7 +376,6 @@ func (s3opt *S3Options) startS3Server() bool {
go grpcS.Serve(grpcLocalL)
}
go grpcS.Serve(grpcL)
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
if *s3opt.tlsPrivateKey != "" {
// Check for port conflict when both HTTP and HTTPS are enabled on the same port
-11
View File
@@ -22,17 +22,6 @@ hosts = [
topic = "seaweedfs_filer"
offsetFile = "./last.offset"
offsetSaveIntervalSeconds = 10
# SASL Authentication
sasl_enabled = false
sasl_mechanism = "PLAIN" # PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
sasl_username = ""
sasl_password = ""
# TLS/SSL
tls_enabled = false
tls_ca_cert = "" # path to CA certificate PEM file
tls_client_cert = "" # path to client certificate PEM file (for mTLS)
tls_client_key = "" # path to client private key PEM file (for mTLS)
tls_insecure_skip_verify = false
[notification.aws_sqs]
-22
View File
@@ -330,28 +330,6 @@ func runServer(cmd *Command, args []string) bool {
}
filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder
// Register Unix socket paths for gRPC services running in this process
// so local inter-service communication uses Unix sockets instead of TCP.
// Resolve gRPC ports early (same calculation each service does internally).
for _, svc := range []struct {
starting *bool
portGrpc *int
port *int
name string
}{
{isStartingMasterServer, masterOptions.portGrpc, masterOptions.port, "master"},
{isStartingVolumeServer, serverOptions.v.portGrpc, serverOptions.v.port, "volume"},
{isStartingFiler, filerOptions.portGrpc, filerOptions.port, "filer"},
{isStartingS3, s3Options.portGrpc, s3Options.port, "s3"},
} {
if *svc.starting {
if *svc.portGrpc == 0 {
*svc.portGrpc = 10000 + *svc.port
}
pb.RegisterLocalGrpcSocket(*svc.portGrpc, fmt.Sprintf("/tmp/seaweedfs-%s-grpc-%d.sock", svc.name, *svc.portGrpc))
}
}
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
if *isStartingFiler {
-1
View File
@@ -415,7 +415,6 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe
glog.Fatalf("start gRPC service failed, %s", err)
}
}()
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
return grpcS
}
@@ -27,7 +27,6 @@ type FilerOperations interface {
CountDirectoryEntries(ctx context.Context, dirPath util.FullPath, limit int) (count int, err error)
DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32, ifNotModifiedAfter int64) error
GetEntryAttributes(ctx context.Context, p util.FullPath) (attributes map[string][]byte, err error)
IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool, error)
}
// folderState tracks the state of a folder for empty folder cleanup
@@ -107,11 +106,24 @@ func (efc *EmptyFolderCleaner) IsEnabled() bool {
// ownsFolder checks if this filer owns the folder via consistent hashing
func (efc *EmptyFolderCleaner) ownsFolder(folder string) bool {
primary := efc.lockRing.GetPrimary(folder)
if primary == "" {
return true // Single filer case or no servers
servers := efc.lockRing.GetSnapshot()
if len(servers) <= 1 {
return true // Single filer case
}
return primary == efc.host
return efc.hashKeyToServer(folder, servers) == efc.host
}
// hashKeyToServer uses consistent hashing to map a folder to a server
func (efc *EmptyFolderCleaner) hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
if len(servers) == 0 {
return ""
}
x := util.HashStringToLong(key)
if x < 0 {
x = -x
}
x = x % int64(len(servers))
return servers[x]
}
// OnDeleteEvent is called when a file or directory is deleted
@@ -300,16 +312,6 @@ func (efc *EmptyFolderCleaner) executeCleanup(folder string, triggeredBy string)
return
}
// Skip explicitly created directory markers (e.g., PUT /bucket/folder/)
// These have a MIME type set and should be preserved even when empty
if isKeyObj, err := efc.filer.IsDirectoryKeyObject(ctx, util.FullPath(folder)); err != nil {
glog.V(2).Infof("EmptyFolderCleaner: error checking directory key object %s: %v", folder, err)
return
} else if isKeyObj {
glog.V(3).Infof("EmptyFolderCleaner: skipping %s (triggered by %s), explicit directory marker", folder, triggeredBy)
return
}
// Delete the empty folder
glog.Infof("EmptyFolderCleaner: deleting empty folder %s (triggered by %s)", folder, triggeredBy)
if err := efc.deleteFolder(ctx, folder); err != nil {
@@ -12,10 +12,9 @@ import (
)
type mockFilerOps struct {
countFn func(path util.FullPath) (int, error)
deleteFn func(path util.FullPath) error
attrsFn func(path util.FullPath) (map[string][]byte, error)
isDirKeyObjFn func(path util.FullPath) (bool, error)
countFn func(path util.FullPath) (int, error)
deleteFn func(path util.FullPath) error
attrsFn func(path util.FullPath) (map[string][]byte, error)
}
func (m *mockFilerOps) CountDirectoryEntries(_ context.Context, dirPath util.FullPath, _ int) (int, error) {
@@ -39,13 +38,6 @@ func (m *mockFilerOps) GetEntryAttributes(_ context.Context, p util.FullPath) (m
return m.attrsFn(p)
}
func (m *mockFilerOps) IsDirectoryKeyObject(_ context.Context, p util.FullPath) (bool, error) {
if m.isDirKeyObjFn == nil {
return false, nil
}
return m.isDirKeyObjFn(p)
}
func Test_isUnderPath(t *testing.T) {
tests := []struct {
name string
@@ -163,7 +155,7 @@ func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
"filer2:8888",
"filer3:8888",
}
lockRing.SetSnapshot(servers, 0)
lockRing.SetSnapshot(servers)
// Create cleaner for filer1
cleaner1 := &EmptyFolderCleaner{
@@ -213,7 +205,7 @@ func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) {
// Create a LockRing with a single server
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -251,7 +243,7 @@ func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) {
func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -287,7 +279,7 @@ func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -317,7 +309,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -346,7 +338,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"})
// Create cleaner for filer that doesn't own the folder
cleaner := &EmptyFolderCleaner{
@@ -385,7 +377,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -413,7 +405,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -442,7 +434,7 @@ func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -491,7 +483,7 @@ func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
func TestEmptyFolderCleaner_Stop(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -526,7 +518,7 @@ func TestEmptyFolderCleaner_Stop(t *testing.T) {
func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -581,7 +573,7 @@ func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -615,7 +607,7 @@ func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
@@ -657,7 +649,7 @@ func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
func TestEmptyFolderCleaner_processCleanupQueue_drainsAllOnceTriggered(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
var deleted []string
mock := &mockFilerOps{
@@ -701,7 +693,7 @@ func TestEmptyFolderCleaner_processCleanupQueue_drainsAllOnceTriggered(t *testin
func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"})
var deleted []string
mock := &mockFilerOps{
@@ -741,70 +733,3 @@ func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.
t.Fatalf("expected folder %s to be skipped, got deletions %v", folder, deleted)
}
}
func TestEmptyFolderCleaner_executeCleanup_directoryMarker(t *testing.T) {
testCases := []struct {
name string
isDirKeyObj bool
expectDeletion bool
}{
{
name: "skips explicit directory marker",
isDirKeyObj: true,
expectDeletion: false,
},
{
name: "deletes implicit empty folder",
isDirKeyObj: false,
expectDeletion: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
var deleted []string
mock := &mockFilerOps{
countFn: func(_ util.FullPath) (int, error) {
return 0, nil
},
deleteFn: func(path util.FullPath) error {
deleted = append(deleted, string(path))
return nil
},
isDirKeyObjFn: func(path util.FullPath) (bool, error) {
return tc.isDirKeyObj, nil
},
}
cleaner := &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, time.Minute),
maxCountCheck: 1000,
cacheExpiry: time.Minute,
processorSleep: time.Second,
stopCh: make(chan struct{}),
}
folder := "/buckets/test/folder"
cleaner.executeCleanup(folder, "triggered_item")
if tc.expectDeletion {
if len(deleted) != 1 || deleted[0] != folder {
t.Fatalf("expected implicit empty folder %s to be deleted, got deletions %v", folder, deleted)
}
} else {
if len(deleted) != 0 {
t.Fatalf("expected explicit directory marker %s to be preserved, got deletions %v", folder, deleted)
}
}
})
}
}
+8 -25
View File
@@ -119,7 +119,7 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
address := pb.ServerAddress(node.Address)
snapshot = append(snapshot, address)
}
f.Dlm.LockRing.SetSnapshot(snapshot, 0)
f.Dlm.LockRing.SetSnapshot(snapshot)
glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
// Initialize the empty folder cleaner using the same LockRing as Dlm for consistent hashing
@@ -130,17 +130,14 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
if update.NodeType != cluster.FilerType {
return
}
// Lock ring is now managed by the master via LockRingUpdate,
// so we no longer call AddServer/RemoveServer here.
f.MetaAggregator.OnPeerUpdate(update, startFrom)
})
f.MasterClient.SetOnLockRingUpdateFn(func(update *master_pb.LockRingUpdate) {
var servers []pb.ServerAddress
for _, s := range update.Servers {
servers = append(servers, pb.ServerAddress(s))
address := pb.ServerAddress(update.Address)
if update.IsAdd {
f.Dlm.LockRing.AddServer(address)
} else {
f.Dlm.LockRing.RemoveServer(address)
}
glog.V(0).Infof("LockRing: applying master ring update v%d: %v", update.Version, servers)
f.Dlm.LockRing.SetSnapshot(servers, update.Version)
f.MetaAggregator.OnPeerUpdate(update, startFrom)
})
for _, peerUpdate := range existingNodes {
@@ -562,17 +559,3 @@ func (f *Filer) GetEntryAttributes(ctx context.Context, p util.FullPath) (map[st
}
return entry.Extended, nil
}
func (f *Filer) IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool, error) {
entry, err := f.FindEntry(ctx, p)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return false, nil
}
return false, err
}
if entry == nil {
return false, nil
}
return entry.IsDirectory() && entry.Mime != "", nil
}
-4
View File
@@ -26,10 +26,6 @@ func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
}
func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
if metadataEventsSuppressed(ctx) {
return nil
}
var fullpath string
if oldEntry != nil {
fullpath = string(oldEntry.FullPath)
+13 -19
View File
@@ -19,25 +19,17 @@ func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse)
func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
message := event.EventNotification
oldDir := event.Directory
newDir := filer_pb.MetadataEventTargetDirectory(event)
if filer_pb.IsCreate(event) {
if newDir == f.DirBucketsPath && message.NewEntry.IsDirectory {
f.Store.OnBucketCreation(message.NewEntry.Name)
if f.DirBucketsPath == event.Directory {
if filer_pb.IsCreate(event) {
if message.NewEntry.IsDirectory {
f.Store.OnBucketCreation(message.NewEntry.Name)
}
}
}
if filer_pb.IsDelete(event) {
if oldDir == f.DirBucketsPath && message.OldEntry.IsDirectory {
f.Store.OnBucketDeletion(message.OldEntry.Name)
}
}
if filer_pb.IsRename(event) {
if oldDir == f.DirBucketsPath && message.OldEntry.IsDirectory {
f.Store.OnBucketDeletion(message.OldEntry.Name)
}
if newDir == f.DirBucketsPath && message.NewEntry.IsDirectory {
f.Store.OnBucketCreation(message.NewEntry.Name)
if filer_pb.IsDelete(event) {
if message.OldEntry.IsDirectory {
f.Store.OnBucketDeletion(message.OldEntry.Name)
}
}
}
}
@@ -80,8 +72,10 @@ func (f *Filer) onEmptyFolderCleanupEvents(event *filer_pb.SubscribeMetadataResp
}
func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
if !filer_pb.MetadataEventTouchesDirectory(event, DirectoryEtcSeaweedFS) {
return
if DirectoryEtcSeaweedFS != event.Directory {
if DirectoryEtcSeaweedFS != event.EventNotification.NewParentPath {
return
}
}
entry := event.EventNotification.NewEntry
-70
View File
@@ -1,70 +0,0 @@
package filer
import (
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type bucketTrackingStore struct {
created []string
deleted []string
}
func (s *bucketTrackingStore) GetName() string { return "bucket-tracking" }
func (s *bucketTrackingStore) Initialize(configuration util.Configuration, prefix string) error {
return nil
}
func (s *bucketTrackingStore) InsertEntry(context.Context, *Entry) error { return nil }
func (s *bucketTrackingStore) UpdateEntry(context.Context, *Entry) error { return nil }
func (s *bucketTrackingStore) FindEntry(context.Context, util.FullPath) (*Entry, error) {
return nil, filer_pb.ErrNotFound
}
func (s *bucketTrackingStore) DeleteEntry(context.Context, util.FullPath) error { return nil }
func (s *bucketTrackingStore) DeleteFolderChildren(context.Context, util.FullPath) error { return nil }
func (s *bucketTrackingStore) ListDirectoryEntries(context.Context, util.FullPath, string, bool, int64, ListEachEntryFunc) (string, error) {
return "", nil
}
func (s *bucketTrackingStore) ListDirectoryPrefixedEntries(context.Context, util.FullPath, string, bool, int64, string, ListEachEntryFunc) (string, error) {
return "", nil
}
func (s *bucketTrackingStore) BeginTransaction(ctx context.Context) (context.Context, error) {
return ctx, nil
}
func (s *bucketTrackingStore) CommitTransaction(context.Context) error { return nil }
func (s *bucketTrackingStore) RollbackTransaction(context.Context) error { return nil }
func (s *bucketTrackingStore) KvPut(context.Context, []byte, []byte) error { return nil }
func (s *bucketTrackingStore) KvGet(context.Context, []byte) ([]byte, error) {
return nil, ErrKvNotFound
}
func (s *bucketTrackingStore) KvDelete(context.Context, []byte) error { return nil }
func (s *bucketTrackingStore) Shutdown() {}
func (s *bucketTrackingStore) OnBucketCreation(bucket string) { s.created = append(s.created, bucket) }
func (s *bucketTrackingStore) OnBucketDeletion(bucket string) { s.deleted = append(s.deleted, bucket) }
func (s *bucketTrackingStore) CanDropWholeBucket() bool { return false }
func TestOnBucketEventsRenameIntoBucketsRootCreatesBucket(t *testing.T) {
store := &bucketTrackingStore{}
f := &Filer{
DirBucketsPath: "/buckets",
Store: NewFilerStoreWrapper(store),
}
f.onBucketEvents(&filer_pb.SubscribeMetadataResponse{
Directory: "/tmp",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "migrated", IsDirectory: true},
NewEntry: &filer_pb.Entry{Name: "migrated", IsDirectory: true},
NewParentPath: "/buckets",
},
})
if len(store.created) != 1 || store.created[0] != "migrated" {
t.Fatalf("created buckets = %v, want [migrated]", store.created)
}
if len(store.deleted) != 0 {
t.Fatalf("deleted buckets = %v, want []", store.deleted)
}
}
-34
View File
@@ -25,12 +25,10 @@ type VirtualFilerStore interface {
FilerStore
DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
DeleteOneEntry(ctx context.Context, entry *Entry) error
InsertEntryKnownAbsent(ctx context.Context, entry *Entry) error
AddPathSpecificStore(path string, storeId string, store FilerStore)
OnBucketCreation(bucket string)
OnBucketDeletion(bucket string)
CanDropWholeBucket() bool
SameActualStore(a, b util.FullPath) bool
}
type FilerStoreWrapper struct {
@@ -108,13 +106,6 @@ func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerSto
return
}
// SameActualStore reports whether two paths resolve to the same underlying
// store. When path-specific stores are configured, different subtrees may
// be served by different backends.
func (fsw *FilerStoreWrapper) SameActualStore(a, b util.FullPath) bool {
return fsw.getActualStore(a) == fsw.getActualStore(b)
}
func (fsw *FilerStoreWrapper) getDefaultStore() (store FilerStore) {
return fsw.defaultStore
}
@@ -152,31 +143,6 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
return actualStore.InsertEntry(ctx, entry)
}
// InsertEntryKnownAbsent skips the pre-insert FindEntry path when the caller has
// already established that the target path does not exist.
func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry *Entry) error {
ctx = context.WithoutCancel(ctx)
actualStore := fsw.getActualStore(entry.FullPath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "insert").Observe(time.Since(start).Seconds())
}()
filer_pb.BeforeEntrySerialization(entry.GetChunks())
normalizeEntryMimeForStore(entry)
if len(entry.HardLinkId) > 0 {
glog.V(4).InfofCtx(ctx, "InsertEntryKnownAbsent %s has HardLinkId %x counter=%d",
entry.FullPath, entry.HardLinkId, entry.HardLinkCounter)
if err := fsw.setHardLink(ctx, entry); err != nil {
return fmt.Errorf("setHardLink %x: %v", entry.HardLinkId, err)
}
}
return actualStore.InsertEntry(ctx, entry)
}
func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
ctx = context.WithoutCancel(ctx)
actualStore := fsw.getActualStore(entry.FullPath)
-20
View File
@@ -1,20 +0,0 @@
package filer
import "context"
type suppressMetadataEventsKey struct{}
// WithSuppressedMetadataEvents disables automatic metadata event emission for
// nested filer operations that are part of a larger logical change, such as a
// rename implemented via create+delete.
func WithSuppressedMetadataEvents(ctx context.Context) context.Context {
return context.WithValue(ctx, suppressMetadataEventsKey{}, true)
}
func metadataEventsSuppressed(ctx context.Context) bool {
if ctx == nil {
return false
}
suppressed, _ := ctx.Value(suppressMetadataEventsKey{}).(bool)
return suppressed
}
+1 -19
View File
@@ -141,26 +141,12 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
var urlStrings []string
var err error
for _, backoff := range getLookupFileIdBackoffSchedule {
if err := ctx.Err(); err != nil {
return nil, err
}
urlStrings, err = masterClient.GetLookupFileIdFunction()(ctx, chunkView.FileId)
if err == nil && len(urlStrings) > 0 {
break
}
if err := ctx.Err(); err != nil {
return nil, err
}
glog.V(4).InfofCtx(ctx, "waiting for chunk: %s", chunkView.FileId)
timer := time.NewTimer(backoff)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return nil, ctx.Err()
case <-timer.C:
}
time.Sleep(backoff)
}
if err != nil {
glog.V(1).InfofCtx(ctx, "operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
@@ -193,10 +179,6 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
jwt := jwtFunc(chunkView.FileId)
written, err := retriedStreamFetchChunkData(ctx, writer, urlStrings, jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
if err != nil && ctx.Err() != nil {
return ctx.Err()
}
// If read failed, try to invalidate cache and re-lookup
if err != nil && written == 0 {
if invalidator, ok := masterClient.(CacheInvalidator); ok {
-106
View File
@@ -1,11 +1,8 @@
package filer
import (
"bytes"
"context"
"errors"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
@@ -176,106 +173,3 @@ func TestRetryLogicSkipsSameUrls(t *testing.T) {
t.Error("Expected different URLs to not be equal")
}
}
func TestCanceledStreamSkipsCacheInvalidation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
fileId := "3,canceled"
mock := &mockMasterClient{
lookupFunc: func(ctx context.Context, fid string) ([]string, error) {
return []string{"http://server:8080"}, nil
},
}
chunks := []*filer_pb.FileChunk{
{
FileId: fileId,
Offset: 0,
Size: 10,
},
}
streamFn, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
if err != nil {
t.Fatalf("PrepareStreamContentWithThrottler failed: %v", err)
}
cancel()
err = streamFn(&bytes.Buffer{})
if err != context.Canceled {
t.Fatalf("expected context.Canceled, got %v", err)
}
if len(mock.invalidatedFileIds) != 0 {
t.Fatalf("expected no cache invalidation on cancellation, got %v", mock.invalidatedFileIds)
}
}
func TestPrepareStreamContentSkipsLookupWhenContextAlreadyCanceled(t *testing.T) {
oldSchedule := getLookupFileIdBackoffSchedule
getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond}
t.Cleanup(func() {
getLookupFileIdBackoffSchedule = oldSchedule
})
ctx, cancel := context.WithCancel(context.Background())
cancel()
lookupCalls := 0
mock := &mockMasterClient{
lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
lookupCalls++
return nil, errors.New("lookup should not run")
},
}
chunks := []*filer_pb.FileChunk{
{
FileId: "3,precanceled",
Offset: 0,
Size: 10,
},
}
_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got %v", err)
}
if lookupCalls != 0 {
t.Fatalf("expected no lookup calls after cancellation, got %d", lookupCalls)
}
}
func TestPrepareStreamContentStopsLookupRetriesAfterContextCancellation(t *testing.T) {
oldSchedule := getLookupFileIdBackoffSchedule
getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
t.Cleanup(func() {
getLookupFileIdBackoffSchedule = oldSchedule
})
ctx, cancel := context.WithCancel(context.Background())
lookupCalls := 0
mock := &mockMasterClient{
lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
lookupCalls++
cancel()
return nil, context.Canceled
},
}
chunks := []*filer_pb.FileChunk{
{
FileId: "3,cancel-during-lookup",
Offset: 0,
Size: 10,
},
}
_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context.Canceled, got %v", err)
}
if lookupCalls != 1 {
t.Fatalf("expected lookup retries to stop after cancellation, got %d calls", lookupCalls)
}
}
+10 -10
View File
@@ -46,11 +46,19 @@ func (wfs *WFS) subscribeFilerConfEvents() (*meta_cache.MetadataFollower, error)
}
processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
if !isFilerConfUpdateEvent(resp, confDir, confName) {
message := resp.EventNotification
if message.NewEntry == nil {
return nil
}
content := resp.EventNotification.NewEntry.Content
dir := resp.Directory
name := resp.EventNotification.NewEntry.Name
if dir != confDir || name != confName {
return nil
}
content := message.NewEntry.Content
fc := filer.NewFilerConf()
if len(content) > 0 {
if err = fc.LoadFromBytes(content); err != nil {
@@ -68,14 +76,6 @@ func (wfs *WFS) subscribeFilerConfEvents() (*meta_cache.MetadataFollower, error)
}, nil
}
func isFilerConfUpdateEvent(resp *filer_pb.SubscribeMetadataResponse, confDir, confName string) bool {
if resp == nil || resp.EventNotification == nil || resp.EventNotification.NewEntry == nil {
return false
}
return filer_pb.MetadataEventTargetDirectory(resp) == confDir &&
resp.EventNotification.NewEntry.Name == confName
}
func (wfs *WFS) wormEnforcedForEntry(path util.FullPath, entry *filer_pb.Entry) (wormEnforced, wormEnabled bool) {
if entry == nil || wfs.FilerConf == nil {
return false, false
-23
View File
@@ -1,23 +0,0 @@
package mount
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestIsFilerConfUpdateEventMatchesRenameTarget(t *testing.T) {
event := &filer_pb.SubscribeMetadataResponse{
Directory: "/tmp",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: filer.FilerConfName},
NewEntry: &filer_pb.Entry{Name: filer.FilerConfName},
NewParentPath: filer.DirectoryEtcSeaweedFS,
},
}
if !isFilerConfUpdateEvent(event, filer.DirectoryEtcSeaweedFS, filer.FilerConfName) {
t.Fatalf("expected rename target to match filer.conf watcher")
}
}
+1 -35
View File
@@ -184,29 +184,6 @@ func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPa
return nil
}
func (mc *MetaCache) shouldHideEntry(fullpath util.FullPath) bool {
if mc.includeSystemEntries {
return false
}
dir, name := fullpath.DirAndName()
return IsHiddenSystemEntry(dir, name)
}
func (mc *MetaCache) purgeEntryLocked(ctx context.Context, fullpath util.FullPath, isDirectory bool) error {
if fullpath == "" {
return nil
}
if err := mc.localStore.DeleteEntry(ctx, fullpath); err != nil {
return err
}
if isDirectory {
if err := mc.localStore.DeleteFolderChildren(ctx, fullpath); err != nil {
return err
}
}
return nil
}
func (mc *MetaCache) ApplyMetadataResponse(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
if resp == nil || resp.EventNotification == nil {
return nil
@@ -545,9 +522,7 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
}
var oldPath util.FullPath
var newPath util.FullPath
var newEntry *filer.Entry
hideNewPath := false
if message.OldEntry != nil {
oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name)
}
@@ -557,20 +532,11 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
if message.NewParentPath != "" {
dir = message.NewParentPath
}
newPath = util.NewFullPath(dir, message.NewEntry.Name)
hideNewPath = mc.shouldHideEntry(newPath)
if !hideNewPath {
newEntry = filer.FromPbEntry(dir, message.NewEntry)
}
newEntry = filer.FromPbEntry(dir, message.NewEntry)
}
mc.Lock()
err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert)
if err == nil && hideNewPath {
if purgeErr := mc.purgeEntryLocked(ctx, newPath, message.NewEntry.IsDirectory); purgeErr != nil {
err = purgeErr
}
}
// When a directory is deleted or moved, remove its cached descendants
// so stale children cannot be served from the local cache.
if err == nil && oldPath != "" && message.OldEntry != nil && message.OldEntry.IsDirectory {
@@ -2,7 +2,6 @@ package meta_cache
import (
"context"
"os"
"path/filepath"
"sync"
"testing"
@@ -297,114 +296,6 @@ func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) {
}
}
func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
})
defer mc.Shutdown()
createResp := &filer_pb.SubscribeMetadataResponse{
Directory: "/",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "topics",
Attributes: &filer_pb.FuseAttributes{
Crtime: 1,
Mtime: 1,
FileMode: uint32(os.ModeDir | 0o755),
},
IsDirectory: true,
},
},
}
if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply create: %v", err)
}
entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics"))
if err != filer_pb.ErrNotFound {
t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound)
}
if entry != nil {
t.Fatalf("hidden entry still cached: %+v", entry)
}
}
func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
"/src": true,
})
defer mc.Shutdown()
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/topics",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: os.ModeDir | 0o755,
},
}); err != nil {
t.Fatalf("insert stale hidden dir: %v", err)
}
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/topics/leaked.txt",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: 0o644,
FileSize: 7,
},
}); err != nil {
t.Fatalf("insert leaked hidden child: %v", err)
}
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/src/visible",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: os.ModeDir | 0o755,
},
}); err != nil {
t.Fatalf("insert source dir: %v", err)
}
renameResp := &filer_pb.SubscribeMetadataResponse{
Directory: "/src",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "visible",
IsDirectory: true,
},
NewEntry: &filer_pb.Entry{
Name: "topics",
Attributes: &filer_pb.FuseAttributes{
Crtime: 2,
Mtime: 2,
FileMode: uint32(os.ModeDir | 0o755),
},
IsDirectory: true,
},
NewParentPath: "/",
},
}
if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply rename: %v", err)
}
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil {
t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
}
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil {
t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
}
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil {
t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
}
}
func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) {
t.Helper()
+1 -17
View File
@@ -1,8 +1,6 @@
package kafka
import (
"fmt"
"github.com/Shopify/sarama"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
@@ -29,29 +27,15 @@ func (k *KafkaQueue) Initialize(configuration util.Configuration, prefix string)
return k.initialize(
configuration.GetStringSlice(prefix+"hosts"),
configuration.GetString(prefix+"topic"),
SASLTLSConfig{
SASLEnabled: configuration.GetBool(prefix + "sasl_enabled"),
SASLMechanism: configuration.GetString(prefix + "sasl_mechanism"),
SASLUsername: configuration.GetString(prefix + "sasl_username"),
SASLPassword: configuration.GetString(prefix + "sasl_password"),
TLSEnabled: configuration.GetBool(prefix + "tls_enabled"),
TLSCACert: configuration.GetString(prefix + "tls_ca_cert"),
TLSClientCert: configuration.GetString(prefix + "tls_client_cert"),
TLSClientKey: configuration.GetString(prefix + "tls_client_key"),
TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
},
)
}
func (k *KafkaQueue) initialize(hosts []string, topic string, saslTLS SASLTLSConfig) (err error) {
func (k *KafkaQueue) initialize(hosts []string, topic string) (err error) {
config := sarama.NewConfig()
config.Producer.RequiredAcks = sarama.WaitForLocal
config.Producer.Partitioner = sarama.NewHashPartitioner
config.Producer.Return.Successes = true
config.Producer.Return.Errors = true
if err = ConfigureSASLTLS(config, saslTLS); err != nil {
return fmt.Errorf("kafka producer security configuration: %w", err)
}
k.producer, err = sarama.NewAsyncProducer(hosts, config)
if err != nil {
return err
-112
View File
@@ -1,112 +0,0 @@
package kafka
import (
"crypto/tls"
"crypto/x509"
"fmt"
"os"
"strings"
"github.com/Shopify/sarama"
"github.com/xdg-go/scram"
)
// SASLTLSConfig holds SASL and TLS configuration for Kafka connections.
type SASLTLSConfig struct {
SASLEnabled bool
SASLMechanism string
SASLUsername string
SASLPassword string
TLSEnabled bool
TLSCACert string
TLSClientCert string
TLSClientKey string
TLSInsecureSkipVerify bool
}
// ConfigureSASLTLS applies SASL and TLS settings to a sarama config.
func ConfigureSASLTLS(config *sarama.Config, st SASLTLSConfig) error {
if st.SASLEnabled {
config.Net.SASL.Enable = true
config.Net.SASL.User = st.SASLUsername
config.Net.SASL.Password = st.SASLPassword
mechanism := strings.ToUpper(st.SASLMechanism)
switch mechanism {
case "PLAIN", "":
config.Net.SASL.Mechanism = sarama.SASLTypePlaintext
case "SCRAM-SHA-256":
config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &scramClient{HashGeneratorFcn: scram.SHA256}
}
case "SCRAM-SHA-512":
config.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient {
return &scramClient{HashGeneratorFcn: scram.SHA512}
}
default:
return fmt.Errorf("unsupported SASL mechanism: %s", mechanism)
}
}
if st.TLSEnabled {
if (st.TLSClientCert == "") != (st.TLSClientKey == "") {
return fmt.Errorf("both tls_client_cert and tls_client_key must be provided for mTLS, or neither")
}
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: st.TLSInsecureSkipVerify,
}
if st.TLSCACert != "" {
caCert, err := os.ReadFile(st.TLSCACert)
if err != nil {
return fmt.Errorf("failed to read CA certificate: %w", err)
}
caCertPool := x509.NewCertPool()
if !caCertPool.AppendCertsFromPEM(caCert) {
return fmt.Errorf("failed to parse CA certificate")
}
tlsConfig.RootCAs = caCertPool
}
if st.TLSClientCert != "" && st.TLSClientKey != "" {
cert, err := tls.LoadX509KeyPair(st.TLSClientCert, st.TLSClientKey)
if err != nil {
return fmt.Errorf("failed to load client certificate/key: %w", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
config.Net.TLS.Enable = true
config.Net.TLS.Config = tlsConfig
}
return nil
}
// scramClient implements the sarama.SCRAMClient interface.
type scramClient struct {
*scram.ClientConversation
scram.HashGeneratorFcn
}
func (c *scramClient) Begin(userName, password, authzID string) (err error) {
client, err := c.HashGeneratorFcn.NewClient(userName, password, authzID)
if err != nil {
return err
}
c.ClientConversation = client.NewConversation()
return nil
}
func (c *scramClient) Step(challenge string) (string, error) {
return c.ClientConversation.Step(challenge)
}
func (c *scramClient) Done() bool {
return c.ClientConversation.Done()
}
@@ -1,65 +0,0 @@
package kafka
import (
"strings"
"testing"
"github.com/Shopify/sarama"
)
func TestConfigureSASLTLSRejectsPartialMTLSConfig(t *testing.T) {
tests := []struct {
name string
cfg SASLTLSConfig
}{
{
name: "missing key",
cfg: SASLTLSConfig{
TLSEnabled: true,
TLSClientCert: "/tmp/client.crt",
},
},
{
name: "missing cert",
cfg: SASLTLSConfig{
TLSEnabled: true,
TLSClientKey: "/tmp/client.key",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ConfigureSASLTLS(sarama.NewConfig(), tt.cfg)
if err == nil {
t.Fatal("expected error")
}
if !strings.Contains(err.Error(), "both tls_client_cert and tls_client_key must be provided") {
t.Fatalf("unexpected error: %v", err)
}
})
}
}
func TestConfigureSASLTLSConfiguresSCRAMSHA256(t *testing.T) {
config := sarama.NewConfig()
err := ConfigureSASLTLS(config, SASLTLSConfig{
SASLEnabled: true,
SASLMechanism: "SCRAM-SHA-256",
SASLUsername: "alice",
SASLPassword: "secret",
})
if err != nil {
t.Fatalf("ConfigureSASLTLS returned error: %v", err)
}
if !config.Net.SASL.Enable {
t.Fatal("expected SASL to be enabled")
}
if config.Net.SASL.Mechanism != sarama.SASLTypeSCRAMSHA256 {
t.Fatalf("unexpected mechanism: %v", config.Net.SASL.Mechanism)
}
if config.Net.SASL.SCRAMClientGeneratorFunc == nil {
t.Fatal("expected SCRAM client generator")
}
}
+4 -17
View File
@@ -5,7 +5,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type filter struct {
@@ -41,37 +40,25 @@ func newFilter(cfg *config) *filter {
}
func (f *filter) shouldPublish(key string, notification *filer_pb.EventNotification) bool {
if !f.matchesPath(key, notification) {
if !f.matchesPath(key) {
return false
}
eventType := detectEventType(key, notification)
eventType := detectEventType(notification)
return f.eventTypes[eventType]
}
func (f *filter) matchesPath(key string, notification *filer_pb.EventNotification) bool {
func (f *filter) matchesPath(key string) bool {
if len(f.pathPrefixes) == 0 {
return true
}
if f.matchesAnyPathPrefix(key) {
return true
}
if notification != nil && notification.NewEntry != nil && notification.NewParentPath != "" {
newKey := string(util.FullPath(notification.NewParentPath).Child(notification.NewEntry.Name))
return f.matchesAnyPathPrefix(newKey)
}
return false
}
func (f *filter) matchesAnyPathPrefix(key string) bool {
for _, prefix := range f.pathPrefixes {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}
+10 -53
View File
@@ -9,7 +9,6 @@ import (
func TestFilterEventTypes(t *testing.T) {
tests := []struct {
name string
key string
eventTypes []string
notification *filer_pb.EventNotification
expectedType eventType
@@ -17,7 +16,6 @@ func TestFilterEventTypes(t *testing.T) {
}{
{
name: "create event - allowed",
key: "/test/test.txt",
eventTypes: []string{"create", "delete"},
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -27,7 +25,6 @@ func TestFilterEventTypes(t *testing.T) {
},
{
name: "create event - not allowed",
key: "/test/test.txt",
eventTypes: []string{"delete", "update"},
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -37,7 +34,6 @@ func TestFilterEventTypes(t *testing.T) {
},
{
name: "delete event - allowed",
key: "/test/test.txt",
eventTypes: []string{"create", "delete"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -47,19 +43,16 @@ func TestFilterEventTypes(t *testing.T) {
},
{
name: "update event - allowed",
key: "/test/test.txt",
eventTypes: []string{"update"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "test.txt"},
NewEntry: &filer_pb.Entry{Name: "test.txt"},
NewParentPath: "/test",
OldEntry: &filer_pb.Entry{Name: "test.txt"},
NewEntry: &filer_pb.Entry{Name: "test.txt"},
},
expectedType: eventTypeUpdate,
shouldPublish: true,
},
{
name: "rename event - allowed",
key: "/old/path/old.txt",
eventTypes: []string{"rename"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "old.txt"},
@@ -69,21 +62,8 @@ func TestFilterEventTypes(t *testing.T) {
expectedType: eventTypeRename,
shouldPublish: true,
},
{
name: "rename event same name different parent - allowed",
key: "/old/path/file.txt",
eventTypes: []string{"rename"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file.txt"},
NewEntry: &filer_pb.Entry{Name: "file.txt"},
NewParentPath: "/new/path",
},
expectedType: eventTypeRename,
shouldPublish: true,
},
{
name: "rename event - not allowed",
key: "/old/path/old.txt",
eventTypes: []string{"create", "delete", "update"},
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "old.txt"},
@@ -95,7 +75,6 @@ func TestFilterEventTypes(t *testing.T) {
},
{
name: "all events allowed when empty",
key: "/test/test.txt",
eventTypes: []string{},
notification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
@@ -110,12 +89,12 @@ func TestFilterEventTypes(t *testing.T) {
cfg := &config{eventTypes: tt.eventTypes}
f := newFilter(cfg)
eventType := detectEventType(tt.key, tt.notification)
eventType := detectEventType(tt.notification)
if eventType != tt.expectedType {
t.Errorf("detectEventType() = %v, want %v", eventType, tt.expectedType)
}
shouldPublish := f.shouldPublish(tt.key, tt.notification)
shouldPublish := f.shouldPublish("/test/path", tt.notification)
if shouldPublish != tt.shouldPublish {
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
}
@@ -127,82 +106,60 @@ func TestFilterPathPrefixes(t *testing.T) {
tests := []struct {
name string
pathPrefixes []string
eventTypes []string
key string
notification *filer_pb.EventNotification
shouldPublish bool
}{
{
name: "matches single prefix",
pathPrefixes: []string{"/data/"},
eventTypes: []string{"create"},
key: "/data/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: true,
},
{
name: "matches one of multiple prefixes",
pathPrefixes: []string{"/data/", "/logs/", "/tmp/"},
eventTypes: []string{"create"},
key: "/logs/app.log",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "app.log"}},
shouldPublish: true,
},
{
name: "no match",
pathPrefixes: []string{"/data/", "/logs/"},
eventTypes: []string{"create"},
key: "/other/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: false,
},
{
name: "empty prefixes allows all",
pathPrefixes: []string{},
eventTypes: []string{"create"},
key: "/any/path/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: true,
},
{
name: "exact prefix match",
pathPrefixes: []string{"/data"},
eventTypes: []string{"create"},
key: "/data",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "data"}},
shouldPublish: true,
},
{
name: "partial match not allowed",
pathPrefixes: []string{"/data/"},
eventTypes: []string{"create"},
key: "/database/file.txt",
notification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "file.txt"}},
shouldPublish: false,
},
{
name: "rename matches destination prefix",
pathPrefixes: []string{"/watched/"},
eventTypes: []string{"rename"},
key: "/outside/old.txt",
notification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "old.txt"},
NewEntry: &filer_pb.Entry{Name: "new.txt"},
NewParentPath: "/watched",
},
shouldPublish: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := &config{
pathPrefixes: tt.pathPrefixes,
eventTypes: tt.eventTypes,
eventTypes: []string{"create"},
}
f := newFilter(cfg)
shouldPublish := f.shouldPublish(tt.key, tt.notification)
notification := &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{Name: "test.txt"},
}
shouldPublish := f.shouldPublish(tt.key, notification)
if shouldPublish != tt.shouldPublish {
t.Errorf("shouldPublish() = %v, want %v", shouldPublish, tt.shouldPublish)
}
+4 -8
View File
@@ -59,7 +59,7 @@ func newWebhookMessage(key string, message proto.Message) *webhookMessage {
return nil
}
eventType := string(detectEventType(key, notification))
eventType := string(detectEventType(notification))
return &webhookMessage{
Key: key,
@@ -157,9 +157,10 @@ func (c *config) validate() error {
return nil
}
func detectEventType(key string, notification *filer_pb.EventNotification) eventType {
func detectEventType(notification *filer_pb.EventNotification) eventType {
hasOldEntry := notification.OldEntry != nil
hasNewEntry := notification.NewEntry != nil
hasNewParentPath := notification.NewParentPath != ""
if !hasOldEntry && hasNewEntry {
return eventTypeCreate
@@ -170,12 +171,7 @@ func detectEventType(key string, notification *filer_pb.EventNotification) event
}
if hasOldEntry && hasNewEntry {
oldDir, _ := util.FullPath(key).DirAndName()
newDir := notification.NewParentPath
if newDir == "" {
newDir = oldDir
}
if oldDir != newDir || notification.OldEntry.Name != notification.NewEntry.Name {
if hasNewParentPath {
return eventTypeRename
}
+24 -39
View File
@@ -97,13 +97,6 @@ var (
once sync.Once
)
var uploadRetryableAssignErrList = []string{
"transport",
"is read only",
"failed to write to local disk",
"Volume Size ",
}
// HTTPClient interface for testing
type HTTPClient interface {
Do(req *http.Request) (*http.Response, error)
@@ -135,34 +128,6 @@ func newUploader(httpClient HTTPClient) *Uploader {
}
}
func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, err error), uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, data []byte) (fileId string, uploadResult *UploadResult, err error) {
doUploadFunc := func() error {
var host string
var auth security.EncodedJwt
fileId, host, auth, err = assignFn()
if err != nil {
return err
}
uploadOption.UploadUrl = genFileUrlFn(host, fileId)
uploadOption.Jwt = auth
uploadResult, err = uploader.retriedUploadData(context.Background(), data, uploadOption)
return err
}
if uploadOption.RetryForever {
util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) {
glog.V(0).Infof("upload content: %v", err)
return true
})
} else {
err = util.MultiRetry("uploadWithRetry", uploadRetryableAssignErrList, doUploadFunc)
}
return
}
// UploadWithRetry will retry both assigning volume request and uploading content
// The option parameter does not need to specify UploadUrl and Jwt, which will come from assigning volume.
func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.AssignVolumeRequest, uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, reader io.Reader) (fileId string, uploadResult *UploadResult, err error, data []byte) {
@@ -179,7 +144,11 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi
glog.V(4).Infof("upload read %d bytes from %s", len(data), uploadOption.SourceUrl)
}
fileId, uploadResult, err = uploader.uploadWithRetryData(func() (fileId string, host string, auth security.EncodedJwt, err error) {
doUploadFunc := func() error {
var host string
var auth security.EncodedJwt
// grpc assign volume
if grpcAssignErr := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, assignErr := client.AssignVolume(context.Background(), assignRequest)
@@ -197,10 +166,26 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assi
return nil
}); grpcAssignErr != nil {
err = fmt.Errorf("filerGrpcAddress assign volume: %w", grpcAssignErr)
return fmt.Errorf("filerGrpcAddress assign volume: %w", grpcAssignErr)
}
return
}, uploadOption, genFileUrlFn, data)
uploadOption.UploadUrl = genFileUrlFn(host, fileId)
uploadOption.Jwt = auth
var uploadErr error
uploadResult, uploadErr = uploader.retriedUploadData(context.Background(), data, uploadOption)
return uploadErr
}
if uploadOption.RetryForever {
util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) {
glog.V(0).Infof("upload content: %v", err)
return true
})
} else {
uploadErrList := []string{"transport", "is read only"}
err = util.MultiRetry("uploadWithRetry", uploadErrList, doUploadFunc)
}
return
}
-124
View File
@@ -1,124 +0,0 @@
package operation
import (
"fmt"
"io"
"net/http"
"strings"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/weed/security"
)
type scriptedHTTPResponse struct {
status int
body string
}
type scriptedHTTPClient struct {
mu sync.Mutex
responses map[string][]scriptedHTTPResponse
calls []string
}
func testIsUploadRetryableAssignError(err error) bool {
if err == nil {
return false
}
for _, retryable := range uploadRetryableAssignErrList {
if strings.Contains(err.Error(), retryable) {
return true
}
}
return false
}
func (c *scriptedHTTPClient) Do(req *http.Request) (*http.Response, error) {
c.mu.Lock()
defer c.mu.Unlock()
url := req.URL.String()
c.calls = append(c.calls, url)
plans := c.responses[url]
if len(plans) == 0 {
return nil, fmt.Errorf("unexpected request to %s", url)
}
plan := plans[0]
c.responses[url] = plans[1:]
return &http.Response{
StatusCode: plan.status,
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(plan.body)),
}, nil
}
func TestIsUploadRetryableAssignError(t *testing.T) {
testCases := []struct {
name string
err error
want bool
}{
{name: "nil", err: nil, want: false},
{name: "transport", err: fmt.Errorf("transport is closing"), want: true},
{name: "read only", err: fmt.Errorf("volume 1 is read only"), want: true},
{name: "volume full", err: fmt.Errorf("failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"), want: true},
{name: "other permanent", err: fmt.Errorf("mismatching cookie"), want: false},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if got := testIsUploadRetryableAssignError(tc.err); got != tc.want {
t.Fatalf("testIsUploadRetryableAssignError(%v) = %v, want %v", tc.err, got, tc.want)
}
})
}
}
func TestUploadWithRetryDataReassignsOnVolumeSizeExceeded(t *testing.T) {
httpClient := &scriptedHTTPClient{
responses: map[string][]scriptedHTTPResponse{
"http://volume-a/1,first": {
{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
},
"http://volume-b/2,second": {
{status: http.StatusCreated, body: `{"name":"test.bin","size":3}`},
},
},
}
uploader := newUploader(httpClient)
assignCalls := 0
fileID, uploadResult, err := uploader.uploadWithRetryData(func() (string, string, security.EncodedJwt, error) {
assignCalls++
if assignCalls == 1 {
return "1,first", "volume-a", "", nil
}
return "2,second", "volume-b", "", nil
}, &UploadOption{Filename: "test.bin"}, func(host, fileId string) string {
return "http://" + host + "/" + fileId
}, []byte("abc"))
if err != nil {
t.Fatalf("expected success after reassignment, got %v", err)
}
if fileID != "2,second" {
t.Fatalf("expected second file id, got %s", fileID)
}
if assignCalls != 2 {
t.Fatalf("expected 2 assign attempts, got %d", assignCalls)
}
if uploadResult == nil || uploadResult.Name != "test.bin" {
t.Fatalf("expected successful upload result, got %#v", uploadResult)
}
if len(httpClient.calls) != 4 {
t.Fatalf("expected 4 upload attempts (3 same-url retries + 1 reassigned upload), got %d", len(httpClient.calls))
}
if httpClient.calls[0] != "http://volume-a/1,first" || httpClient.calls[3] != "http://volume-b/2,second" {
t.Fatalf("unexpected upload call sequence: %#v", httpClient.calls)
}
}
-17
View File
@@ -84,8 +84,6 @@ service SeaweedFiler {
// distributed lock management internal use only
rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
}
rpc ReplicateLock(ReplicateLockRequest) returns (ReplicateLockResponse) {
}
}
//////////////////////////////////////////////////
@@ -524,7 +522,6 @@ message LockResponse {
string lock_owner = 2;
string lock_host_moved_to = 3;
string error = 4;
int64 generation = 5;
}
message UnlockRequest {
string name = 1;
@@ -547,26 +544,12 @@ message Lock {
string renew_token = 2;
int64 expired_at_ns = 3;
string owner = 4;
int64 generation = 5;
bool is_backup = 6;
int64 seq = 7;
}
message TransferLocksRequest {
repeated Lock locks = 1;
}
message TransferLocksResponse {
}
message ReplicateLockRequest {
string name = 1;
string renew_token = 2;
int64 expired_at_ns = 3;
string owner = 4;
int64 generation = 5;
bool is_unlock = 6;
int64 seq = 7;
}
message ReplicateLockResponse {
}
//////////////////////////////////////////////////
// StreamMutateEntry: ordered bidirectional streaming for all filer mutations.
+62 -247
View File
@@ -3991,7 +3991,6 @@ type LockResponse struct {
LockOwner string `protobuf:"bytes,2,opt,name=lock_owner,json=lockOwner,proto3" json:"lock_owner,omitempty"`
LockHostMovedTo string `protobuf:"bytes,3,opt,name=lock_host_moved_to,json=lockHostMovedTo,proto3" json:"lock_host_moved_to,omitempty"`
Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
Generation int64 `protobuf:"varint,5,opt,name=generation,proto3" json:"generation,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -4054,13 +4053,6 @@ func (x *LockResponse) GetError() string {
return ""
}
func (x *LockResponse) GetGeneration() int64 {
if x != nil {
return x.Generation
}
return 0
}
type UnlockRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
@@ -4275,9 +4267,6 @@ type Lock struct {
RenewToken string `protobuf:"bytes,2,opt,name=renew_token,json=renewToken,proto3" json:"renew_token,omitempty"`
ExpiredAtNs int64 `protobuf:"varint,3,opt,name=expired_at_ns,json=expiredAtNs,proto3" json:"expired_at_ns,omitempty"`
Owner string `protobuf:"bytes,4,opt,name=owner,proto3" json:"owner,omitempty"`
Generation int64 `protobuf:"varint,5,opt,name=generation,proto3" json:"generation,omitempty"`
IsBackup bool `protobuf:"varint,6,opt,name=is_backup,json=isBackup,proto3" json:"is_backup,omitempty"`
Seq int64 `protobuf:"varint,7,opt,name=seq,proto3" json:"seq,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -4340,27 +4329,6 @@ func (x *Lock) GetOwner() string {
return ""
}
func (x *Lock) GetGeneration() int64 {
if x != nil {
return x.Generation
}
return 0
}
func (x *Lock) GetIsBackup() bool {
if x != nil {
return x.IsBackup
}
return false
}
func (x *Lock) GetSeq() int64 {
if x != nil {
return x.Seq
}
return 0
}
type TransferLocksRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Locks []*Lock `protobuf:"bytes,1,rep,name=locks,proto3" json:"locks,omitempty"`
@@ -4441,134 +4409,6 @@ func (*TransferLocksResponse) Descriptor() ([]byte, []int) {
return file_filer_proto_rawDescGZIP(), []int{66}
}
type ReplicateLockRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
RenewToken string `protobuf:"bytes,2,opt,name=renew_token,json=renewToken,proto3" json:"renew_token,omitempty"`
ExpiredAtNs int64 `protobuf:"varint,3,opt,name=expired_at_ns,json=expiredAtNs,proto3" json:"expired_at_ns,omitempty"`
Owner string `protobuf:"bytes,4,opt,name=owner,proto3" json:"owner,omitempty"`
Generation int64 `protobuf:"varint,5,opt,name=generation,proto3" json:"generation,omitempty"`
IsUnlock bool `protobuf:"varint,6,opt,name=is_unlock,json=isUnlock,proto3" json:"is_unlock,omitempty"`
Seq int64 `protobuf:"varint,7,opt,name=seq,proto3" json:"seq,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ReplicateLockRequest) Reset() {
*x = ReplicateLockRequest{}
mi := &file_filer_proto_msgTypes[67]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ReplicateLockRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ReplicateLockRequest) ProtoMessage() {}
func (x *ReplicateLockRequest) ProtoReflect() protoreflect.Message {
mi := &file_filer_proto_msgTypes[67]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ReplicateLockRequest.ProtoReflect.Descriptor instead.
func (*ReplicateLockRequest) Descriptor() ([]byte, []int) {
return file_filer_proto_rawDescGZIP(), []int{67}
}
func (x *ReplicateLockRequest) GetName() string {
if x != nil {
return x.Name
}
return ""
}
func (x *ReplicateLockRequest) GetRenewToken() string {
if x != nil {
return x.RenewToken
}
return ""
}
func (x *ReplicateLockRequest) GetExpiredAtNs() int64 {
if x != nil {
return x.ExpiredAtNs
}
return 0
}
func (x *ReplicateLockRequest) GetOwner() string {
if x != nil {
return x.Owner
}
return ""
}
func (x *ReplicateLockRequest) GetGeneration() int64 {
if x != nil {
return x.Generation
}
return 0
}
func (x *ReplicateLockRequest) GetIsUnlock() bool {
if x != nil {
return x.IsUnlock
}
return false
}
func (x *ReplicateLockRequest) GetSeq() int64 {
if x != nil {
return x.Seq
}
return 0
}
type ReplicateLockResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ReplicateLockResponse) Reset() {
*x = ReplicateLockResponse{}
mi := &file_filer_proto_msgTypes[68]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ReplicateLockResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ReplicateLockResponse) ProtoMessage() {}
func (x *ReplicateLockResponse) ProtoReflect() protoreflect.Message {
mi := &file_filer_proto_msgTypes[68]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ReplicateLockResponse.ProtoReflect.Descriptor instead.
func (*ReplicateLockResponse) Descriptor() ([]byte, []int) {
return file_filer_proto_rawDescGZIP(), []int{68}
}
type StreamMutateEntryRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId uint64 `protobuf:"varint,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
@@ -4585,7 +4425,7 @@ type StreamMutateEntryRequest struct {
func (x *StreamMutateEntryRequest) Reset() {
*x = StreamMutateEntryRequest{}
mi := &file_filer_proto_msgTypes[69]
mi := &file_filer_proto_msgTypes[67]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -4597,7 +4437,7 @@ func (x *StreamMutateEntryRequest) String() string {
func (*StreamMutateEntryRequest) ProtoMessage() {}
func (x *StreamMutateEntryRequest) ProtoReflect() protoreflect.Message {
mi := &file_filer_proto_msgTypes[69]
mi := &file_filer_proto_msgTypes[67]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -4610,7 +4450,7 @@ func (x *StreamMutateEntryRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use StreamMutateEntryRequest.ProtoReflect.Descriptor instead.
func (*StreamMutateEntryRequest) Descriptor() ([]byte, []int) {
return file_filer_proto_rawDescGZIP(), []int{69}
return file_filer_proto_rawDescGZIP(), []int{67}
}
func (x *StreamMutateEntryRequest) GetRequestId() uint64 {
@@ -4710,7 +4550,7 @@ type StreamMutateEntryResponse struct {
func (x *StreamMutateEntryResponse) Reset() {
*x = StreamMutateEntryResponse{}
mi := &file_filer_proto_msgTypes[70]
mi := &file_filer_proto_msgTypes[68]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -4722,7 +4562,7 @@ func (x *StreamMutateEntryResponse) String() string {
func (*StreamMutateEntryResponse) ProtoMessage() {}
func (x *StreamMutateEntryResponse) ProtoReflect() protoreflect.Message {
mi := &file_filer_proto_msgTypes[70]
mi := &file_filer_proto_msgTypes[68]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -4735,7 +4575,7 @@ func (x *StreamMutateEntryResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use StreamMutateEntryResponse.ProtoReflect.Descriptor instead.
func (*StreamMutateEntryResponse) Descriptor() ([]byte, []int) {
return file_filer_proto_rawDescGZIP(), []int{70}
return file_filer_proto_rawDescGZIP(), []int{68}
}
func (x *StreamMutateEntryResponse) GetRequestId() uint64 {
@@ -4849,7 +4689,7 @@ type LocateBrokerResponse_Resource struct {
func (x *LocateBrokerResponse_Resource) Reset() {
*x = LocateBrokerResponse_Resource{}
mi := &file_filer_proto_msgTypes[74]
mi := &file_filer_proto_msgTypes[72]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -4861,7 +4701,7 @@ func (x *LocateBrokerResponse_Resource) String() string {
func (*LocateBrokerResponse_Resource) ProtoMessage() {}
func (x *LocateBrokerResponse_Resource) ProtoReflect() protoreflect.Message {
mi := &file_filer_proto_msgTypes[74]
mi := &file_filer_proto_msgTypes[72]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -4915,7 +4755,7 @@ type FilerConf_PathConf struct {
func (x *FilerConf_PathConf) Reset() {
*x = FilerConf_PathConf{}
mi := &file_filer_proto_msgTypes[75]
mi := &file_filer_proto_msgTypes[73]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -4927,7 +4767,7 @@ func (x *FilerConf_PathConf) String() string {
func (*FilerConf_PathConf) ProtoMessage() {}
func (x *FilerConf_PathConf) ProtoReflect() protoreflect.Message {
mi := &file_filer_proto_msgTypes[75]
mi := &file_filer_proto_msgTypes[73]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -5418,17 +5258,14 @@ const file_filer_proto_rawDesc = "" +
"\vrenew_token\x18\x03 \x01(\tR\n" +
"renewToken\x12\x19\n" +
"\bis_moved\x18\x04 \x01(\bR\aisMoved\x12\x14\n" +
"\x05owner\x18\x05 \x01(\tR\x05owner\"\xb1\x01\n" +
"\x05owner\x18\x05 \x01(\tR\x05owner\"\x91\x01\n" +
"\fLockResponse\x12\x1f\n" +
"\vrenew_token\x18\x01 \x01(\tR\n" +
"renewToken\x12\x1d\n" +
"\n" +
"lock_owner\x18\x02 \x01(\tR\tlockOwner\x12+\n" +
"\x12lock_host_moved_to\x18\x03 \x01(\tR\x0flockHostMovedTo\x12\x14\n" +
"\x05error\x18\x04 \x01(\tR\x05error\x12\x1e\n" +
"\n" +
"generation\x18\x05 \x01(\x03R\n" +
"generation\"_\n" +
"\x05error\x18\x04 \x01(\tR\x05error\"_\n" +
"\rUnlockRequest\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
"\vrenew_token\x18\x02 \x01(\tR\n" +
@@ -5441,33 +5278,16 @@ const file_filer_proto_rawDesc = "" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" +
"\bis_moved\x18\x02 \x01(\bR\aisMoved\"-\n" +
"\x15FindLockOwnerResponse\x12\x14\n" +
"\x05owner\x18\x01 \x01(\tR\x05owner\"\xc4\x01\n" +
"\x05owner\x18\x01 \x01(\tR\x05owner\"u\n" +
"\x04Lock\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
"\vrenew_token\x18\x02 \x01(\tR\n" +
"renewToken\x12\"\n" +
"\rexpired_at_ns\x18\x03 \x01(\x03R\vexpiredAtNs\x12\x14\n" +
"\x05owner\x18\x04 \x01(\tR\x05owner\x12\x1e\n" +
"\n" +
"generation\x18\x05 \x01(\x03R\n" +
"generation\x12\x1b\n" +
"\tis_backup\x18\x06 \x01(\bR\bisBackup\x12\x10\n" +
"\x03seq\x18\a \x01(\x03R\x03seq\"<\n" +
"\x05owner\x18\x04 \x01(\tR\x05owner\"<\n" +
"\x14TransferLocksRequest\x12$\n" +
"\x05locks\x18\x01 \x03(\v2\x0e.filer_pb.LockR\x05locks\"\x17\n" +
"\x15TransferLocksResponse\"\xd4\x01\n" +
"\x14ReplicateLockRequest\x12\x12\n" +
"\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n" +
"\vrenew_token\x18\x02 \x01(\tR\n" +
"renewToken\x12\"\n" +
"\rexpired_at_ns\x18\x03 \x01(\x03R\vexpiredAtNs\x12\x14\n" +
"\x05owner\x18\x04 \x01(\tR\x05owner\x12\x1e\n" +
"\n" +
"generation\x18\x05 \x01(\x03R\n" +
"generation\x12\x1b\n" +
"\tis_unlock\x18\x06 \x01(\bR\bisUnlock\x12\x10\n" +
"\x03seq\x18\a \x01(\x03R\x03seq\"\x17\n" +
"\x15ReplicateLockResponse\"\xe6\x02\n" +
"\x15TransferLocksResponse\"\xe6\x02\n" +
"\x18StreamMutateEntryRequest\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\x04R\trequestId\x12E\n" +
@@ -5501,7 +5321,7 @@ const file_filer_proto_rawDesc = "" +
"\x0ePARENT_IS_FILE\x10\x02\x12\x19\n" +
"\x15EXISTING_IS_DIRECTORY\x10\x03\x12\x14\n" +
"\x10EXISTING_IS_FILE\x10\x04\x12\x18\n" +
"\x14ENTRY_ALREADY_EXISTS\x10\x052\xaf\x12\n" +
"\x14ENTRY_ALREADY_EXISTS\x10\x052\xdb\x11\n" +
"\fSeaweedFiler\x12g\n" +
"\x14LookupDirectoryEntry\x12%.filer_pb.LookupDirectoryEntryRequest\x1a&.filer_pb.LookupDirectoryEntryResponse\"\x00\x12N\n" +
"\vListEntries\x12\x1c.filer_pb.ListEntriesRequest\x1a\x1d.filer_pb.ListEntriesResponse\"\x000\x01\x12L\n" +
@@ -5529,8 +5349,7 @@ const file_filer_proto_rawDesc = "" +
"\x0fDistributedLock\x12\x15.filer_pb.LockRequest\x1a\x16.filer_pb.LockResponse\"\x00\x12H\n" +
"\x11DistributedUnlock\x12\x17.filer_pb.UnlockRequest\x1a\x18.filer_pb.UnlockResponse\"\x00\x12R\n" +
"\rFindLockOwner\x12\x1e.filer_pb.FindLockOwnerRequest\x1a\x1f.filer_pb.FindLockOwnerResponse\"\x00\x12R\n" +
"\rTransferLocks\x12\x1e.filer_pb.TransferLocksRequest\x1a\x1f.filer_pb.TransferLocksResponse\"\x00\x12R\n" +
"\rReplicateLock\x12\x1e.filer_pb.ReplicateLockRequest\x1a\x1f.filer_pb.ReplicateLockResponse\"\x00BO\n" +
"\rTransferLocks\x12\x1e.filer_pb.TransferLocksRequest\x1a\x1f.filer_pb.TransferLocksResponse\"\x00BO\n" +
"\x10seaweedfs.clientB\n" +
"FilerProtoZ/github.com/seaweedfs/seaweedfs/weed/pb/filer_pbb\x06proto3"
@@ -5547,7 +5366,7 @@ func file_filer_proto_rawDescGZIP() []byte {
}
var file_filer_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 76)
var file_filer_proto_msgTypes = make([]protoimpl.MessageInfo, 74)
var file_filer_proto_goTypes = []any{
(SSEType)(0), // 0: filer_pb.SSEType
(FilerError)(0), // 1: filer_pb.FilerError
@@ -5618,22 +5437,20 @@ var file_filer_proto_goTypes = []any{
(*Lock)(nil), // 66: filer_pb.Lock
(*TransferLocksRequest)(nil), // 67: filer_pb.TransferLocksRequest
(*TransferLocksResponse)(nil), // 68: filer_pb.TransferLocksResponse
(*ReplicateLockRequest)(nil), // 69: filer_pb.ReplicateLockRequest
(*ReplicateLockResponse)(nil), // 70: filer_pb.ReplicateLockResponse
(*StreamMutateEntryRequest)(nil), // 71: filer_pb.StreamMutateEntryRequest
(*StreamMutateEntryResponse)(nil), // 72: filer_pb.StreamMutateEntryResponse
nil, // 73: filer_pb.Entry.ExtendedEntry
nil, // 74: filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
nil, // 75: filer_pb.LookupVolumeResponse.LocationsMapEntry
(*LocateBrokerResponse_Resource)(nil), // 76: filer_pb.LocateBrokerResponse.Resource
(*FilerConf_PathConf)(nil), // 77: filer_pb.FilerConf.PathConf
(*StreamMutateEntryRequest)(nil), // 69: filer_pb.StreamMutateEntryRequest
(*StreamMutateEntryResponse)(nil), // 70: filer_pb.StreamMutateEntryResponse
nil, // 71: filer_pb.Entry.ExtendedEntry
nil, // 72: filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
nil, // 73: filer_pb.LookupVolumeResponse.LocationsMapEntry
(*LocateBrokerResponse_Resource)(nil), // 74: filer_pb.LocateBrokerResponse.Resource
(*FilerConf_PathConf)(nil), // 75: filer_pb.FilerConf.PathConf
}
var file_filer_proto_depIdxs = []int32{
7, // 0: filer_pb.LookupDirectoryEntryResponse.entry:type_name -> filer_pb.Entry
7, // 1: filer_pb.ListEntriesResponse.entry:type_name -> filer_pb.Entry
10, // 2: filer_pb.Entry.chunks:type_name -> filer_pb.FileChunk
13, // 3: filer_pb.Entry.attributes:type_name -> filer_pb.FuseAttributes
73, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
71, // 4: filer_pb.Entry.extended:type_name -> filer_pb.Entry.ExtendedEntry
6, // 5: filer_pb.Entry.remote_entry:type_name -> filer_pb.RemoteEntry
7, // 6: filer_pb.FullEntry.entry:type_name -> filer_pb.Entry
7, // 7: filer_pb.EventNotification.old_entry:type_name -> filer_pb.Entry
@@ -5646,22 +5463,22 @@ var file_filer_proto_depIdxs = []int32{
44, // 14: filer_pb.CreateEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
1, // 15: filer_pb.CreateEntryResponse.error_code:type_name -> filer_pb.FilerError
7, // 16: filer_pb.UpdateEntryRequest.entry:type_name -> filer_pb.Entry
74, // 17: filer_pb.UpdateEntryRequest.expected_extended:type_name -> filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
72, // 17: filer_pb.UpdateEntryRequest.expected_extended:type_name -> filer_pb.UpdateEntryRequest.ExpectedExtendedEntry
44, // 18: filer_pb.UpdateEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
10, // 19: filer_pb.AppendToEntryRequest.chunks:type_name -> filer_pb.FileChunk
44, // 20: filer_pb.DeleteEntryResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
9, // 21: filer_pb.StreamRenameEntryResponse.event_notification:type_name -> filer_pb.EventNotification
30, // 22: filer_pb.AssignVolumeResponse.location:type_name -> filer_pb.Location
30, // 23: filer_pb.Locations.locations:type_name -> filer_pb.Location
75, // 24: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
73, // 24: filer_pb.LookupVolumeResponse.locations_map:type_name -> filer_pb.LookupVolumeResponse.LocationsMapEntry
32, // 25: filer_pb.CollectionListResponse.collections:type_name -> filer_pb.Collection
9, // 26: filer_pb.SubscribeMetadataResponse.event_notification:type_name -> filer_pb.EventNotification
44, // 27: filer_pb.SubscribeMetadataResponse.events:type_name -> filer_pb.SubscribeMetadataResponse
45, // 28: filer_pb.SubscribeMetadataResponse.log_file_refs:type_name -> filer_pb.LogFileChunkRef
10, // 29: filer_pb.LogFileChunkRef.chunks:type_name -> filer_pb.FileChunk
7, // 30: filer_pb.TraverseBfsMetadataResponse.entry:type_name -> filer_pb.Entry
76, // 31: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
77, // 32: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
74, // 31: filer_pb.LocateBrokerResponse.resources:type_name -> filer_pb.LocateBrokerResponse.Resource
75, // 32: filer_pb.FilerConf.locations:type_name -> filer_pb.FilerConf.PathConf
7, // 33: filer_pb.CacheRemoteObjectToLocalClusterResponse.entry:type_name -> filer_pb.Entry
44, // 34: filer_pb.CacheRemoteObjectToLocalClusterResponse.metadata_event:type_name -> filer_pb.SubscribeMetadataResponse
66, // 35: filer_pb.TransferLocksRequest.locks:type_name -> filer_pb.Lock
@@ -5682,7 +5499,7 @@ var file_filer_proto_depIdxs = []int32{
20, // 50: filer_pb.SeaweedFiler.DeleteEntry:input_type -> filer_pb.DeleteEntryRequest
22, // 51: filer_pb.SeaweedFiler.AtomicRenameEntry:input_type -> filer_pb.AtomicRenameEntryRequest
24, // 52: filer_pb.SeaweedFiler.StreamRenameEntry:input_type -> filer_pb.StreamRenameEntryRequest
71, // 53: filer_pb.SeaweedFiler.StreamMutateEntry:input_type -> filer_pb.StreamMutateEntryRequest
69, // 53: filer_pb.SeaweedFiler.StreamMutateEntry:input_type -> filer_pb.StreamMutateEntryRequest
26, // 54: filer_pb.SeaweedFiler.AssignVolume:input_type -> filer_pb.AssignVolumeRequest
28, // 55: filer_pb.SeaweedFiler.LookupVolume:input_type -> filer_pb.LookupVolumeRequest
33, // 56: filer_pb.SeaweedFiler.CollectionList:input_type -> filer_pb.CollectionListRequest
@@ -5700,36 +5517,34 @@ var file_filer_proto_depIdxs = []int32{
62, // 68: filer_pb.SeaweedFiler.DistributedUnlock:input_type -> filer_pb.UnlockRequest
64, // 69: filer_pb.SeaweedFiler.FindLockOwner:input_type -> filer_pb.FindLockOwnerRequest
67, // 70: filer_pb.SeaweedFiler.TransferLocks:input_type -> filer_pb.TransferLocksRequest
69, // 71: filer_pb.SeaweedFiler.ReplicateLock:input_type -> filer_pb.ReplicateLockRequest
3, // 72: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
5, // 73: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
15, // 74: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
17, // 75: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
19, // 76: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
21, // 77: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
23, // 78: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
25, // 79: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse
72, // 80: filer_pb.SeaweedFiler.StreamMutateEntry:output_type -> filer_pb.StreamMutateEntryResponse
27, // 81: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
31, // 82: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
34, // 83: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
36, // 84: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
38, // 85: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
40, // 86: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse
42, // 87: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
47, // 88: filer_pb.SeaweedFiler.TraverseBfsMetadata:output_type -> filer_pb.TraverseBfsMetadataResponse
44, // 89: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
44, // 90: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
54, // 91: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
56, // 92: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
59, // 93: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse
61, // 94: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse
63, // 95: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse
65, // 96: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse
68, // 97: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse
70, // 98: filer_pb.SeaweedFiler.ReplicateLock:output_type -> filer_pb.ReplicateLockResponse
72, // [72:99] is the sub-list for method output_type
45, // [45:72] is the sub-list for method input_type
3, // 71: filer_pb.SeaweedFiler.LookupDirectoryEntry:output_type -> filer_pb.LookupDirectoryEntryResponse
5, // 72: filer_pb.SeaweedFiler.ListEntries:output_type -> filer_pb.ListEntriesResponse
15, // 73: filer_pb.SeaweedFiler.CreateEntry:output_type -> filer_pb.CreateEntryResponse
17, // 74: filer_pb.SeaweedFiler.UpdateEntry:output_type -> filer_pb.UpdateEntryResponse
19, // 75: filer_pb.SeaweedFiler.AppendToEntry:output_type -> filer_pb.AppendToEntryResponse
21, // 76: filer_pb.SeaweedFiler.DeleteEntry:output_type -> filer_pb.DeleteEntryResponse
23, // 77: filer_pb.SeaweedFiler.AtomicRenameEntry:output_type -> filer_pb.AtomicRenameEntryResponse
25, // 78: filer_pb.SeaweedFiler.StreamRenameEntry:output_type -> filer_pb.StreamRenameEntryResponse
70, // 79: filer_pb.SeaweedFiler.StreamMutateEntry:output_type -> filer_pb.StreamMutateEntryResponse
27, // 80: filer_pb.SeaweedFiler.AssignVolume:output_type -> filer_pb.AssignVolumeResponse
31, // 81: filer_pb.SeaweedFiler.LookupVolume:output_type -> filer_pb.LookupVolumeResponse
34, // 82: filer_pb.SeaweedFiler.CollectionList:output_type -> filer_pb.CollectionListResponse
36, // 83: filer_pb.SeaweedFiler.DeleteCollection:output_type -> filer_pb.DeleteCollectionResponse
38, // 84: filer_pb.SeaweedFiler.Statistics:output_type -> filer_pb.StatisticsResponse
40, // 85: filer_pb.SeaweedFiler.Ping:output_type -> filer_pb.PingResponse
42, // 86: filer_pb.SeaweedFiler.GetFilerConfiguration:output_type -> filer_pb.GetFilerConfigurationResponse
47, // 87: filer_pb.SeaweedFiler.TraverseBfsMetadata:output_type -> filer_pb.TraverseBfsMetadataResponse
44, // 88: filer_pb.SeaweedFiler.SubscribeMetadata:output_type -> filer_pb.SubscribeMetadataResponse
44, // 89: filer_pb.SeaweedFiler.SubscribeLocalMetadata:output_type -> filer_pb.SubscribeMetadataResponse
54, // 90: filer_pb.SeaweedFiler.KvGet:output_type -> filer_pb.KvGetResponse
56, // 91: filer_pb.SeaweedFiler.KvPut:output_type -> filer_pb.KvPutResponse
59, // 92: filer_pb.SeaweedFiler.CacheRemoteObjectToLocalCluster:output_type -> filer_pb.CacheRemoteObjectToLocalClusterResponse
61, // 93: filer_pb.SeaweedFiler.DistributedLock:output_type -> filer_pb.LockResponse
63, // 94: filer_pb.SeaweedFiler.DistributedUnlock:output_type -> filer_pb.UnlockResponse
65, // 95: filer_pb.SeaweedFiler.FindLockOwner:output_type -> filer_pb.FindLockOwnerResponse
68, // 96: filer_pb.SeaweedFiler.TransferLocks:output_type -> filer_pb.TransferLocksResponse
71, // [71:97] is the sub-list for method output_type
45, // [45:71] is the sub-list for method input_type
45, // [45:45] is the sub-list for extension type_name
45, // [45:45] is the sub-list for extension extendee
0, // [0:45] is the sub-list for field type_name
@@ -5740,13 +5555,13 @@ func file_filer_proto_init() {
if File_filer_proto != nil {
return
}
file_filer_proto_msgTypes[69].OneofWrappers = []any{
file_filer_proto_msgTypes[67].OneofWrappers = []any{
(*StreamMutateEntryRequest_CreateRequest)(nil),
(*StreamMutateEntryRequest_UpdateRequest)(nil),
(*StreamMutateEntryRequest_DeleteRequest)(nil),
(*StreamMutateEntryRequest_RenameRequest)(nil),
}
file_filer_proto_msgTypes[70].OneofWrappers = []any{
file_filer_proto_msgTypes[68].OneofWrappers = []any{
(*StreamMutateEntryResponse_CreateResponse)(nil),
(*StreamMutateEntryResponse_UpdateResponse)(nil),
(*StreamMutateEntryResponse_DeleteResponse)(nil),
@@ -5758,7 +5573,7 @@ func file_filer_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_filer_proto_rawDesc), len(file_filer_proto_rawDesc)),
NumEnums: 2,
NumMessages: 76,
NumMessages: 74,
NumExtensions: 0,
NumServices: 1,
},
-38
View File
@@ -45,7 +45,6 @@ const (
SeaweedFiler_DistributedUnlock_FullMethodName = "/filer_pb.SeaweedFiler/DistributedUnlock"
SeaweedFiler_FindLockOwner_FullMethodName = "/filer_pb.SeaweedFiler/FindLockOwner"
SeaweedFiler_TransferLocks_FullMethodName = "/filer_pb.SeaweedFiler/TransferLocks"
SeaweedFiler_ReplicateLock_FullMethodName = "/filer_pb.SeaweedFiler/ReplicateLock"
)
// SeaweedFilerClient is the client API for SeaweedFiler service.
@@ -79,7 +78,6 @@ type SeaweedFilerClient interface {
FindLockOwner(ctx context.Context, in *FindLockOwnerRequest, opts ...grpc.CallOption) (*FindLockOwnerResponse, error)
// distributed lock management internal use only
TransferLocks(ctx context.Context, in *TransferLocksRequest, opts ...grpc.CallOption) (*TransferLocksResponse, error)
ReplicateLock(ctx context.Context, in *ReplicateLockRequest, opts ...grpc.CallOption) (*ReplicateLockResponse, error)
}
type seaweedFilerClient struct {
@@ -398,16 +396,6 @@ func (c *seaweedFilerClient) TransferLocks(ctx context.Context, in *TransferLock
return out, nil
}
func (c *seaweedFilerClient) ReplicateLock(ctx context.Context, in *ReplicateLockRequest, opts ...grpc.CallOption) (*ReplicateLockResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ReplicateLockResponse)
err := c.cc.Invoke(ctx, SeaweedFiler_ReplicateLock_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// SeaweedFilerServer is the server API for SeaweedFiler service.
// All implementations must embed UnimplementedSeaweedFilerServer
// for forward compatibility.
@@ -439,7 +427,6 @@ type SeaweedFilerServer interface {
FindLockOwner(context.Context, *FindLockOwnerRequest) (*FindLockOwnerResponse, error)
// distributed lock management internal use only
TransferLocks(context.Context, *TransferLocksRequest) (*TransferLocksResponse, error)
ReplicateLock(context.Context, *ReplicateLockRequest) (*ReplicateLockResponse, error)
mustEmbedUnimplementedSeaweedFilerServer()
}
@@ -528,9 +515,6 @@ func (UnimplementedSeaweedFilerServer) FindLockOwner(context.Context, *FindLockO
func (UnimplementedSeaweedFilerServer) TransferLocks(context.Context, *TransferLocksRequest) (*TransferLocksResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method TransferLocks not implemented")
}
func (UnimplementedSeaweedFilerServer) ReplicateLock(context.Context, *ReplicateLockRequest) (*ReplicateLockResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReplicateLock not implemented")
}
func (UnimplementedSeaweedFilerServer) mustEmbedUnimplementedSeaweedFilerServer() {}
func (UnimplementedSeaweedFilerServer) testEmbeddedByValue() {}
@@ -974,24 +958,6 @@ func _SeaweedFiler_TransferLocks_Handler(srv interface{}, ctx context.Context, d
return interceptor(ctx, in, info, handler)
}
func _SeaweedFiler_ReplicateLock_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReplicateLockRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SeaweedFilerServer).ReplicateLock(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: SeaweedFiler_ReplicateLock_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedFilerServer).ReplicateLock(ctx, req.(*ReplicateLockRequest))
}
return interceptor(ctx, in, info, handler)
}
// SeaweedFiler_ServiceDesc is the grpc.ServiceDesc for SeaweedFiler service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -1079,10 +1045,6 @@ var SeaweedFiler_ServiceDesc = grpc.ServiceDesc{
MethodName: "TransferLocks",
Handler: _SeaweedFiler_TransferLocks_Handler,
},
{
MethodName: "ReplicateLock",
Handler: _SeaweedFiler_ReplicateLock_Handler,
},
},
Streams: []grpc.StreamDesc{
{
-114
View File
@@ -11,7 +11,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/viant/ptrie"
"google.golang.org/protobuf/proto"
)
@@ -248,119 +247,6 @@ func IsRename(event *SubscribeMetadataResponse) bool {
event.EventNotification.NewEntry.Name != event.EventNotification.OldEntry.Name)
}
func MetadataEventSourceDirectory(event *SubscribeMetadataResponse) string {
if event == nil {
return ""
}
return event.Directory
}
func MetadataEventTargetDirectory(event *SubscribeMetadataResponse) string {
if event == nil {
return ""
}
if event.EventNotification != nil && event.EventNotification.NewParentPath != "" {
return event.EventNotification.NewParentPath
}
return event.Directory
}
func metadataEventSourceEntryName(event *SubscribeMetadataResponse) string {
if event == nil || event.EventNotification == nil {
return ""
}
if event.EventNotification.OldEntry != nil {
return event.EventNotification.OldEntry.Name
}
if event.EventNotification.NewEntry != nil {
return event.EventNotification.NewEntry.Name
}
return ""
}
func metadataEventTargetEntryName(event *SubscribeMetadataResponse) string {
if event == nil || event.EventNotification == nil {
return ""
}
if event.EventNotification.NewEntry != nil {
return event.EventNotification.NewEntry.Name
}
if event.EventNotification.OldEntry != nil {
return event.EventNotification.OldEntry.Name
}
return ""
}
func MetadataEventSourceFullPath(event *SubscribeMetadataResponse) string {
return util.Join(MetadataEventSourceDirectory(event), metadataEventSourceEntryName(event))
}
func MetadataEventTargetFullPath(event *SubscribeMetadataResponse) string {
return util.Join(MetadataEventTargetDirectory(event), metadataEventTargetEntryName(event))
}
func MetadataEventTouchesDirectory(event *SubscribeMetadataResponse, dir string) bool {
if MetadataEventSourceDirectory(event) == dir {
return true
}
return event != nil &&
event.EventNotification != nil &&
event.EventNotification.NewEntry != nil &&
MetadataEventTargetDirectory(event) == dir
}
func MetadataEventTouchesDirectoryPrefix(event *SubscribeMetadataResponse, prefix string) bool {
if strings.HasPrefix(MetadataEventSourceDirectory(event), prefix) {
return true
}
return event != nil &&
event.EventNotification != nil &&
event.EventNotification.NewEntry != nil &&
strings.HasPrefix(MetadataEventTargetDirectory(event), prefix)
}
func MetadataEventMatchesSubscription(event *SubscribeMetadataResponse, pathPrefix string, pathPrefixes []string, directories []string) bool {
if event == nil {
return false
}
if metadataEventMatchesPath(MetadataEventSourceFullPath(event), MetadataEventSourceDirectory(event), pathPrefix, pathPrefixes, directories) {
return true
}
return event.EventNotification != nil &&
event.EventNotification.NewEntry != nil &&
metadataEventMatchesPath(MetadataEventTargetFullPath(event), MetadataEventTargetDirectory(event), pathPrefix, pathPrefixes, directories)
}
func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes []string, directories []string) bool {
if hasPrefixIn(fullPath, pathPrefixes) {
return true
}
if matchByDirectory(dirPath, directories) {
return true
}
return strings.HasPrefix(fullPath, pathPrefix)
}
func hasPrefixIn(text string, prefixes []string) bool {
for _, p := range prefixes {
if strings.HasPrefix(text, p) {
return true
}
}
return false
}
func matchByDirectory(dirPath string, directories []string) bool {
for _, dir := range directories {
if dirPath == dir {
return true
}
}
return false
}
var _ = ptrie.KeyProvider(&FilerConf_PathConf{})
func (fp *FilerConf_PathConf) Key() interface{} {
-70
View File
@@ -15,73 +15,3 @@ func TestFileIdSize(t *testing.T) {
println(len(fileIdStr))
println(len(bytes))
}
func TestMetadataEventMatchesSubscription(t *testing.T) {
event := &SubscribeMetadataResponse{
Directory: "/tmp",
EventNotification: &EventNotification{
OldEntry: &Entry{Name: "old-name"},
NewEntry: &Entry{Name: "new-name"},
NewParentPath: "/watched",
},
}
tests := []struct {
name string
pathPrefix string
pathPrefixes []string
directories []string
}{
{
name: "primary path prefix matches rename target",
pathPrefix: "/watched/new-name",
},
{
name: "additional path prefix matches rename target",
pathPrefix: "/data",
pathPrefixes: []string{"/watched"},
},
{
name: "directory watch matches rename target directory",
pathPrefix: "/data",
directories: []string{"/watched"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if !MetadataEventMatchesSubscription(event, tt.pathPrefix, tt.pathPrefixes, tt.directories) {
t.Fatalf("MetadataEventMatchesSubscription returned false")
}
})
}
}
func TestMetadataEventTouchesDirectoryHelpers(t *testing.T) {
renameInto := &SubscribeMetadataResponse{
Directory: "/tmp",
EventNotification: &EventNotification{
OldEntry: &Entry{Name: "filer.conf"},
NewEntry: &Entry{Name: "filer.conf"},
NewParentPath: "/etc/seaweedfs",
},
}
if got := MetadataEventTargetDirectory(renameInto); got != "/etc/seaweedfs" {
t.Fatalf("MetadataEventTargetDirectory = %q, want /etc/seaweedfs", got)
}
if !MetadataEventTouchesDirectory(renameInto, "/etc/seaweedfs") {
t.Fatalf("expected rename target to touch /etc/seaweedfs")
}
renameOut := &SubscribeMetadataResponse{
Directory: "/etc/remote",
EventNotification: &EventNotification{
OldEntry: &Entry{Name: "remote.conf"},
NewEntry: &Entry{Name: "remote.conf"},
NewParentPath: "/tmp",
},
}
if !MetadataEventTouchesDirectoryPrefix(renameOut, "/etc/remote") {
t.Fatalf("expected rename source to touch /etc/remote")
}
}
+45 -6
View File
@@ -255,14 +255,53 @@ func processOneLogEntry(logEntry *filer_pb.LogEntry, filter PathFilter, processE
const systemLogDir = "/topics/.system/log"
func matchesFilter(resp *filer_pb.SubscribeMetadataResponse, filter PathFilter) bool {
fullpath := filer_pb.MetadataEventSourceFullPath(resp)
var entryName string
if resp.EventNotification != nil {
if resp.EventNotification.OldEntry != nil {
entryName = resp.EventNotification.OldEntry.Name
} else if resp.EventNotification.NewEntry != nil {
entryName = resp.EventNotification.NewEntry.Name
}
}
fullpath := util.Join(resp.Directory, entryName)
// Skip internal meta log entries
if strings.HasPrefix(fullpath, systemLogDir) {
return false
}
return filer_pb.MetadataEventMatchesSubscription(resp, filter.PathPrefix, filter.AdditionalPathPrefixes, filter.DirectoriesToWatch)
// Check AdditionalPathPrefixes
for _, p := range filter.AdditionalPathPrefixes {
if strings.HasPrefix(fullpath, p) {
return true
}
}
// Check DirectoriesToWatch (exact directory match)
for _, dir := range filter.DirectoriesToWatch {
if resp.Directory == dir {
return true
}
}
// Check primary PathPrefix
if filter.PathPrefix == "" || filter.PathPrefix == "/" {
return true
}
if strings.HasPrefix(fullpath, filter.PathPrefix) {
return true
}
// Check rename target
if resp.EventNotification != nil && resp.EventNotification.NewParentPath != "" {
newFullPath := util.Join(resp.EventNotification.NewParentPath, entryName)
if strings.HasPrefix(newFullPath, filter.PathPrefix) {
return true
}
}
return false
}
// isChunkNotFound checks if an error indicates a missing volume chunk.
@@ -284,10 +323,10 @@ type logEntryHeapItem struct {
type logEntryHeap []*logEntryHeapItem
func (h logEntryHeap) Len() int { return len(h) }
func (h logEntryHeap) Less(i, j int) bool { return h[i].entry.TsNs < h[j].entry.TsNs }
func (h logEntryHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *logEntryHeap) Push(x any) { *h = append(*h, x.(*logEntryHeapItem)) }
func (h logEntryHeap) Len() int { return len(h) }
func (h logEntryHeap) Less(i, j int) bool { return h[i].entry.TsNs < h[j].entry.TsNs }
func (h logEntryHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h *logEntryHeap) Push(x any) { *h = append(*h, x.(*logEntryHeapItem)) }
func (h *logEntryHeap) Pop() any {
old := *h
n := len(old)
-69
View File
@@ -4,9 +4,7 @@ import (
"context"
"fmt"
"math/rand/v2"
"net"
"net/http"
"os"
"strconv"
"strings"
"sync"
@@ -44,12 +42,6 @@ var (
// cache grpc connections
grpcClients = make(map[string]*versionedGrpcClient)
grpcClientsLock sync.Mutex
// localGrpcSockets maps gRPC port numbers to Unix socket paths.
// When registered (by mini mode), gRPC clients connect via Unix socket
// instead of TCP for local services.
localGrpcSockets = make(map[int]string)
localGrpcSocketsLock sync.RWMutex
)
type versionedGrpcClient struct {
@@ -63,59 +55,6 @@ func init() {
http.DefaultTransport.(*http.Transport).MaxIdleConns = 1024
}
// RegisterLocalGrpcSocket registers a Unix socket path for a gRPC port.
// When a gRPC client dials an address on this port, it uses the Unix socket.
func RegisterLocalGrpcSocket(grpcPort int, socketPath string) {
localGrpcSocketsLock.Lock()
defer localGrpcSocketsLock.Unlock()
localGrpcSockets[grpcPort] = socketPath
}
// GetLocalGrpcSocket returns the Unix socket path for a gRPC port, or empty if not registered.
func GetLocalGrpcSocket(grpcPort int) string {
localGrpcSocketsLock.RLock()
defer localGrpcSocketsLock.RUnlock()
return localGrpcSockets[grpcPort]
}
// resolveLocalGrpcSocket extracts the port from a gRPC address and returns
// the registered Unix socket path, if any.
func resolveLocalGrpcSocket(address string) string {
_, portStr, err := net.SplitHostPort(address)
if err != nil {
return ""
}
port, err := strconv.Atoi(portStr)
if err != nil {
return ""
}
return GetLocalGrpcSocket(port)
}
// ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket
// if one is registered for the given port.
func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) {
socketPath := GetLocalGrpcSocket(grpcPort)
if socketPath == "" {
return
}
if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err)
}
listener, err := net.Listen("unix", socketPath)
if err != nil {
glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err)
return
}
glog.V(0).Infof("gRPC also listening on Unix socket %s", socketPath)
go func() {
if err := grpcServer.Serve(listener); err != nil && err != grpc.ErrServerStopped {
glog.Errorf("gRPC Unix socket server error on %s: %v", socketPath, err)
}
os.Remove(socketPath)
}()
}
func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
var options []grpc.ServerOption
options = append(options,
@@ -158,14 +97,6 @@ func NewGrpcServer(opts ...grpc.ServerOption) *grpc.Server {
func GrpcDial(ctx context.Context, address string, waitForReady bool, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
var options []grpc.DialOption
// Route through Unix socket if one is registered for this address's port
if socketPath := resolveLocalGrpcSocket(address); socketPath != "" {
options = append(options, grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "unix", socketPath)
}))
}
options = append(options,
grpc.WithDefaultCallOptions(
grpc.MaxCallSendMsgSize(Max_Message_Size),
-11
View File
@@ -197,17 +197,6 @@ message ClusterNodeUpdate {
message KeepConnectedResponse {
VolumeLocation volume_location = 1;
ClusterNodeUpdate cluster_node_update = 2;
LockRingUpdate lock_ring_update = 3;
}
// LockRingUpdate is sent by the master to all filers when the lock ring
// membership changes. The master batches rapid changes (e.g., node drop + join)
// and sends the complete member list atomically, avoiding intermediate ring
// states that would cause unnecessary lock churn.
message LockRingUpdate {
string filer_group = 1;
repeated string servers = 2;
int64 version = 3;
}
message LookupVolumeRequest {
File diff suppressed because it is too large Load Diff
+76
View File
@@ -2,6 +2,7 @@ package engine
import (
"context"
"encoding/binary"
"fmt"
"io"
"strings"
@@ -17,6 +18,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
jsonpb "google.golang.org/protobuf/encoding/protojson"
@@ -507,3 +509,77 @@ func (c *BrokerClient) GetUnflushedMessages(ctx context.Context, namespace, topi
return logEntries, nil
}
// getEarliestBufferStart finds the earliest buffer_start index from disk files in the partition
//
// This method handles three scenarios for seamless broker querying:
// 1. Live log files exist: Uses their buffer_start metadata (most recent boundaries)
// 2. Only Parquet files exist: Uses Parquet buffer_start metadata (preserved from archived sources)
// 3. Mixed files: Uses earliest buffer_start from all sources for comprehensive coverage
//
// This ensures continuous real-time querying capability even after log file compaction/archival
func (c *BrokerClient) getEarliestBufferStart(ctx context.Context, partitionPath string) (int64, error) {
filerClient, err := c.GetFilerClient()
if err != nil {
return 0, fmt.Errorf("failed to get filer client: %v", err)
}
var earliestBufferIndex int64 = -1 // -1 means no buffer_start found
var logFileCount, parquetFileCount int
var bufferStartSources []string // Track which files provide buffer_start
err = filer_pb.ReadDirAllEntries(ctx, filerClient, util.FullPath(partitionPath), "", func(entry *filer_pb.Entry, isLast bool) error {
// Skip directories
if entry.IsDirectory {
return nil
}
// Count file types for scenario detection
if strings.HasSuffix(entry.Name, ".parquet") {
parquetFileCount++
} else {
logFileCount++
}
// Extract buffer_start from file extended attributes (both log files and parquet files)
bufferStart := c.getBufferStartFromEntry(entry)
if bufferStart != nil && bufferStart.StartIndex > 0 {
if earliestBufferIndex == -1 || bufferStart.StartIndex < earliestBufferIndex {
earliestBufferIndex = bufferStart.StartIndex
}
bufferStartSources = append(bufferStartSources, entry.Name)
}
return nil
})
if err != nil {
return 0, fmt.Errorf("failed to scan partition directory: %v", err)
}
if earliestBufferIndex == -1 {
return 0, fmt.Errorf("no buffer_start metadata found in partition")
}
return earliestBufferIndex, nil
}
// getBufferStartFromEntry extracts LogBufferStart from file entry metadata
// Only supports binary format (used by both log files and Parquet files)
func (c *BrokerClient) getBufferStartFromEntry(entry *filer_pb.Entry) *LogBufferStart {
if entry.Extended == nil {
return nil
}
if startData, exists := entry.Extended["buffer_start"]; exists {
// Only support binary format
if len(startData) == 8 {
startIndex := int64(binary.BigEndian.Uint64(startData))
if startIndex > 0 {
return &LogBufferStart{StartIndex: startIndex}
}
}
}
return nil
}
+21
View File
@@ -170,6 +170,27 @@ func (e *SQLEngine) convertRawValueToSchemaValue(rawValue interface{}) *schema_p
}
}
// convertJSONValueToSchemaValue converts JSON values to schema_pb.Value
func (e *SQLEngine) convertJSONValueToSchemaValue(jsonValue interface{}) *schema_pb.Value {
switch v := jsonValue.(type) {
case string:
return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: v}}
case float64:
// JSON numbers are always float64, try to detect if it's actually an integer
if v == float64(int64(v)) {
return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: int64(v)}}
}
return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: v}}
case bool:
return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: v}}
case nil:
return nil
default:
// Convert other types to string
return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: fmt.Sprintf("%v", v)}}
}
}
// Helper functions for aggregation processing
// isNullValue checks if a schema_pb.Value is null or empty
+355
View File
@@ -2175,6 +2175,361 @@ func (e *SQLEngine) executeRegularSelectWithHybridScanner(ctx context.Context, h
return e.ConvertToSQLResultWithExpressions(hybridScanner, results, stmt.SelectExprs), nil
}
// executeSelectStatementWithBrokerStats handles SELECT queries with broker buffer statistics capture
// This is used by EXPLAIN queries to capture complete data source information including broker memory
func (e *SQLEngine) executeSelectStatementWithBrokerStats(ctx context.Context, stmt *SelectStatement, plan *QueryExecutionPlan) (*QueryResult, error) {
// Parse FROM clause to get table (topic) information
if len(stmt.From) != 1 {
err := fmt.Errorf("SELECT supports single table queries only")
return &QueryResult{Error: err}, err
}
// Extract table reference
var database, tableName string
switch table := stmt.From[0].(type) {
case *AliasedTableExpr:
switch tableExpr := table.Expr.(type) {
case TableName:
tableName = tableExpr.Name.String()
if tableExpr.Qualifier != nil && tableExpr.Qualifier.String() != "" {
database = tableExpr.Qualifier.String()
}
default:
err := fmt.Errorf("unsupported table expression: %T", tableExpr)
return &QueryResult{Error: err}, err
}
default:
err := fmt.Errorf("unsupported FROM clause: %T", table)
return &QueryResult{Error: err}, err
}
// Use current database context if not specified
if database == "" {
database = e.catalog.GetCurrentDatabase()
if database == "" {
database = "default"
}
}
// Auto-discover and register topic if not already in catalog
if _, err := e.catalog.GetTableInfo(database, tableName); err != nil {
// Topic not in catalog, try to discover and register it
if regErr := e.discoverAndRegisterTopic(ctx, database, tableName); regErr != nil {
// Return error immediately for non-existent topics instead of falling back to sample data
return &QueryResult{Error: regErr}, regErr
}
}
// Create HybridMessageScanner for the topic (reads both live logs + Parquet files)
// Get filerClient from broker connection (works with both real and mock brokers)
var filerClient filer_pb.FilerClient
var filerClientErr error
filerClient, filerClientErr = e.catalog.brokerClient.GetFilerClient()
if filerClientErr != nil {
// Return error if filer client is not available for topic access
return &QueryResult{Error: filerClientErr}, filerClientErr
}
hybridScanner, err := NewHybridMessageScanner(filerClient, e.catalog.brokerClient, database, tableName, e)
if err != nil {
// Handle quiet topics gracefully: topics exist but have no active schema/brokers
if IsNoSchemaError(err) {
// Return empty result for quiet topics (normal in production environments)
return &QueryResult{
Columns: []string{},
Rows: [][]sqltypes.Value{},
Database: database,
Table: tableName,
}, nil
}
// Return error for other access issues (truly non-existent topics, etc.)
topicErr := fmt.Errorf("failed to access topic %s.%s: %v", database, tableName, err)
return &QueryResult{Error: topicErr}, topicErr
}
// Parse SELECT columns and detect aggregation functions
var columns []string
var aggregations []AggregationSpec
selectAll := false
hasAggregations := false
_ = hasAggregations // Used later in aggregation routing
// Track required base columns for arithmetic expressions
baseColumnsSet := make(map[string]bool)
for _, selectExpr := range stmt.SelectExprs {
switch expr := selectExpr.(type) {
case *StarExpr:
selectAll = true
case *AliasedExpr:
switch col := expr.Expr.(type) {
case *ColName:
colName := col.Name.String()
columns = append(columns, colName)
baseColumnsSet[colName] = true
case *ArithmeticExpr:
// Handle arithmetic expressions like id+user_id and string concatenation like name||suffix
columns = append(columns, e.getArithmeticExpressionAlias(col))
// Extract base columns needed for this arithmetic expression
e.extractBaseColumns(col, baseColumnsSet)
case *SQLVal:
// Handle string/numeric literals like 'good', 123, etc.
columns = append(columns, e.getSQLValAlias(col))
case *FuncExpr:
// Distinguish between aggregation functions and string functions
funcName := strings.ToUpper(col.Name.String())
if e.isAggregationFunction(funcName) {
// Handle aggregation functions
aggSpec, err := e.parseAggregationFunction(col, expr)
if err != nil {
return &QueryResult{Error: err}, err
}
aggregations = append(aggregations, *aggSpec)
hasAggregations = true
} else if e.isStringFunction(funcName) {
// Handle string functions like UPPER, LENGTH, etc.
columns = append(columns, e.getStringFunctionAlias(col))
// Extract base columns needed for this string function
e.extractBaseColumnsFromFunction(col, baseColumnsSet)
} else if e.isDateTimeFunction(funcName) {
// Handle datetime functions like CURRENT_DATE, NOW, EXTRACT, DATE_TRUNC
columns = append(columns, e.getDateTimeFunctionAlias(col))
// Extract base columns needed for this datetime function
e.extractBaseColumnsFromFunction(col, baseColumnsSet)
} else {
return &QueryResult{Error: fmt.Errorf("unsupported function: %s", funcName)}, fmt.Errorf("unsupported function: %s", funcName)
}
default:
err := fmt.Errorf("unsupported SELECT expression: %T", col)
return &QueryResult{Error: err}, err
}
default:
err := fmt.Errorf("unsupported SELECT expression: %T", expr)
return &QueryResult{Error: err}, err
}
}
// If we have aggregations, use aggregation query path
if hasAggregations {
return e.executeAggregationQuery(ctx, hybridScanner, aggregations, stmt)
}
// Parse WHERE clause for predicate pushdown
var predicate func(*schema_pb.RecordValue) bool
if stmt.Where != nil {
predicate, err = e.buildPredicateWithContext(stmt.Where.Expr, stmt.SelectExprs)
if err != nil {
return &QueryResult{Error: err}, err
}
}
// Parse LIMIT and OFFSET clauses
// Use -1 to distinguish "no LIMIT" from "LIMIT 0"
limit := -1
offset := 0
if stmt.Limit != nil && stmt.Limit.Rowcount != nil {
switch limitExpr := stmt.Limit.Rowcount.(type) {
case *SQLVal:
if limitExpr.Type == IntVal {
var parseErr error
limit64, parseErr := strconv.ParseInt(string(limitExpr.Val), 10, 64)
if parseErr != nil {
return &QueryResult{Error: parseErr}, parseErr
}
if limit64 > math.MaxInt32 || limit64 < 0 {
return &QueryResult{Error: fmt.Errorf("LIMIT value %d is out of valid range", limit64)}, fmt.Errorf("LIMIT value %d is out of valid range", limit64)
}
limit = int(limit64)
}
}
}
// Parse OFFSET clause if present
if stmt.Limit != nil && stmt.Limit.Offset != nil {
switch offsetExpr := stmt.Limit.Offset.(type) {
case *SQLVal:
if offsetExpr.Type == IntVal {
var parseErr error
offset64, parseErr := strconv.ParseInt(string(offsetExpr.Val), 10, 64)
if parseErr != nil {
return &QueryResult{Error: parseErr}, parseErr
}
if offset64 > math.MaxInt32 || offset64 < 0 {
return &QueryResult{Error: fmt.Errorf("OFFSET value %d is out of valid range", offset64)}, fmt.Errorf("OFFSET value %d is out of valid range", offset64)
}
offset = int(offset64)
}
}
}
// Build hybrid scan options
// Extract time filters from WHERE clause to optimize scanning
startTimeNs, stopTimeNs := int64(0), int64(0)
if stmt.Where != nil {
startTimeNs, stopTimeNs = e.extractTimeFilters(stmt.Where.Expr)
}
hybridScanOptions := HybridScanOptions{
StartTimeNs: startTimeNs, // Extracted from WHERE clause time comparisons
StopTimeNs: stopTimeNs, // Extracted from WHERE clause time comparisons
Limit: limit,
Offset: offset,
Predicate: predicate,
}
if !selectAll {
// Convert baseColumnsSet to slice for hybrid scan options
baseColumns := make([]string, 0, len(baseColumnsSet))
for columnName := range baseColumnsSet {
baseColumns = append(baseColumns, columnName)
}
// Use base columns (not expression aliases) for data retrieval
if len(baseColumns) > 0 {
hybridScanOptions.Columns = baseColumns
} else {
// If no base columns found (shouldn't happen), use original columns
hybridScanOptions.Columns = columns
}
}
// Execute the hybrid scan with stats capture for EXPLAIN
var results []HybridScanResult
if plan != nil {
// EXPLAIN mode - capture broker buffer stats
var stats *HybridScanStats
results, stats, err = hybridScanner.ScanWithStats(ctx, hybridScanOptions)
if err != nil {
return &QueryResult{Error: err}, err
}
// Populate plan with broker buffer information
if stats != nil {
plan.BrokerBufferQueried = stats.BrokerBufferQueried
plan.BrokerBufferMessages = stats.BrokerBufferMessages
plan.BufferStartIndex = stats.BufferStartIndex
// Add broker_buffer to data sources if buffer was queried
if stats.BrokerBufferQueried {
// Check if broker_buffer is already in data sources
hasBrokerBuffer := false
for _, source := range plan.DataSources {
if source == "broker_buffer" {
hasBrokerBuffer = true
break
}
}
if !hasBrokerBuffer {
plan.DataSources = append(plan.DataSources, "broker_buffer")
}
}
}
// Populate execution plan details with source file information for Data Sources Tree
if partitions, discoverErr := e.discoverTopicPartitions(database, tableName); discoverErr == nil {
// Add partition paths to execution plan details
plan.Details["partition_paths"] = partitions
// Persist time filter details for downstream pruning/diagnostics
plan.Details[PlanDetailStartTimeNs] = startTimeNs
plan.Details[PlanDetailStopTimeNs] = stopTimeNs
// Collect actual file information for each partition
var parquetFiles []string
var liveLogFiles []string
parquetSources := make(map[string]bool)
var parquetReadErrors []string
var liveLogListErrors []string
for _, partitionPath := range partitions {
// Get parquet files for this partition
if parquetStats, err := hybridScanner.ReadParquetStatistics(partitionPath); err == nil {
// Prune files by time range with debug logging
filteredStats := pruneParquetFilesByTime(ctx, parquetStats, hybridScanner, startTimeNs, stopTimeNs)
// Further prune by column statistics from WHERE clause
if stmt.Where != nil {
beforeColumnPrune := len(filteredStats)
filteredStats = e.pruneParquetFilesByColumnStats(ctx, filteredStats, stmt.Where.Expr)
columnPrunedCount := beforeColumnPrune - len(filteredStats)
if columnPrunedCount > 0 {
// Track column statistics optimization
if !contains(plan.OptimizationsUsed, "column_statistics_pruning") {
plan.OptimizationsUsed = append(plan.OptimizationsUsed, "column_statistics_pruning")
}
}
}
for _, stats := range filteredStats {
parquetFiles = append(parquetFiles, fmt.Sprintf("%s/%s", partitionPath, stats.FileName))
}
} else {
parquetReadErrors = append(parquetReadErrors, fmt.Sprintf("%s: %v", partitionPath, err))
}
// Merge accurate parquet sources from metadata
if sources, err := e.getParquetSourceFilesFromMetadata(partitionPath); err == nil {
for src := range sources {
parquetSources[src] = true
}
}
// Get live log files for this partition
if liveFiles, err := e.collectLiveLogFileNames(hybridScanner.filerClient, partitionPath); err == nil {
for _, fileName := range liveFiles {
// Exclude live log files that have been converted to parquet (deduplicated)
if parquetSources[fileName] {
continue
}
liveLogFiles = append(liveLogFiles, fmt.Sprintf("%s/%s", partitionPath, fileName))
}
} else {
liveLogListErrors = append(liveLogListErrors, fmt.Sprintf("%s: %v", partitionPath, err))
}
}
if len(parquetFiles) > 0 {
plan.Details["parquet_files"] = parquetFiles
}
if len(liveLogFiles) > 0 {
plan.Details["live_log_files"] = liveLogFiles
}
if len(parquetReadErrors) > 0 {
plan.Details["error_parquet_statistics"] = parquetReadErrors
}
if len(liveLogListErrors) > 0 {
plan.Details["error_live_log_listing"] = liveLogListErrors
}
// Update scan statistics for execution plan display
plan.PartitionsScanned = len(partitions)
plan.ParquetFilesScanned = len(parquetFiles)
plan.LiveLogFilesScanned = len(liveLogFiles)
} else {
// Handle partition discovery error
plan.Details["error_partition_discovery"] = discoverErr.Error()
}
} else {
// Normal mode - just get results
results, err = hybridScanner.Scan(ctx, hybridScanOptions)
if err != nil {
return &QueryResult{Error: err}, err
}
}
// Convert to SQL result format
if selectAll {
if len(columns) > 0 {
// SELECT *, specific_columns - include both auto-discovered and explicit columns
return hybridScanner.ConvertToSQLResultWithMixedColumns(results, columns), nil
} else {
// SELECT * only - let converter determine all columns (excludes system columns)
columns = nil
return hybridScanner.ConvertToSQLResult(results, columns), nil
}
}
// Handle custom column expressions (including arithmetic)
return e.ConvertToSQLResultWithExpressions(hybridScanner, results, stmt.SelectExprs), nil
}
// extractTimeFilters extracts time range filters from WHERE clause for optimization
// This allows push-down of time-based queries to improve scan performance
// Returns (startTimeNs, stopTimeNs) where 0 means unbounded
+63
View File
@@ -1254,6 +1254,69 @@ func TestSQLEngine_LogBufferDeduplication_ServerRestartScenario(t *testing.T) {
// prevent false positive duplicates across server restarts
}
func TestBrokerClient_BinaryBufferStartFormat(t *testing.T) {
// Test scenario: getBufferStartFromEntry should only support binary format
// This tests the standardized binary format for buffer_start metadata
realBrokerClient := &BrokerClient{}
// Test binary format (used by both log files and Parquet files)
binaryEntry := &filer_pb.Entry{
Name: "2025-01-07-14-30-45",
IsDirectory: false,
Extended: map[string][]byte{
"buffer_start": func() []byte {
// Binary format: 8-byte BigEndian
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, uint64(2000001))
return buf
}(),
},
}
bufferStart := realBrokerClient.getBufferStartFromEntry(binaryEntry)
assert.NotNil(t, bufferStart)
assert.Equal(t, int64(2000001), bufferStart.StartIndex, "Should parse binary buffer_start metadata")
// Test Parquet file (same binary format)
parquetEntry := &filer_pb.Entry{
Name: "2025-01-07-14-30.parquet",
IsDirectory: false,
Extended: map[string][]byte{
"buffer_start": func() []byte {
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, uint64(1500001))
return buf
}(),
},
}
bufferStart = realBrokerClient.getBufferStartFromEntry(parquetEntry)
assert.NotNil(t, bufferStart)
assert.Equal(t, int64(1500001), bufferStart.StartIndex, "Should parse binary buffer_start from Parquet file")
// Test missing metadata
emptyEntry := &filer_pb.Entry{
Name: "no-metadata",
IsDirectory: false,
Extended: nil,
}
bufferStart = realBrokerClient.getBufferStartFromEntry(emptyEntry)
assert.Nil(t, bufferStart, "Should return nil for entry without buffer_start metadata")
// Test invalid format (wrong size)
invalidEntry := &filer_pb.Entry{
Name: "invalid-metadata",
IsDirectory: false,
Extended: map[string][]byte{
"buffer_start": []byte("invalid"),
},
}
bufferStart = realBrokerClient.getBufferStartFromEntry(invalidEntry)
assert.Nil(t, bufferStart, "Should return nil for invalid buffer_start metadata")
}
// TestGetSQLValAlias tests the getSQLValAlias function, particularly for SQL injection prevention
func TestGetSQLValAlias(t *testing.T) {
engine := &SQLEngine{}
+128
View File
@@ -320,6 +320,12 @@ func (hms *HybridMessageScanner) ScanWithStats(ctx context.Context, options Hybr
return results, stats, nil
}
// scanUnflushedData queries brokers for unflushed in-memory data using buffer_start deduplication
func (hms *HybridMessageScanner) scanUnflushedData(ctx context.Context, partition topic.Partition, options HybridScanOptions) ([]HybridScanResult, error) {
results, _, err := hms.scanUnflushedDataWithStats(ctx, partition, options)
return results, err
}
// scanUnflushedDataWithStats queries brokers for unflushed data and returns statistics
func (hms *HybridMessageScanner) scanUnflushedDataWithStats(ctx context.Context, partition topic.Partition, options HybridScanOptions) ([]HybridScanResult, *HybridScanStats, error) {
var results []HybridScanResult
@@ -430,6 +436,27 @@ func (hms *HybridMessageScanner) scanUnflushedDataWithStats(ctx context.Context,
return results, stats, nil
}
// convertDataMessageToRecord converts mq_pb.DataMessage to schema_pb.RecordValue
func (hms *HybridMessageScanner) convertDataMessageToRecord(msg *mq_pb.DataMessage) (*schema_pb.RecordValue, string, error) {
// Parse the message data as RecordValue
recordValue := &schema_pb.RecordValue{}
if err := proto.Unmarshal(msg.Value, recordValue); err != nil {
return nil, "", fmt.Errorf("failed to unmarshal message data: %v", err)
}
// Add system columns
if recordValue.Fields == nil {
recordValue.Fields = make(map[string]*schema_pb.Value)
}
// Add timestamp
recordValue.Fields[SW_COLUMN_NAME_TIMESTAMP] = &schema_pb.Value{
Kind: &schema_pb.Value_Int64Value{Int64Value: msg.TsNs},
}
return recordValue, string(msg.Key), nil
}
// discoverTopicPartitions discovers the actual partitions for this topic by scanning the filesystem
// This finds real partition directories like v2025-09-01-07-16-34/0000-0630/
func (hms *HybridMessageScanner) discoverTopicPartitions(ctx context.Context) ([]topic.Partition, error) {
@@ -494,6 +521,15 @@ func (hms *HybridMessageScanner) discoverTopicPartitions(ctx context.Context) ([
return allPartitions, nil
}
// scanPartitionHybrid scans a specific partition using the hybrid approach
// This is where the magic happens - seamlessly reading ALL data sources:
// 1. Unflushed in-memory data from brokers (REAL-TIME)
// 2. Live logs + Parquet files from disk (FLUSHED/ARCHIVED)
func (hms *HybridMessageScanner) scanPartitionHybrid(ctx context.Context, partition topic.Partition, options HybridScanOptions) ([]HybridScanResult, error) {
results, _, err := hms.scanPartitionHybridWithStats(ctx, partition, options)
return results, err
}
// scanPartitionHybridWithStats scans a specific partition using streaming merge for memory efficiency
// PERFORMANCE IMPROVEMENT: Uses heap-based streaming merge instead of collecting all data and sorting
// - Memory usage: O(k) where k = number of data sources, instead of O(n) where n = total records
@@ -611,6 +647,23 @@ func (hms *HybridMessageScanner) countLiveLogFiles(partition topic.Partition) (i
return fileCount, nil
}
// isControlEntry checks if a log entry is a control entry without actual data
// Based on MQ system analysis, control entries are:
// 1. DataMessages with populated Ctrl field (publisher close signals)
// 2. Entries with empty keys (as filtered by subscriber)
// NOTE: Messages with empty data but valid keys (like NOOP messages) are NOT control entries
func (hms *HybridMessageScanner) isControlEntry(logEntry *filer_pb.LogEntry) bool {
// Pre-decode DataMessage if needed
var dataMessage *mq_pb.DataMessage
if len(logEntry.Data) > 0 {
dataMessage = &mq_pb.DataMessage{}
if err := proto.Unmarshal(logEntry.Data, dataMessage); err != nil {
dataMessage = nil // Failed to decode, treat as raw data
}
}
return hms.isControlEntryWithDecoded(logEntry, dataMessage)
}
// isControlEntryWithDecoded checks if a log entry is a control entry using pre-decoded DataMessage
// This avoids duplicate protobuf unmarshaling when the DataMessage is already decoded
func (hms *HybridMessageScanner) isControlEntryWithDecoded(logEntry *filer_pb.LogEntry, dataMessage *mq_pb.DataMessage) bool {
@@ -629,6 +682,26 @@ func (hms *HybridMessageScanner) isControlEntryWithDecoded(logEntry *filer_pb.Lo
return false
}
// isNullOrEmpty checks if a schema_pb.Value is null or empty
func isNullOrEmpty(value *schema_pb.Value) bool {
if value == nil {
return true
}
switch v := value.Kind.(type) {
case *schema_pb.Value_StringValue:
return v.StringValue == ""
case *schema_pb.Value_BytesValue:
return len(v.BytesValue) == 0
case *schema_pb.Value_ListValue:
return v.ListValue == nil || len(v.ListValue.Values) == 0
case nil:
return true // No kind set means null
default:
return false
}
}
// isSchemaless checks if the scanner is configured for a schema-less topic
// Schema-less topics only have system fields: _ts_ns, _key, and _value
func (hms *HybridMessageScanner) isSchemaless() bool {
@@ -663,6 +736,61 @@ func (hms *HybridMessageScanner) isSchemaless() bool {
return hasValue && dataFieldCount == 1
}
// convertLogEntryToRecordValue converts a filer_pb.LogEntry to schema_pb.RecordValue
// This handles both:
// 1. Live log entries (raw message format)
// 2. Parquet entries (already in schema_pb.RecordValue format)
// 3. Schema-less topics (raw bytes in _value field)
func (hms *HybridMessageScanner) convertLogEntryToRecordValue(logEntry *filer_pb.LogEntry) (*schema_pb.RecordValue, string, error) {
// For schema-less topics, put raw data directly into _value field
if hms.isSchemaless() {
recordValue := &schema_pb.RecordValue{
Fields: make(map[string]*schema_pb.Value),
}
recordValue.Fields[SW_COLUMN_NAME_TIMESTAMP] = &schema_pb.Value{
Kind: &schema_pb.Value_Int64Value{Int64Value: logEntry.TsNs},
}
recordValue.Fields[SW_COLUMN_NAME_KEY] = &schema_pb.Value{
Kind: &schema_pb.Value_BytesValue{BytesValue: logEntry.Key},
}
recordValue.Fields[SW_COLUMN_NAME_VALUE] = &schema_pb.Value{
Kind: &schema_pb.Value_BytesValue{BytesValue: logEntry.Data},
}
return recordValue, "live_log", nil
}
// Try to unmarshal as RecordValue first (Parquet format)
recordValue := &schema_pb.RecordValue{}
if err := proto.Unmarshal(logEntry.Data, recordValue); err == nil {
// This is an archived message from Parquet files
// FIX: Add system columns from LogEntry to RecordValue
if recordValue.Fields == nil {
recordValue.Fields = make(map[string]*schema_pb.Value)
}
// Add system columns from LogEntry
recordValue.Fields[SW_COLUMN_NAME_TIMESTAMP] = &schema_pb.Value{
Kind: &schema_pb.Value_Int64Value{Int64Value: logEntry.TsNs},
}
recordValue.Fields[SW_COLUMN_NAME_KEY] = &schema_pb.Value{
Kind: &schema_pb.Value_BytesValue{BytesValue: logEntry.Key},
}
return recordValue, "parquet_archive", nil
}
// If not a RecordValue, this is raw live message data - parse with schema
return hms.parseRawMessageWithSchema(logEntry)
}
// min returns the minimum of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}
// parseRawMessageWithSchema parses raw live message data using the topic's schema
// This provides proper type conversion and field mapping instead of treating everything as strings
func (hms *HybridMessageScanner) parseRawMessageWithSchema(logEntry *filer_pb.LogEntry) (*schema_pb.RecordValue, string, error) {
+86
View File
@@ -6,6 +6,8 @@ import (
"math/big"
"time"
"github.com/parquet-go/parquet-go"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/mq/schema"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -170,6 +172,90 @@ func (ps *ParquetScanner) scanPartition(ctx context.Context, partition topic.Par
return results, nil
}
// scanParquetFile scans a single Parquet file (real implementation)
func (ps *ParquetScanner) scanParquetFile(ctx context.Context, entry *filer_pb.Entry, options ScanOptions) ([]ScanResult, error) {
var results []ScanResult
// Create reader for the Parquet file (same pattern as logstore)
lookupFileIdFn := filer.LookupFn(ps.filerClient)
fileSize := filer.FileSize(entry)
visibleIntervals, _ := filer.NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, entry.Chunks, 0, int64(fileSize))
chunkViews := filer.ViewFromVisibleIntervals(visibleIntervals, 0, int64(fileSize))
readerCache := filer.NewReaderCache(32, ps.chunkCache, lookupFileIdFn)
readerAt := filer.NewChunkReaderAtFromClient(ctx, readerCache, chunkViews, int64(fileSize), filer.DefaultPrefetchCount)
// Create Parquet reader
parquetReader := parquet.NewReader(readerAt)
defer parquetReader.Close()
rows := make([]parquet.Row, 128) // Read in batches like logstore
for {
rowCount, readErr := parquetReader.ReadRows(rows)
// Process rows even if EOF
for i := 0; i < rowCount; i++ {
// Convert Parquet row to schema value
recordValue, err := schema.ToRecordValue(ps.recordSchema, ps.parquetLevels, rows[i])
if err != nil {
return nil, fmt.Errorf("failed to convert row: %v", err)
}
// Extract system columns
timestamp := recordValue.Fields[SW_COLUMN_NAME_TIMESTAMP].GetInt64Value()
key := recordValue.Fields[SW_COLUMN_NAME_KEY].GetBytesValue()
// Apply time filtering
if options.StartTimeNs > 0 && timestamp < options.StartTimeNs {
continue
}
if options.StopTimeNs > 0 && timestamp >= options.StopTimeNs {
break // Assume data is time-ordered
}
// Apply predicate filtering (WHERE clause)
if options.Predicate != nil && !options.Predicate(recordValue) {
continue
}
// Apply column projection
values := make(map[string]*schema_pb.Value)
if len(options.Columns) == 0 {
// Select all columns (excluding system columns from user view)
for name, value := range recordValue.Fields {
if name != SW_COLUMN_NAME_TIMESTAMP && name != SW_COLUMN_NAME_KEY {
values[name] = value
}
}
} else {
// Select specified columns only
for _, columnName := range options.Columns {
if value, exists := recordValue.Fields[columnName]; exists {
values[columnName] = value
}
}
}
results = append(results, ScanResult{
Values: values,
Timestamp: timestamp,
Key: key,
})
// Apply row limit
if options.Limit > 0 && len(results) >= options.Limit {
return results, nil
}
}
if readErr != nil {
break // EOF or error
}
}
return results, nil
}
// generateSampleData creates sample data for testing when no real Parquet files exist
func (ps *ParquetScanner) generateSampleData(options ScanOptions) []ScanResult {
now := time.Now().UnixNano()
+8
View File
@@ -50,6 +50,14 @@ func (e *SQLEngine) getSystemColumnDisplayName(columnName string) string {
}
}
// isSystemColumnDisplayName checks if a column name is a system column display name
func (e *SQLEngine) isSystemColumnDisplayName(columnName string) bool {
lowerName := strings.ToLower(columnName)
return lowerName == SW_DISPLAY_NAME_TIMESTAMP ||
lowerName == SW_COLUMN_NAME_KEY ||
lowerName == SW_COLUMN_NAME_SOURCE
}
// getSystemColumnInternalName returns the internal name for a system column display name
func (e *SQLEngine) getSystemColumnInternalName(displayName string) string {
lowerName := strings.ToLower(displayName)
+28 -96
View File
@@ -3,6 +3,7 @@ package replication
import (
"context"
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -38,125 +39,56 @@ func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_p
if message.IsFromOtherCluster && r.sink.GetName() == "filer" {
return nil
}
oldEntry := message.OldEntry
newEntry := message.NewEntry
newParentPath := message.NewParentPath
oldInSource := util.IsEqualOrUnder(key, r.source.Dir) && !r.isExcluded(key)
// For rename events (both old and new entry present), check both paths
// against the source directory. Convert cross-boundary renames to
// create or delete so the sink stays consistent.
if oldEntry != nil && newEntry != nil {
newFullPath, targetParent := metadataEventTarget(key, newEntry, newParentPath)
newInSource := util.IsEqualOrUnder(newFullPath, r.source.Dir) && !r.isExcluded(newFullPath)
if !oldInSource && !newInSource {
return nil
}
if !oldInSource {
// Rename into watched directory: treat as create
oldEntry = nil
key = newFullPath
newParentPath = targetParent
} else if !newInSource {
// Rename out of watched directory: treat as delete
newEntry = nil
newParentPath = ""
}
} else if !oldInSource {
if !strings.HasPrefix(key, r.source.Dir) {
glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
return nil
}
for _, excludeDir := range r.excludeDirs {
if strings.HasPrefix(key, excludeDir) {
glog.V(4).Infof("skipping %v of exclude dir %v", key, excludeDir)
return nil
}
}
var dateKey string
if r.sink.IsIncremental() {
var mTime int64
if newEntry != nil {
mTime = newEntry.Attributes.Mtime
} else if oldEntry != nil {
mTime = oldEntry.Attributes.Mtime
if message.NewEntry != nil {
mTime = message.NewEntry.Attributes.Mtime
} else if message.OldEntry != nil {
mTime = message.OldEntry.Attributes.Mtime
}
dateKey = time.Unix(mTime, 0).Format("2006-01-02")
}
oldSinkKey := r.sourceToSinkKey(key, dateKey)
glog.V(3).Infof("replicate %s => %s", key, oldSinkKey)
newSinkKey := oldSinkKey
newSinkParentPath := newParentPath
if oldEntry != nil && newEntry != nil {
targetSourceKey, targetSourceParent := metadataEventTarget(key, newEntry, newParentPath)
newSinkKey = r.sourceToSinkKey(targetSourceKey, dateKey)
newSinkParentPath = r.sourceToSinkPath(targetSourceParent, dateKey)
} else if newParentPath != "" && util.IsEqualOrUnder(newParentPath, r.source.Dir) {
newSinkParentPath = r.sourceToSinkPath(newParentPath, dateKey)
newKey := util.Join(r.sink.GetSinkToDirectory(), dateKey, key[len(r.source.Dir):])
glog.V(3).Infof("replicate %s => %s", key, newKey)
key = newKey
if message.OldEntry != nil && message.NewEntry == nil {
glog.V(4).Infof("deleting %v", key)
return r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
}
if oldEntry != nil && newEntry == nil {
glog.V(4).Infof("deleting %v", oldSinkKey)
return r.sink.DeleteEntry(oldSinkKey, oldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
if message.OldEntry == nil && message.NewEntry != nil {
glog.V(4).Infof("creating %v", key)
return r.sink.CreateEntry(key, message.NewEntry, message.Signatures)
}
if oldEntry == nil && newEntry != nil {
glog.V(4).Infof("creating %v", oldSinkKey)
return r.sink.CreateEntry(oldSinkKey, newEntry, message.Signatures)
}
if oldEntry == nil && newEntry == nil {
if message.OldEntry == nil && message.NewEntry == nil {
glog.V(0).Infof("weird message %+v", message)
return nil
}
if oldSinkKey != newSinkKey && r.sink.GetName() != "filer" {
if err := r.sink.DeleteEntry(oldSinkKey, oldEntry.IsDirectory, false, message.Signatures); err != nil {
return fmt.Errorf("delete old entry %v: %w", oldSinkKey, err)
}
glog.V(4).Infof("creating renamed %v", newSinkKey)
return r.sink.CreateEntry(newSinkKey, newEntry, message.Signatures)
}
foundExisting, err := r.sink.UpdateEntry(oldSinkKey, oldEntry, newSinkParentPath, newEntry, message.DeleteChunks, message.Signatures)
foundExisting, err := r.sink.UpdateEntry(key, message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
if foundExisting {
glog.V(4).Infof("updated %v", oldSinkKey)
glog.V(4).Infof("updated %v", key)
return err
}
err = r.sink.DeleteEntry(oldSinkKey, oldEntry.IsDirectory, false, message.Signatures)
err = r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, false, message.Signatures)
if err != nil {
return fmt.Errorf("delete old entry %v: %w", oldSinkKey, err)
return fmt.Errorf("delete old entry %v: %w", key, err)
}
glog.V(4).Infof("creating missing %v", newSinkKey)
return r.sink.CreateEntry(newSinkKey, newEntry, message.Signatures)
}
func (r *Replicator) isExcluded(path string) bool {
for _, excludeDir := range r.excludeDirs {
if util.IsEqualOrUnder(path, excludeDir) {
return true
}
}
return false
}
func (r *Replicator) sourceToSinkKey(sourceKey, dateKey string) string {
return util.Join(r.sink.GetSinkToDirectory(), dateKey, sourceKey[len(r.source.Dir):])
}
func (r *Replicator) sourceToSinkPath(sourcePath, dateKey string) string {
return util.Join(r.sink.GetSinkToDirectory(), dateKey, sourcePath[len(r.source.Dir):])
}
func metadataEventTarget(key string, newEntry *filer_pb.Entry, newParentPath string) (targetKey, targetParent string) {
if newEntry == nil {
return "", ""
}
targetParent = newParentPath
if targetParent == "" {
targetParent, _ = util.FullPath(key).DirAndName()
}
return util.Join(targetParent, newEntry.Name), targetParent
glog.V(4).Infof("creating missing %v", key)
return r.sink.CreateEntry(key, message.NewEntry, message.Signatures)
}
func ReadFilerSignature(grpcDialOption grpc.DialOption, filer pb.ServerAddress) (filerSignature int32, readErr error) {
-284
View File
@@ -1,284 +0,0 @@
package replication
import (
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var _ sink.ReplicationSink = (*recordingSink)(nil)
type deleteCall struct {
key string
isDirectory bool
}
type createCall struct {
key string
}
type updateCall struct {
key string
newParentPath string
}
type recordingSink struct {
name string
sinkToDirectory string
incremental bool
updateFoundExisting bool
deleteCalls []deleteCall
createCalls []createCall
updateCalls []updateCall
}
func (s *recordingSink) GetName() string {
return s.name
}
func (s *recordingSink) Initialize(util.Configuration, string) error {
return nil
}
func (s *recordingSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
s.deleteCalls = append(s.deleteCalls, deleteCall{key: key, isDirectory: isDirectory})
return nil
}
func (s *recordingSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
s.createCalls = append(s.createCalls, createCall{key: key})
return nil
}
func (s *recordingSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
s.updateCalls = append(s.updateCalls, updateCall{key: key, newParentPath: newParentPath})
return s.updateFoundExisting, nil
}
func (s *recordingSink) GetSinkToDirectory() string {
return s.sinkToDirectory
}
func (s *recordingSink) SetSourceFiler(*source.FilerSource) {}
func (s *recordingSink) IsIncremental() bool {
return s.incremental
}
func TestReplicateRenameUsesTargetKeyForNonFilerSink(t *testing.T) {
s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "renamed.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/source/new",
})
if err != nil {
t.Fatalf("Replicate rename: %v", err)
}
if len(s.updateCalls) != 0 {
t.Fatalf("expected non-filer rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
}
if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
}
}
func TestReplicateRenameUsesUpdateForFilerSink(t *testing.T) {
s := &recordingSink{
name: "filer",
sinkToDirectory: "/dest",
updateFoundExisting: true,
}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "renamed.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/source/new",
})
if err != nil {
t.Fatalf("Replicate rename: %v", err)
}
if len(s.updateCalls) != 1 {
t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
}
if s.updateCalls[0].key != "/dest/old/file.txt" {
t.Fatalf("update key = %q, want /dest/old/file.txt", s.updateCalls[0].key)
}
if s.updateCalls[0].newParentPath != "/dest/new" {
t.Fatalf("update newParentPath = %q, want /dest/new", s.updateCalls[0].newParentPath)
}
if len(s.deleteCalls) != 0 || len(s.createCalls) != 0 {
t.Fatalf("unexpected delete/create calls: deletes=%+v creates=%+v", s.deleteCalls, s.createCalls)
}
}
func TestReplicateRenameFallbackCreatesTargetKey(t *testing.T) {
s := &recordingSink{
name: "filer",
sinkToDirectory: "/dest",
updateFoundExisting: false,
}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "renamed.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/source/new",
})
if err != nil {
t.Fatalf("Replicate rename fallback: %v", err)
}
if len(s.updateCalls) != 1 {
t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
}
if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
}
}
func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
tests := []struct {
name string
candidate string
other string
expected bool
}{
{name: "equal", candidate: "/foo", other: "/foo", expected: true},
{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
{name: "root", candidate: "/foo/bar", other: "/", expected: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
}
})
}
}
func TestReplicateRenameOutToSiblingPrefixBecomesDelete(t *testing.T) {
s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/foo"},
}
err := r.Replicate(context.Background(), "/foo/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/foobar/new",
})
if err != nil {
t.Fatalf("Replicate rename out to sibling prefix: %v", err)
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
}
if len(s.createCalls) != 0 || len(s.updateCalls) != 0 {
t.Fatalf("unexpected create/update calls: creates=%+v updates=%+v", s.createCalls, s.updateCalls)
}
}
func TestReplicateRenameFromExcludedDirBecomesCreate(t *testing.T) {
s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/foo"},
excludeDirs: []string{"/foo/excluded"},
}
err := r.Replicate(context.Background(), "/foo/excluded/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/foo/live",
})
if err != nil {
t.Fatalf("Replicate rename from excluded dir: %v", err)
}
if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/live/file.txt" {
t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
}
if len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
t.Fatalf("unexpected delete/update calls: deletes=%+v updates=%+v", s.deleteCalls, s.updateCalls)
}
}
-4
View File
@@ -50,10 +50,6 @@ func (fs *FilerSource) DoInitialize(address, grpcAddress string, dir string, rea
return nil
}
func (fs *FilerSource) SetGrpcDialOption(option grpc.DialOption) {
fs.grpcDialOption = option
}
func (fs *FilerSource) LookupFileId(ctx context.Context, part string) (fileUrls []string, err error) {
vid2Locations := make(map[string]*filer_pb.Locations)
+6 -20
View File
@@ -9,7 +9,6 @@ import (
"github.com/Shopify/sarama"
"github.com/seaweedfs/seaweedfs/weed/glog"
kafkanotif "github.com/seaweedfs/seaweedfs/weed/notification/kafka"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
@@ -37,38 +36,25 @@ func (k *KafkaInput) Initialize(configuration util.Configuration, prefix string)
configuration.GetString(prefix+"topic"),
configuration.GetString(prefix+"offsetFile"),
configuration.GetInt(prefix+"offsetSaveIntervalSeconds"),
kafkanotif.SASLTLSConfig{
SASLEnabled: configuration.GetBool(prefix + "sasl_enabled"),
SASLMechanism: configuration.GetString(prefix + "sasl_mechanism"),
SASLUsername: configuration.GetString(prefix + "sasl_username"),
SASLPassword: configuration.GetString(prefix + "sasl_password"),
TLSEnabled: configuration.GetBool(prefix + "tls_enabled"),
TLSCACert: configuration.GetString(prefix + "tls_ca_cert"),
TLSClientCert: configuration.GetString(prefix + "tls_client_cert"),
TLSClientKey: configuration.GetString(prefix + "tls_client_key"),
TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
},
)
}
func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string, offsetSaveIntervalSeconds int, saslTLS kafkanotif.SASLTLSConfig) (err error) {
func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string, offsetSaveIntervalSeconds int) (err error) {
config := sarama.NewConfig()
config.Consumer.Return.Errors = true
if err = kafkanotif.ConfigureSASLTLS(config, saslTLS); err != nil {
return fmt.Errorf("kafka consumer security configuration: %w", err)
}
k.consumer, err = sarama.NewConsumer(hosts, config)
if err != nil {
return fmt.Errorf("create kafka consumer: %w", err)
panic(err)
} else {
glog.V(0).Infof("connected to %v", hosts)
}
glog.V(0).Infof("connected to %v", hosts)
k.topic = topic
k.messageChan = make(chan *sarama.ConsumerMessage, 1)
partitions, err := k.consumer.Partitions(topic)
if err != nil {
return fmt.Errorf("get kafka partitions for topic %q: %w", topic, err)
panic(err)
}
progress := loadProgress(offsetFile)
@@ -91,7 +77,7 @@ func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string,
}
partitionConsumer, err := k.consumer.ConsumePartition(topic, partition, offset)
if err != nil {
return fmt.Errorf("consume kafka topic %q partition %d: %w", topic, partition, err)
panic(err)
}
go func() {
for {
+12
View File
@@ -1039,6 +1039,13 @@ func getMD5HashBase64(data []byte) string {
return base64.StdEncoding.EncodeToString(getMD5Sum(data))
}
// getSHA256Sum returns SHA-256 sum of given data.
func getSHA256Sum(data []byte) []byte {
hash := sha256.New()
hash.Write(data)
return hash.Sum(nil)
}
// getMD5Sum returns MD5 sum of given data.
func getMD5Sum(data []byte) []byte {
hash := md5.New()
@@ -1046,6 +1053,11 @@ func getMD5Sum(data []byte) []byte {
return hash.Sum(nil)
}
// getMD5Hash returns MD5 hash in hex encoding of given data.
func getMD5Hash(data []byte) string {
return hex.EncodeToString(getMD5Sum(data))
}
var ignoredHeaders = map[string]bool{
"Authorization": true,
"Content-Type": true,
+4
View File
@@ -539,6 +539,10 @@ func testJWTAuthentication(t *testing.T, iam *IdentityAccessManagement, token st
return iam.authenticateJWTWithIAM(req)
}
func testJWTAuthorization(t *testing.T, iam *IdentityAccessManagement, identity *Identity, action Action, bucket, object, token string) bool {
return testJWTAuthorizationWithRole(t, iam, identity, action, bucket, object, token, "TestRole")
}
func testJWTAuthorizationWithRole(t *testing.T, iam *IdentityAccessManagement, identity *Identity, action Action, bucket, object, token, roleName string) bool {
// Create test request
req := httptest.NewRequest("GET", "/"+bucket+"/"+object, http.NoBody)
+67
View File
@@ -0,0 +1,67 @@
package s3api
import (
"bytes"
"crypto/md5"
"encoding/base64"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// ResponseRecorder that also implements http.Flusher
type recorderFlusher struct{ *httptest.ResponseRecorder }
func (r recorderFlusher) Flush() {}
// TestSSECRangeRequestsSupported verifies that HTTP Range requests are now supported
// for SSE-C encrypted objects since the IV is stored in metadata and CTR mode allows seeking
func TestSSECRangeRequestsSupported(t *testing.T) {
// Create a request with Range header and valid SSE-C headers
req := httptest.NewRequest(http.MethodGet, "/b/o", nil)
req.Header.Set("Range", "bytes=10-20")
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
key := make([]byte, 32)
for i := range key {
key[i] = byte(i)
}
s := md5.Sum(key)
keyMD5 := base64.StdEncoding.EncodeToString(s[:])
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKey, base64.StdEncoding.EncodeToString(key))
req.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyMD5)
// Attach mux vars to avoid panic in error writer
req = mux.SetURLVars(req, map[string]string{"bucket": "b", "object": "o"})
// Create a mock HTTP response that simulates SSE-C encrypted object metadata
proxyResponse := &http.Response{
StatusCode: 200,
Header: make(http.Header),
Body: io.NopCloser(bytes.NewReader([]byte("mock encrypted data"))),
}
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionCustomerAlgorithm, "AES256")
proxyResponse.Header.Set(s3_constants.AmzServerSideEncryptionCustomerKeyMD5, keyMD5)
// Call the function under test - should no longer reject range requests
s3a := &S3ApiServer{
option: &S3ApiServerOption{
BucketsPath: "/buckets",
},
}
rec := httptest.NewRecorder()
w := recorderFlusher{rec}
// Pass nil for entry since this test focuses on Range request handling
statusCode, _ := s3a.handleSSECResponse(req, proxyResponse, w, nil)
// Range requests should now be allowed to proceed (will be handled by filer layer)
// The exact status code depends on the object existence and filer response
if statusCode == http.StatusRequestedRangeNotSatisfiable {
t.Fatalf("Range requests should no longer be rejected for SSE-C objects, got status %d", statusCode)
}
}

Some files were not shown because too many files have changed in this diff Show More