mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-15 11:05:51 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e2270ec25 | ||
|
|
8093ecf4c8 | ||
|
|
6840371774 | ||
|
|
d67fc47a61 | ||
|
|
d0f3fad133 | ||
|
|
624a13ecc6 | ||
|
|
492e700d1f | ||
|
|
b5ee5b871e | ||
|
|
6fb21c9ec0 | ||
|
|
41260a55f6 | ||
|
|
3031ab5d97 | ||
|
|
acb44a68a9 | ||
|
|
6927de6003 | ||
|
|
613a286cde | ||
|
|
55ac46f7a9 | ||
|
|
3f8035deb7 | ||
|
|
a4c8d31cb8 | ||
|
|
6bac506842 | ||
|
|
2c031d9d63 | ||
|
|
d69f45ee83 | ||
|
|
3aaf80b33b | ||
|
|
85f04b54fd | ||
|
|
6f93f87446 | ||
|
|
73949daa98 | ||
|
|
603a1221a7 | ||
|
|
f11cecc697 | ||
|
|
1076ebdf4a | ||
|
|
bd0bc06df7 | ||
|
|
50a9312e1e | ||
|
|
1bc430a4ec | ||
|
|
63d2024b82 | ||
|
|
72da592b27 | ||
|
|
cfbfb60817 | ||
|
|
9a65616cd0 | ||
|
|
985d0473bf | ||
|
|
6d692ab013 | ||
|
|
aa4252d491 | ||
|
|
02f2042360 | ||
|
|
dd812413fe | ||
|
|
547befc070 | ||
|
|
17ba8b3c7c | ||
|
|
59db4ddc6c | ||
|
|
e7ae0d6049 | ||
|
|
fac021544e | ||
|
|
448de42b05 | ||
|
|
d050464316 | ||
|
|
c96eb7b8ee | ||
|
|
b2eefb8a73 | ||
|
|
774c9927a7 | ||
|
|
c3d13c60e9 | ||
|
|
4bf5e92b64 | ||
|
|
5f1a742726 | ||
|
|
d3dc03c85a | ||
|
|
ea65746947 | ||
|
|
c023493cf0 | ||
|
|
924a683397 | ||
|
|
4d306e1279 | ||
|
|
c4f2f514fa | ||
|
|
5d80ac39b8 |
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.38.2
|
||||
uses: github/codeql-action/init@v4.38.3
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4.38.2
|
||||
uses: github/codeql-action/autobuild@v4.38.3
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -49,4 +49,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4.38.2
|
||||
uses: github/codeql-action/analyze@v4.38.3
|
||||
|
||||
@@ -405,7 +405,7 @@ jobs:
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4.38.2
|
||||
uses: github/codeql-action/upload-sarif@v4.38.3
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
|
||||
@@ -460,7 +460,7 @@ jobs:
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4.38.2
|
||||
uses: github/codeql-action/upload-sarif@v4.38.3
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
@@ -148,3 +148,21 @@ jobs:
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
- name: Test linux/386
|
||||
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
|
||||
|
||||
build-azurekms:
|
||||
name: Build and test with azurekms tag
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
# weed/kms/azure is excluded from every default build, so without this
|
||||
# nothing compiles it and tag-gated code silently rots.
|
||||
- name: Build and test with azurekms
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags azurekms ./...
|
||||
go test -tags azurekms ./kms/...
|
||||
|
||||
@@ -95,6 +95,41 @@ jobs:
|
||||
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
|
||||
echo "S3 rollout settings render correctly"
|
||||
|
||||
echo "=== Testing the S3 config checksum is stable across renders ==="
|
||||
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
|
||||
# Secret drew new keys for the checksum whenever lookup found nothing (on
|
||||
# install), so the first upgrade changed the checksum and restarted the
|
||||
# filer and S3 pods although nothing had changed.
|
||||
s3_checksum() { # <template> <helm args...>: the checksum/s3config of one workload
|
||||
local template=$1; shift
|
||||
local sum
|
||||
sum=$(helm template test $CHART_DIR --show-only "$template" "$@" \
|
||||
| grep 'checksum/s3config:' | awk '{print $2}')
|
||||
test -n "$sum" || { echo "no checksum/s3config in $template" >&2; return 1; }
|
||||
echo "$sum"
|
||||
}
|
||||
s3_auth_args=(--set s3.enabled=true --set s3.enableAuth=true)
|
||||
s3_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")
|
||||
filer_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}")
|
||||
test "$s3_sum" = "$filer_sum" # one Secret, one checksum
|
||||
test "$s3_sum" = "$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")"
|
||||
s3_creds_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}" \
|
||||
--set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
|
||||
test "$s3_sum" != "$s3_creds_sum"
|
||||
filer_audit_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
|
||||
--set-json 'filer.s3.auditLogConfig={"fluent_port":24224}')
|
||||
test "$filer_sum" != "$filer_audit_sum"
|
||||
# Settings the Secret does not read must not restart the S3 and filer pods.
|
||||
test "$filer_sum" = "$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
|
||||
--set s3.replicas=3 --set s3.resources.limits.cpu=1)"
|
||||
all_in_one_args=(--set allInOne.enabled=true --set allInOne.s3.enabled=true --set allInOne.s3.enableAuth=true)
|
||||
aio_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")
|
||||
test "$aio_sum" = "$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")"
|
||||
aio_creds_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}" \
|
||||
--set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
|
||||
test "$aio_sum" != "$aio_creds_sum"
|
||||
echo "S3 config checksum is stable and follows only what the S3 Secret reads"
|
||||
|
||||
echo "=== Testing S3 credentials from an existing secret ==="
|
||||
credential_args=(
|
||||
--set s3.credentials.admin.existingSecret=minio-root
|
||||
|
||||
@@ -44,7 +44,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.19.2
|
||||
github.com/klauspost/compress v1.20.1
|
||||
github.com/klauspost/reedsolomon v1.14.2
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.8
|
||||
@@ -90,14 +90,14 @@ require (
|
||||
gocloud.dev v0.46.0
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.46.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
|
||||
golang.org/x/image v0.46.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sys v0.48.0
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
golang.org/x/tools v0.49.0 // indirect
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
golang.org/x/tools v0.51.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.297.0
|
||||
google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect
|
||||
@@ -122,10 +122,10 @@ require (
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.1
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.4
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.109.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/cognusion/imaging v1.0.4
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
@@ -137,7 +137,7 @@ require (
|
||||
github.com/hashicorp/raft v1.8.0
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.4.2
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/jhump/protoreflect v1.18.1
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
@@ -156,11 +156,11 @@ require (
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/twmb/avro v1.9.0
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1
|
||||
go.uber.org/atomic v1.12.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
)
|
||||
@@ -182,7 +182,7 @@ require (
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.8.0 // indirect
|
||||
github.com/apache/thrift v0.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -318,20 +318,20 @@ require (
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.0
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1
|
||||
github.com/aws/smithy-go v1.28.1
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
@@ -469,10 +469,10 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20240819112322-98a264d392f6 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.5 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.2 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
@@ -490,7 +490,7 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.46.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.46.0
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/time v0.16.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
|
||||
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace $existingS3ConfigSecret) | default dict }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
|
||||
{{- else }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
|
||||
{{- end }}
|
||||
{{- $_ := set $podAnnotations "checksum/master-config" (include (print .Template.BasePath "/master/master-configmap.yaml") . | sha256sum) }}
|
||||
annotations:
|
||||
|
||||
@@ -48,7 +48,7 @@ spec:
|
||||
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.filer.s3.existingConfigSecret) | default dict }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
|
||||
{{- else }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
|
||||
@@ -44,7 +44,7 @@ spec:
|
||||
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.s3.existingConfigSecret) | default dict }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
|
||||
{{- else }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
|
||||
@@ -692,3 +692,28 @@ Usage: {{ include "seaweedfs.service.loadBalancerFields" .Values.s3.service }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Checksum that restarts the S3-serving pods when the generated S3 config changes. It hashes the
|
||||
values s3-secret.yaml is built from, not the rendered Secret: that template keeps its generated keys
|
||||
with lookup, so rendering it again for a checksum draws new random keys whenever lookup finds
|
||||
nothing (on install), and the first upgrade then changed the checksum and restarted the pods
|
||||
although nothing had changed. Only the values that template reads are hashed, so scaling the S3
|
||||
gateway or changing its resources does not restart the filer. Keep this list in step with
|
||||
s3-secret.yaml.
|
||||
Keys the chart generated are not part of it. Deleting the Secret to rotate them regenerates them on
|
||||
the next upgrade without restarting the pods; restart them afterwards (kubectl rollout restart).
|
||||
Folding the stored keys in would bring the first-upgrade restart back, since lookup finds nothing
|
||||
on install.
|
||||
*/}}
|
||||
{{- define "seaweedfs.s3.configChecksum" -}}
|
||||
{{- dict
|
||||
"secretName" (include "seaweedfs.fullname" .)
|
||||
"enabled" (list .Values.s3.enabled .Values.filer.s3.enabled .Values.allInOne.enabled .Values.allInOne.s3.enabled)
|
||||
"enableAuth" (list .Values.s3.enableAuth .Values.filer.s3.enableAuth .Values.allInOne.s3.enableAuth)
|
||||
"existingConfigSecret" (list .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret .Values.allInOne.s3.existingConfigSecret)
|
||||
"credentials" .Values.s3.credentials
|
||||
"legacySecret" (list .Values.s3.reuseLegacySecret .Values.s3.legacySecretName)
|
||||
"auditLogConfig" (list .Values.s3.auditLogConfig .Values.filer.s3.auditLogConfig)
|
||||
| toJson | sha256sum -}}
|
||||
{{- end -}}
|
||||
|
||||
+1050
-1061
File diff suppressed because it is too large
Load Diff
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 54 KiB |
Generated
+15
-8
@@ -234,9 +234,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.16.2"
|
||||
version = "1.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
|
||||
checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"zeroize",
|
||||
@@ -244,14 +244,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.39.1"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399"
|
||||
checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
"dunce",
|
||||
"fs_extra",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2677,6 +2678,12 @@ dependencies = [
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.18.1"
|
||||
@@ -3389,9 +3396,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.37"
|
||||
version = "0.23.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -3436,9 +3443,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.13"
|
||||
version = "0.103.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
|
||||
@@ -986,7 +986,12 @@ impl VolumeServer for VolumeGrpcService {
|
||||
let original_cookie = n.cookie;
|
||||
if !is_ec_volume {
|
||||
let store = self.state.store.read().unwrap();
|
||||
match store.read_volume_needle(file_id.volume_id, &mut n) {
|
||||
let mut read_option = crate::server::handlers::delete_read_option();
|
||||
match store.read_volume_needle_with_option(
|
||||
file_id.volume_id,
|
||||
&mut n,
|
||||
&mut read_option,
|
||||
) {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
results.push(volume_server_pb::DeleteResult {
|
||||
@@ -10997,6 +11002,50 @@ mod tests {
|
||||
(r.status, r.error.clone())
|
||||
}
|
||||
|
||||
/// A batch delete only needs the needle's cookie, which is stored in the
|
||||
/// header, so it must not read the payload.
|
||||
#[tokio::test]
|
||||
async fn batch_delete_reads_needle_meta_only() {
|
||||
use crate::storage::volume::needle_read_hook;
|
||||
use std::sync::Mutex;
|
||||
|
||||
const ID: u64 = 0x6e7a_0b01;
|
||||
let (service, _tmp) = make_local_service_with_volume("", None);
|
||||
let data = vec![7u8; 512 * 1024];
|
||||
let mut n = Needle {
|
||||
id: NeedleId(ID),
|
||||
cookie: Cookie(0x3344),
|
||||
data: data.clone(),
|
||||
data_size: data.len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
service
|
||||
.state
|
||||
.store
|
||||
.write()
|
||||
.unwrap()
|
||||
.write_volume_needle(VolumeId(1), &mut n, false)
|
||||
.unwrap();
|
||||
|
||||
let reads = Arc::new(Mutex::new(Vec::new()));
|
||||
let _hook = needle_read_hook::register(NeedleId(ID), {
|
||||
let reads = reads.clone();
|
||||
move |len| reads.lock().unwrap().push(len)
|
||||
});
|
||||
|
||||
let (status, _) = batch_delete_1(&service, ID, 0x3345, false).await;
|
||||
assert_eq!(status, 400);
|
||||
let (status, error) = batch_delete_1(&service, ID, 0x3344, false).await;
|
||||
assert_eq!(status, 202, "{error}");
|
||||
|
||||
let reads = reads.lock().unwrap().clone();
|
||||
assert!(
|
||||
!reads.is_empty() && reads.iter().all(|&len| len < 4096),
|
||||
"a batch delete must not read the needle's payload ({} bytes): {reads:?}",
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
|
||||
const GO_EC_NEEDLE_NOT_FOUND: &str =
|
||||
"locate in local ec volume: FindNeedleFromEcx: needle not found";
|
||||
|
||||
|
||||
+1416
-208
File diff suppressed because it is too large
Load Diff
@@ -736,6 +736,20 @@ impl Store {
|
||||
vol.read_needle(n)
|
||||
}
|
||||
|
||||
/// Read a needle from a volume using the given read option.
|
||||
///
|
||||
/// On return, `read_option.is_meta_only` tells the caller whether the
|
||||
/// payload was skipped.
|
||||
pub fn read_volume_needle_with_option(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
n: &mut Needle,
|
||||
read_option: &mut ReadOption,
|
||||
) -> Result<i32, VolumeError> {
|
||||
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
|
||||
vol.read_needle_with_option(n, read_option)
|
||||
}
|
||||
|
||||
/// Read a needle from a volume, optionally reading deleted needles.
|
||||
pub fn read_volume_needle_opt(
|
||||
&self,
|
||||
|
||||
@@ -2009,12 +2009,29 @@ impl Volume {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
match self.read_needle_data_at_unlocked(
|
||||
n,
|
||||
nv.offset.to_actual_offset(),
|
||||
read_size,
|
||||
read_option,
|
||||
) {
|
||||
// With `must_meta_only`, read the header and trailer first. If the
|
||||
// needle is compressed or is a chunk manifest, the caller needs its
|
||||
// data, so read the payload as well.
|
||||
//
|
||||
// A version 1 needle has no data size field and no metadata after its
|
||||
// data, so the metadata reader would treat the whole payload as
|
||||
// metadata. It is always read in full.
|
||||
let offset = nv.offset.to_actual_offset();
|
||||
// `is_meta_only` is a result: clear it in case the option is reused.
|
||||
read_option.is_meta_only = false;
|
||||
let meta_first = read_option.must_meta_only && self.version() != VERSION_1;
|
||||
let mut read = if meta_first {
|
||||
self.read_needle_meta_at_unlocked(n, offset, read_size)
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
if read.is_ok() && meta_first && !n.is_compressed() && !n.is_chunk_manifest() {
|
||||
read_option.is_meta_only = true;
|
||||
}
|
||||
if read.is_ok() && !read_option.is_meta_only {
|
||||
read = self.read_needle_data_at_unlocked(n, offset, read_size, read_option);
|
||||
}
|
||||
match read {
|
||||
Ok(()) => self.check_read_write_error(None),
|
||||
Err(VolumeError::Io(ref e)) => {
|
||||
self.check_read_write_error(Some(e));
|
||||
@@ -6080,6 +6097,120 @@ mod tests {
|
||||
assert_eq!(read_n.cookie, Cookie(0x12345678));
|
||||
}
|
||||
|
||||
/// With `must_meta_only` set, the payload is skipped whatever the needle's
|
||||
/// size. This is the option a delete uses. A chunk manifest is the
|
||||
/// exception: its data is still read, because a delete needs it to find
|
||||
/// the chunks.
|
||||
#[test]
|
||||
fn test_read_needle_must_meta_only() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut v = make_test_volume(tmp.path().to_str().unwrap());
|
||||
let data = vec![7u8; 8192];
|
||||
let mut plain = Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x1234),
|
||||
data: data.clone(),
|
||||
data_size: data.len() as u32,
|
||||
..Needle::default()
|
||||
};
|
||||
v.write_needle(&mut plain, true, false).unwrap();
|
||||
let mut manifest = Needle {
|
||||
id: NeedleId(2),
|
||||
data: b"{}".to_vec(),
|
||||
data_size: 2,
|
||||
..Needle::default()
|
||||
};
|
||||
manifest.set_is_chunk_manifest();
|
||||
v.write_needle(&mut manifest, true, false).unwrap();
|
||||
|
||||
let read = |id: u64, read_option: &mut ReadOption| {
|
||||
let mut n = Needle {
|
||||
id: NeedleId(id),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle_with_option(&mut n, read_option).unwrap();
|
||||
n
|
||||
};
|
||||
let meta_only = || ReadOption {
|
||||
attempt_meta_only: true,
|
||||
must_meta_only: true,
|
||||
..ReadOption::default()
|
||||
};
|
||||
|
||||
let mut read_option = meta_only();
|
||||
let n = read(1, &mut read_option);
|
||||
assert!(read_option.is_meta_only);
|
||||
assert!(n.data.is_empty(), "{} bytes of data read", n.data.len());
|
||||
assert_eq!(n.cookie, Cookie(0x1234));
|
||||
assert_eq!(n.data_size as usize, data.len());
|
||||
|
||||
// Without `must_meta_only`, the needle is read in full.
|
||||
let mut read_option = ReadOption::default();
|
||||
let n = read(1, &mut read_option);
|
||||
assert!(!read_option.is_meta_only);
|
||||
assert_eq!(n.data, data);
|
||||
|
||||
let mut read_option = meta_only();
|
||||
let n = read(2, &mut read_option);
|
||||
assert!(!read_option.is_meta_only);
|
||||
assert_eq!(n.data, b"{}");
|
||||
|
||||
// The option is reused: a metadata-only result from the first read
|
||||
// must not carry over and skip the chunk manifest's data.
|
||||
let mut read_option = meta_only();
|
||||
read(1, &mut read_option);
|
||||
assert!(read_option.is_meta_only);
|
||||
let n = read(2, &mut read_option);
|
||||
assert!(!read_option.is_meta_only);
|
||||
assert_eq!(n.data, b"{}");
|
||||
}
|
||||
|
||||
/// A version 1 needle is a header, the data and a checksum: it has no data
|
||||
/// size field and no metadata after the data, so the metadata reader would
|
||||
/// take the whole payload for metadata and reject anything over 128 KiB.
|
||||
/// A read with `must_meta_only` must fall back to a full read.
|
||||
#[test]
|
||||
fn test_read_needle_must_meta_only_version_1() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let spec = VolumeSpec {
|
||||
version: VERSION_1,
|
||||
..VolumeSpec::default()
|
||||
};
|
||||
let mut v = Volume::new(dir, dir, VolumeId(1), NeedleMapKind::InMemory, &spec).unwrap();
|
||||
assert_eq!(v.version(), VERSION_1);
|
||||
|
||||
for (id, len) in [(1u64, 11usize), (2, 256 * 1024)] {
|
||||
let data: Vec<u8> = b"hello world".iter().copied().cycle().take(len).collect();
|
||||
// The server only writes version 2 and 3 records, so build the
|
||||
// version 1 record by hand.
|
||||
let size = Size(len as i32);
|
||||
let mut blob = vec![0u8; NEEDLE_HEADER_SIZE];
|
||||
Cookie(0x1234).to_bytes(&mut blob[0..COOKIE_SIZE]);
|
||||
NeedleId(id).to_bytes(&mut blob[COOKIE_SIZE..COOKIE_SIZE + NEEDLE_ID_SIZE]);
|
||||
size.to_bytes(&mut blob[COOKIE_SIZE + NEEDLE_ID_SIZE..NEEDLE_HEADER_SIZE]);
|
||||
blob.extend_from_slice(&data);
|
||||
blob.extend_from_slice(&CRC::new(&data).0.to_be_bytes());
|
||||
blob.resize(get_actual_size(size, VERSION_1) as usize, 0);
|
||||
v.write_needle_blob_and_index(NeedleId(id), &blob, size)
|
||||
.unwrap();
|
||||
|
||||
let mut n = Needle {
|
||||
id: NeedleId(id),
|
||||
..Needle::default()
|
||||
};
|
||||
let mut read_option = ReadOption {
|
||||
attempt_meta_only: true,
|
||||
must_meta_only: true,
|
||||
..ReadOption::default()
|
||||
};
|
||||
v.read_needle_with_option(&mut n, &mut read_option).unwrap();
|
||||
assert!(!read_option.is_meta_only);
|
||||
assert_eq!(n.cookie, Cookie(0x1234));
|
||||
assert_eq!(n.data, data, "{len} byte needle");
|
||||
}
|
||||
}
|
||||
|
||||
/// A durable write goes down the same path and lands the same data; the
|
||||
/// flush is extra work, not different work.
|
||||
#[test]
|
||||
|
||||
Generated
+11
-2
@@ -485,6 +485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"untrusted 0.7.1",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -5124,7 +5125,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom 0.2.17",
|
||||
"libc",
|
||||
"untrusted",
|
||||
"untrusted 0.9.0",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
@@ -5260,7 +5261,7 @@ dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
"untrusted 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5342,7 +5343,9 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"prometheus",
|
||||
"prost",
|
||||
"prost-types",
|
||||
@@ -6213,6 +6216,12 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
|
||||
@@ -12,6 +12,10 @@ name = "seaweed_worker_core"
|
||||
# Helpers the Rust volume server needs as well. A path dependency because
|
||||
# the two trees are separate cargo workspaces with no common root manifest.
|
||||
seaweed-common = { path = "../../../seaweed-common" }
|
||||
# Already in the graph through tonic's `tls-aws-lc` feature; named here so the
|
||||
# admin token can sign with the same HMAC the Go worker uses.
|
||||
aws-lc-rs = "1"
|
||||
base64 = "0.22"
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
prost.workspace = true
|
||||
|
||||
@@ -17,6 +17,15 @@ pub struct WorkerOptions {
|
||||
/// section of security.toml. None means plaintext, which is the default the
|
||||
/// Go worker also takes when no certificates are configured.
|
||||
pub tls: Option<TlsOptions>,
|
||||
/// The shared secret the admin stream requires when the cluster configures
|
||||
/// `jwt.filer_signing.key`; mirrored from that config value. Empty means no
|
||||
/// token is attached, which only works against admins that either bound
|
||||
/// their worker listener to loopback or authenticate with mTLS instead.
|
||||
pub signing_key: Option<String>,
|
||||
/// Seconds the minted admin token lives, mirroring
|
||||
/// `jwt.filer_signing.expires_after_seconds`. Zero mints a token without an
|
||||
/// expiry, matching the Go default.
|
||||
pub signing_key_expires_after_seconds: u64,
|
||||
}
|
||||
|
||||
/// Certificates for the control stream. All three are required together: the
|
||||
@@ -44,6 +53,8 @@ impl Default for WorkerOptions {
|
||||
max_detection_concurrency: 1,
|
||||
max_execution_concurrency: 1,
|
||||
tls: None,
|
||||
signing_key: None,
|
||||
signing_key_expires_after_seconds: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ pub mod metrics;
|
||||
pub mod registry;
|
||||
pub mod senders;
|
||||
pub mod stream;
|
||||
pub mod token;
|
||||
|
||||
/// Generated plugin.proto types.
|
||||
pub mod pb {
|
||||
|
||||
@@ -176,10 +176,19 @@ async fn serve_once(
|
||||
metadata: Default::default(),
|
||||
}))?;
|
||||
|
||||
let mut inbound = client
|
||||
.worker_stream(UnboundedReceiverStream::new(rx))
|
||||
.await?
|
||||
.into_inner();
|
||||
let mut request = tonic::Request::new(UnboundedReceiverStream::new(rx));
|
||||
if let Some(token) = crate::token::admin_token(
|
||||
options.signing_key.as_deref().unwrap_or("").as_bytes(),
|
||||
options.signing_key_expires_after_seconds,
|
||||
) {
|
||||
request.metadata_mut().insert(
|
||||
"authorization",
|
||||
format!("Bearer {token}")
|
||||
.parse()
|
||||
.context("minted admin token is not valid metadata")?,
|
||||
);
|
||||
}
|
||||
let mut inbound = client.worker_stream(request).await?.into_inner();
|
||||
|
||||
let heartbeat = spawn_heartbeat(
|
||||
sender.clone(),
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aws_lc_rs::hmac;
|
||||
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
|
||||
|
||||
/// Mints the HS256 Bearer token the admin control stream requires when
|
||||
/// jwt.filer_signing.key is configured, matching the SeaweedFilerAdminClaims
|
||||
/// the Go worker sends: registered claims only, no session id.
|
||||
pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option<String> {
|
||||
if signing_key.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let claims = if expires_after_secs > 0 {
|
||||
let exp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or_default()
|
||||
+ expires_after_secs;
|
||||
format!("{{\"exp\":{exp}}}")
|
||||
} else {
|
||||
"{}".to_string()
|
||||
};
|
||||
let signing_input = format!(
|
||||
"{}.{}",
|
||||
URL_SAFE_NO_PAD.encode(b"{\"alg\":\"HS256\",\"typ\":\"JWT\"}"),
|
||||
URL_SAFE_NO_PAD.encode(claims.as_bytes())
|
||||
);
|
||||
let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key);
|
||||
let signature = hmac::sign(&key, signing_input.as_bytes());
|
||||
Some(format!(
|
||||
"{signing_input}.{}",
|
||||
URL_SAFE_NO_PAD.encode(signature.as_ref())
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::admin_token;
|
||||
|
||||
#[test]
|
||||
fn no_key_no_token() {
|
||||
assert!(admin_token(b"", 0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minted_token_verifies_shape() {
|
||||
let token = admin_token(b"secret", 60).unwrap();
|
||||
let parts: Vec<&str> = token.split('.').collect();
|
||||
assert_eq!(parts.len(), 3);
|
||||
assert!(!parts.iter().any(|p| p.is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_expiry_has_empty_claims() {
|
||||
let token = admin_token(b"secret", 0).unwrap();
|
||||
let payload = token.split('.').nth(1).unwrap();
|
||||
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
|
||||
let claims = URL_SAFE_NO_PAD.decode(payload).unwrap();
|
||||
assert_eq!(claims, b"{}");
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,18 @@ struct Args {
|
||||
#[arg(long)]
|
||||
tls_server_name: Option<String>,
|
||||
|
||||
/// Shared secret for the admin control stream, the `jwt.filer_signing.key`
|
||||
/// value the Go worker reads from its config file. Required by admins that
|
||||
/// authenticate the worker stream with a signing key.
|
||||
#[arg(long, env = "WEED_FILER_SIGNING_KEY")]
|
||||
filer_signing_key: Option<String>,
|
||||
|
||||
/// Lifetime of the token minted from the signing key, mirroring
|
||||
/// `jwt.filer_signing.expires_after_seconds`. Zero mints a token with no
|
||||
/// expiry.
|
||||
#[arg(long, default_value = "0")]
|
||||
filer_signing_key_expires_after_seconds: u64,
|
||||
|
||||
/// Serve /health, /ready and /metrics on this port, the way
|
||||
/// `weed worker -metricsPort` does. Zero, the default, serves nothing.
|
||||
#[arg(long, default_value = "0", env = "WEED_METRICS_PORT")]
|
||||
@@ -131,6 +143,8 @@ async fn main() -> Result<()> {
|
||||
max_detection_concurrency: args.max_concurrency,
|
||||
max_execution_concurrency: args.max_concurrency,
|
||||
tls,
|
||||
signing_key: args.filer_signing_key,
|
||||
signing_key_expires_after_seconds: args.filer_signing_key_expires_after_seconds,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_p2p
|
||||
|
||||
import (
|
||||
|
||||
@@ -26,6 +26,10 @@ const (
|
||||
ucAPIBase = "/api/2.1/unity-catalog"
|
||||
ucWarehouse = "lakehouse"
|
||||
ucWarehouseKey = "warehouse"
|
||||
// A non-empty s3.sessionToken.N puts UC's credential vendor on its
|
||||
// StaticAwsCredentialGenerator path: it vends the configured access key,
|
||||
// secret key and this token verbatim instead of calling AWS STS.
|
||||
ucVendedSessionToken = "unity-catalog-vended-session-token"
|
||||
|
||||
// Role used by the master-role STS-vended variant of the test. The trust
|
||||
// policy is wide open so any caller can assume it; in production UC
|
||||
@@ -65,6 +69,9 @@ type ucServerOpts struct {
|
||||
// MasterRoleArn populates aws.masterRoleArn. Empty means UC falls back to
|
||||
// static aws.accessKey / aws.secretKey for storage operations.
|
||||
MasterRoleArn string
|
||||
// SessionToken populates s3.sessionToken.0, selecting UC's static
|
||||
// credential generator for the per-bucket config.
|
||||
SessionToken string
|
||||
// ExtraEnv adds environment variables to the UC container, useful for
|
||||
// AWS_ENDPOINT_URL_STS-style overrides.
|
||||
ExtraEnv map[string]string
|
||||
@@ -220,6 +227,7 @@ func (env *testEnv) startUnityCatalog(t *testing.T, ctx context.Context, opts uc
|
||||
fmt.Sprintf("s3.awsRoleArn.0=%s", opts.MasterRoleArn),
|
||||
fmt.Sprintf("s3.accessKey.0=%s", env.accessKey),
|
||||
fmt.Sprintf("s3.secretKey.0=%s", env.secretKey),
|
||||
fmt.Sprintf("s3.sessionToken.0=%s", opts.SessionToken),
|
||||
fmt.Sprintf("s3.endpoint.0=%s", s3EndpointForContainer),
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
@@ -8,6 +8,7 @@ package unity_catalog
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -46,7 +47,7 @@ func TestUnityCatalogDeltaIntegration(t *testing.T) {
|
||||
}
|
||||
|
||||
t.Log(">>> starting Unity Catalog server (static keys)...")
|
||||
env.startUnityCatalog(t, ctx, ucServerOpts{})
|
||||
env.startUnityCatalog(t, ctx, ucServerOpts{SessionToken: ucVendedSessionToken})
|
||||
t.Log(">>> Unity Catalog ready")
|
||||
|
||||
uc := newUCClient(fmt.Sprintf("http://127.0.0.1:%d", env.ucHostPort))
|
||||
@@ -145,28 +146,41 @@ func TestUnityCatalogDeltaIntegration(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("TemporaryTableCredentialsRejected", func(t *testing.T) {
|
||||
// With aws.masterRoleArn empty AND no s3.sessionToken.0 set, UC OSS
|
||||
// always tries to AssumeRole via its internal StsClient (see
|
||||
// AwsCredentialVendor.createPerBucketCredentialGenerator). Against a
|
||||
// non-AWS endpoint, that call doesn't reach a real STS, so UC returns
|
||||
// "S3 bucket configuration not found." or an STS-side error. This is
|
||||
// the gap users hit at <https://github.com/data-engineering-helpers/mds-in-a-box/blob/main/unitycatalog-playground/etc/conf/server.properties#L45>:
|
||||
// "with simple S3 access and secret keys, Unity Catalog does not seem
|
||||
// to work."
|
||||
//
|
||||
// The assertion is therefore inverted: we expect a non-nil error from
|
||||
// /temporary-table-credentials with this configuration. A future
|
||||
// variant can pin s3.sessionToken.0 (UC's StaticAwsCredentialGenerator
|
||||
// path) once SeaweedFS' SigV4 path tolerates the vended session token.
|
||||
t.Run("TemporaryTableCredentials", func(t *testing.T) {
|
||||
// s3.sessionToken.0 is set, so UC's AwsCredentialVendor takes the
|
||||
// StaticAwsCredentialGenerator path and vends the configured keys
|
||||
// verbatim, token included — no STS call at all. SeaweedFS authenticates
|
||||
// the request on the static access key and tolerates the foreign token.
|
||||
if createdTable.TableID == "" {
|
||||
t.Fatalf("created table has empty table_id; cannot request temporary credentials")
|
||||
}
|
||||
_, err := uc.generateTemporaryTableCredentials(ctx, createdTable.TableID, "READ_WRITE")
|
||||
if err == nil {
|
||||
t.Fatalf("expected /temporary-table-credentials to fail with the static-key playground configuration; it succeeded unexpectedly")
|
||||
creds, err := uc.generateTemporaryTableCredentials(ctx, createdTable.TableID, "READ_WRITE")
|
||||
if err != nil {
|
||||
t.Fatalf("temporary-table-credentials: %v", err)
|
||||
}
|
||||
awsCreds := creds.AwsTempCredentials
|
||||
if awsCreds == nil || awsCreds.AccessKeyID == "" || awsCreds.SessionToken == "" {
|
||||
t.Fatalf("expected aws_temp_credentials with a session_token, got %+v", creds)
|
||||
}
|
||||
if awsCreds.AccessKeyID != env.accessKey || awsCreds.SessionToken != ucVendedSessionToken {
|
||||
t.Fatalf("expected UC to vend the configured static credential, got %+v", awsCreds)
|
||||
}
|
||||
|
||||
s3v := env.newHostS3ClientWithCreds(t, ctx, awsCreds.AccessKeyID, awsCreds.SecretAccessKey, awsCreds.SessionToken)
|
||||
probeKey := fmt.Sprintf("%s/%s/%s/vended.txt", ucWarehouseKey, schemaName, tableName)
|
||||
if _, err := s3v.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(ucWarehouse),
|
||||
Key: aws.String(probeKey),
|
||||
Body: strings.NewReader("ok"),
|
||||
}); err != nil {
|
||||
t.Fatalf("PutObject with UC-vended credentials: %v", err)
|
||||
}
|
||||
if _, err := s3v.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(ucWarehouse),
|
||||
Key: aws.String(probeKey),
|
||||
}); err != nil {
|
||||
t.Fatalf("DeleteObject with UC-vended credentials: %v", err)
|
||||
}
|
||||
t.Logf("expected failure (UC static-key path requires AWS STS): %v", err)
|
||||
})
|
||||
|
||||
t.Run("DeleteTableSchemaCatalog", func(t *testing.T) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package sftp
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
@@ -602,6 +603,32 @@ func TestFileContent(t *testing.T) {
|
||||
sftpClient.Remove("/binary.bin")
|
||||
})
|
||||
|
||||
t.Run("WriteAtOffsetKeepsExistingBytes", func(t *testing.T) {
|
||||
file, err := sftpClient.Create("/resume.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = file.Write([]byte("hello world"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, file.Close())
|
||||
|
||||
// Reopen without O_TRUNC and write past the start, as a resumed upload does
|
||||
file, err = sftpClient.OpenFile("/resume.txt", os.O_WRONLY)
|
||||
require.NoError(t, err)
|
||||
_, err = file.WriteAt([]byte("WORLD!"), 6)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, file.Close())
|
||||
|
||||
readFile, err := sftpClient.Open("/resume.txt")
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(readFile)
|
||||
require.NoError(t, err)
|
||||
readFile.Close()
|
||||
|
||||
require.Equal(t, "hello WORLD!", string(content))
|
||||
|
||||
// Clean up
|
||||
sftpClient.Remove("/resume.txt")
|
||||
})
|
||||
|
||||
t.Run("EmptyFile", func(t *testing.T) {
|
||||
file, err := sftpClient.Create("/empty.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -188,6 +188,10 @@ func (s *WorkerGrpcServer) Stop() error {
|
||||
func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStreamServer) error {
|
||||
ctx := stream.Context()
|
||||
|
||||
if err := security.CheckGrpcAdminPeer(ctx, security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key"))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// get client address
|
||||
address := findClientAddress(ctx)
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// JobReplicaHosts are the volume-replica hosts a job will read from.
|
||||
// Used so executor pick prefers a worker on one of those machines.
|
||||
// Sources with ShardIds are EC-cleanup rows, not encode replicas.
|
||||
func JobReplicaHosts(job *plugin_pb.JobSpec) []string {
|
||||
if job == nil {
|
||||
return nil
|
||||
}
|
||||
var addrs []string
|
||||
addrs = append(addrs, pluginworker.ReadStringListConfig(job.Parameters, "replica_servers")...)
|
||||
if src := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", "")); src != "" {
|
||||
addrs = append(addrs, src)
|
||||
}
|
||||
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
|
||||
params := &worker_pb.TaskParams{}
|
||||
if err := proto.Unmarshal(payload, params); err == nil {
|
||||
for _, s := range params.GetSources() {
|
||||
if s == nil || s.Node == "" || len(s.ShardIds) > 0 {
|
||||
continue
|
||||
}
|
||||
addrs = append(addrs, s.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
return util.HostKeysFromAddresses(addrs)
|
||||
}
|
||||
|
||||
// partitionByPreferredHost splits candidates into (host matches, others).
|
||||
// No preferred hosts, or no matches, returns (nil, all) so callers keep
|
||||
// the existing slots+round-robin path.
|
||||
func partitionByPreferredHost(candidates []*WorkerSession, preferredHosts []string) (pref, rest []*WorkerSession) {
|
||||
want := util.HostKeysFromAddresses(preferredHosts)
|
||||
if len(want) == 0 || len(candidates) == 0 {
|
||||
return nil, candidates
|
||||
}
|
||||
set := make(map[string]struct{}, len(want))
|
||||
for _, h := range want {
|
||||
set[h] = struct{}{}
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := set[util.HostKey(c.Address)]; ok {
|
||||
pref = append(pref, c)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, c)
|
||||
}
|
||||
if len(pref) == 0 {
|
||||
return nil, candidates
|
||||
}
|
||||
return pref, rest
|
||||
}
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
@@ -267,6 +269,10 @@ func (r *Plugin) Shutdown() {
|
||||
}
|
||||
|
||||
func (r *Plugin) WorkerStream(stream plugin_pb.PluginControlService_WorkerStreamServer) error {
|
||||
if err := security.CheckGrpcAdminPeer(stream.Context(), security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key"))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
first, err := stream.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
@@ -609,7 +615,7 @@ func (r *Plugin) ExecuteJob(
|
||||
releaseGate := r.acquireDetectionExecutionGate(jobType, true)
|
||||
defer releaseGate()
|
||||
|
||||
executor, err := r.registry.PickExecutor(jobType)
|
||||
executor, err := r.registry.PickExecutor(jobType, JobReplicaHosts(job)...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -887,6 +887,37 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
|
||||
})
|
||||
}
|
||||
|
||||
// bindQueuedJobWorker records the home worker while the job stays pending/queued.
|
||||
func (r *Plugin) bindQueuedJobWorker(job *plugin_pb.JobSpec, workerID string) {
|
||||
if job == nil || strings.TrimSpace(job.JobId) == "" || strings.TrimSpace(workerID) == "" {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
r.jobsMu.Lock()
|
||||
tracked := r.jobs[job.JobId]
|
||||
if tracked == nil {
|
||||
r.jobsMu.Unlock()
|
||||
return
|
||||
}
|
||||
if isTerminalTrackedJobState(tracked.State) {
|
||||
r.jobsMu.Unlock()
|
||||
return
|
||||
}
|
||||
tracked.WorkerID = workerID
|
||||
tracked.Message = "queued on worker " + workerID
|
||||
tracked.UpdatedAt = timeToPtr(now)
|
||||
snap := cloneTrackedJob(*tracked)
|
||||
r.dirtyJobs = true
|
||||
r.jobsMu.Unlock()
|
||||
|
||||
r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
|
||||
detail.WorkerID = snap.WorkerID
|
||||
detail.Message = snap.Message
|
||||
detail.UpdatedAt = snap.UpdatedAt
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
|
||||
reason := "job canceled"
|
||||
if cause != nil {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -28,12 +29,18 @@ const (
|
||||
defaultScheduledMaxResults int32 = 1000
|
||||
defaultScheduledExecutionConcurrency = 1
|
||||
defaultScheduledPerWorkerConcurrency = 1
|
||||
maxScheduledExecutionConcurrency = 128
|
||||
defaultScheduledRetryBackoff = 5 * time.Second
|
||||
defaultClusterContextTimeout = 10 * time.Second
|
||||
defaultWaitingBacklogFloor = 8
|
||||
defaultWaitingBacklogMultiplier = 4
|
||||
maxEstimatedRuntimeCap = 8 * time.Hour
|
||||
// Hold a job on replica-host workers until their reservation load
|
||||
// reaches this; 0 disables the pin and spills immediately.
|
||||
defaultLocalQueuePin = 5
|
||||
// Wait this long after a home worker disappears before stealing
|
||||
// its queue; 0 steals immediately. Covers a reconnect blink.
|
||||
defaultOrphanRehomeGrace = 30 * time.Second
|
||||
maxScheduledExecutionConcurrency = 128
|
||||
defaultScheduledRetryBackoff = 5 * time.Second
|
||||
defaultClusterContextTimeout = 10 * time.Second
|
||||
defaultWaitingBacklogFloor = 8
|
||||
defaultWaitingBacklogMultiplier = 4
|
||||
maxEstimatedRuntimeCap = 8 * time.Hour
|
||||
// How long started jobs may drain past the run window close.
|
||||
scheduledExecutionDrainGrace = 30 * time.Minute
|
||||
)
|
||||
@@ -49,6 +56,12 @@ type schedulerPolicy struct {
|
||||
PerWorkerConcurrency int
|
||||
RetryLimit int
|
||||
ExecutorReserveBackoff time.Duration
|
||||
// LocalQueuePin is K: do not spill off replica-host workers while
|
||||
// min(load(pref)) < K. Zero is the old immediate-spill path.
|
||||
LocalQueuePin int
|
||||
// OrphanRehomeGrace is how long a missing home's queue is held
|
||||
// before steal-without-K. Zero steals as soon as the worker is gone.
|
||||
OrphanRehomeGrace time.Duration
|
||||
}
|
||||
|
||||
// laneSchedulerLoop is the main scheduling goroutine for a single lane.
|
||||
@@ -466,6 +479,8 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
|
||||
PerWorkerConcurrency: int(adminRuntime.PerWorkerExecutionConcurrency),
|
||||
RetryLimit: int(adminRuntime.RetryLimit),
|
||||
ExecutorReserveBackoff: 200 * time.Millisecond,
|
||||
LocalQueuePin: defaultLocalQueuePin,
|
||||
OrphanRehomeGrace: defaultOrphanRehomeGrace,
|
||||
}
|
||||
|
||||
if policy.DetectionInterval < r.schedulerTick {
|
||||
@@ -496,6 +511,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
|
||||
if policy.ExecutionTimeout < defaultScheduledExecutionTimeout {
|
||||
policy.ExecutionTimeout = defaultScheduledExecutionTimeout
|
||||
}
|
||||
if policy.LocalQueuePin < 0 {
|
||||
policy.LocalQueuePin = 0
|
||||
}
|
||||
|
||||
return policy, true, nil
|
||||
}
|
||||
@@ -813,19 +831,22 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
// whole dispatch lets manual shell operations interleave between jobs.
|
||||
jobsRequireLock := LaneRequiresLock(JobTypeLane(jobType))
|
||||
|
||||
jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, len(proposals))
|
||||
for index, proposal := range proposals {
|
||||
job := buildScheduledJobSpec(jobType, proposal, index)
|
||||
r.trackExecutionQueued(job)
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
close(jobQueue)
|
||||
return 0, 0, 0
|
||||
for _, job := range jobs {
|
||||
r.cancelQueuedJob(job, errSchedulerShutdown)
|
||||
}
|
||||
return 0, 0, len(jobs)
|
||||
default:
|
||||
jobQueue <- job
|
||||
}
|
||||
job := buildScheduledJobSpec(jobType, proposal, index)
|
||||
r.trackExecutionQueued(job)
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
close(jobQueue)
|
||||
|
||||
queues := r.assignJobsToWorkerQueues(jobType, jobs, policy)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var statsMu sync.Mutex
|
||||
@@ -842,130 +863,110 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
jobLoop:
|
||||
for job := range jobQueue {
|
||||
for {
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
return
|
||||
}
|
||||
if queues.isEmpty() {
|
||||
return
|
||||
}
|
||||
|
||||
item, targetID := r.takeQueuedJob(queues, jobType, policy)
|
||||
if item == nil {
|
||||
if queues.isEmpty() {
|
||||
return
|
||||
}
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
queues.pushFront(item)
|
||||
return
|
||||
}
|
||||
|
||||
executor, release, reserveErr := r.reserveExecutorByID(jobType, policy, targetID)
|
||||
if reserveErr != nil {
|
||||
queues.pushFront(item)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
return
|
||||
default:
|
||||
var releaseJobLock func()
|
||||
if jobsRequireLock {
|
||||
var lockErr error
|
||||
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, item.job.JobId))
|
||||
if lockErr != nil {
|
||||
release()
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: item.job.JobId,
|
||||
JobType: item.job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
releaseJobLock()
|
||||
release()
|
||||
queues.pushFront(item)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err := r.executeScheduledJobWithExecutor(ctx, executor, item.job, clusterContext, policy)
|
||||
if releaseJobLock != nil {
|
||||
releaseJobLock()
|
||||
}
|
||||
release()
|
||||
if errors.Is(err, errExecutorAtCapacity) {
|
||||
queues.pushFront(item)
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
r.cancelQueuedJob(item.job, err)
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
|
||||
executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy)
|
||||
if reserveErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobType: jobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution reservation failed: %v", reserveErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
var releaseJobLock func()
|
||||
if jobsRequireLock {
|
||||
var lockErr error
|
||||
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, job.JobId))
|
||||
if lockErr != nil {
|
||||
release()
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: job.JobId,
|
||||
JobType: job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
// The lock may have been held by an operator for a
|
||||
// while; re-check the execution window.
|
||||
if ctx.Err() != nil {
|
||||
releaseJobLock()
|
||||
release()
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
}
|
||||
|
||||
err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy)
|
||||
if releaseJobLock != nil {
|
||||
releaseJobLock()
|
||||
}
|
||||
release()
|
||||
if errors.Is(err, errExecutorAtCapacity) {
|
||||
r.trackExecutionQueued(job)
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
r.cancelQueuedJob(job, err)
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: job.JobId,
|
||||
JobType: job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution failed: %v", err),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
statsMu.Lock()
|
||||
successCount++
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
break
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: item.job.JobId,
|
||||
JobType: item.job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution failed: %v", err),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
statsMu.Lock()
|
||||
successCount++
|
||||
statsMu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -976,8 +977,8 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
if drainErr == nil {
|
||||
drainErr = errSchedulerShutdown
|
||||
}
|
||||
for job := range jobQueue {
|
||||
r.cancelQueuedJob(job, drainErr)
|
||||
for _, item := range queues.drain() {
|
||||
r.cancelQueuedJob(item.job, drainErr)
|
||||
canceledCount++
|
||||
}
|
||||
|
||||
@@ -988,6 +989,7 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
ctx context.Context,
|
||||
jobType string,
|
||||
policy schedulerPolicy,
|
||||
preferredHosts []string,
|
||||
) (*WorkerSession, func(), error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
@@ -1015,7 +1017,7 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
|
||||
}
|
||||
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
executors, err := r.registry.ListExecutors(jobType, preferredHosts...)
|
||||
if err != nil {
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
if ctx.Err() != nil {
|
||||
@@ -1026,11 +1028,19 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
continue
|
||||
}
|
||||
|
||||
for _, executor := range executors {
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if !ok {
|
||||
continue
|
||||
pref, rest := partitionByPreferredHost(executors, preferredHosts)
|
||||
lane := JobTypeLane(jobType)
|
||||
loadOf := func(e *WorkerSession) int { return r.executorReservationLoad(e, lane) }
|
||||
|
||||
if len(pref) > 0 {
|
||||
sortExecutorsByLoad(pref, loadOf)
|
||||
if executor, release, ok := r.tryReserveFirst(pref, jobType, policy); ok {
|
||||
return executor, release, nil
|
||||
}
|
||||
}
|
||||
|
||||
sortExecutorsByLoad(rest, loadOf)
|
||||
if executor, release, ok := r.tryReserveFirst(rest, jobType, policy); ok {
|
||||
return executor, release, nil
|
||||
}
|
||||
|
||||
@@ -1051,6 +1061,75 @@ func (r *Plugin) tryReserveExecutorCapacity(
|
||||
return r.tryReserveExecutorCapacityForLane(executor, jobType, policy, JobTypeLane(jobType))
|
||||
}
|
||||
|
||||
func (r *Plugin) tryReserveFirst(executors []*WorkerSession, jobType string, policy schedulerPolicy) (*WorkerSession, func(), bool) {
|
||||
for _, executor := range executors {
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if ok {
|
||||
return executor, release, true
|
||||
}
|
||||
}
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
func (r *Plugin) reserveExecutorByID(jobType string, policy schedulerPolicy, workerID string) (*WorkerSession, func(), error) {
|
||||
workerID = strings.TrimSpace(workerID)
|
||||
if workerID == "" {
|
||||
return nil, nil, fmt.Errorf("executor id is empty")
|
||||
}
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, executor := range executors {
|
||||
if executor == nil || executor.WorkerID != workerID {
|
||||
continue
|
||||
}
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if !ok {
|
||||
return nil, nil, errExecutorAtCapacity
|
||||
}
|
||||
return executor, release, nil
|
||||
}
|
||||
return nil, nil, fmt.Errorf("executor %s is not available for %s", workerID, jobType)
|
||||
}
|
||||
|
||||
func (r *Plugin) executorReservationLoad(executor *WorkerSession, lane SchedulerLane) int {
|
||||
if executor == nil {
|
||||
return 0
|
||||
}
|
||||
load := 0
|
||||
if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsUsed > 0 {
|
||||
load = int(executor.Heartbeat.ExecutionSlotsUsed)
|
||||
}
|
||||
workerID := strings.TrimSpace(executor.WorkerID)
|
||||
if ls := r.lanes[lane]; ls != nil {
|
||||
ls.execMu.Lock()
|
||||
load += ls.execRes[workerID]
|
||||
ls.execMu.Unlock()
|
||||
return load
|
||||
}
|
||||
r.schedulerExecMu.Lock()
|
||||
load += r.schedulerExecReservations[workerID]
|
||||
r.schedulerExecMu.Unlock()
|
||||
return load
|
||||
}
|
||||
|
||||
func sortExecutorsByLoad(executors []*WorkerSession, load func(*WorkerSession) int) {
|
||||
sort.SliceStable(executors, func(i, j int) bool {
|
||||
if executors[i] == nil {
|
||||
return false
|
||||
}
|
||||
if executors[j] == nil {
|
||||
return true
|
||||
}
|
||||
li, lj := load(executors[i]), load(executors[j])
|
||||
if li != lj {
|
||||
return li < lj
|
||||
}
|
||||
return executors[i].WorkerID < executors[j].WorkerID
|
||||
})
|
||||
}
|
||||
|
||||
// tryReserveExecutorCapacityForLane reserves an execution slot on the
|
||||
// per-lane reservation pool so that lanes cannot starve each other.
|
||||
func (r *Plugin) tryReserveExecutorCapacityForLane(
|
||||
@@ -1577,6 +1656,8 @@ func (r *Plugin) DispatchProposals(
|
||||
ExecutionTimeout: defaultScheduledExecutionTimeout,
|
||||
RetryBackoff: defaultScheduledRetryBackoff,
|
||||
ExecutorReserveBackoff: 200 * time.Millisecond,
|
||||
LocalQueuePin: defaultLocalQueuePin,
|
||||
OrphanRehomeGrace: defaultOrphanRehomeGrace,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +61,12 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
|
||||
if policy.JobTypeMaxRuntime != 30*time.Minute {
|
||||
t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
|
||||
}
|
||||
if policy.LocalQueuePin != defaultLocalQueuePin {
|
||||
t.Fatalf("unexpected local queue pin: got=%d", policy.LocalQueuePin)
|
||||
}
|
||||
if policy.OrphanRehomeGrace != defaultOrphanRehomeGrace {
|
||||
t.Fatalf("unexpected orphan rehome grace: got=%v", policy.OrphanRehomeGrace)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
|
||||
@@ -138,13 +144,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
|
||||
ExecutorReserveBackoff: time.Millisecond,
|
||||
}
|
||||
|
||||
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 2: %v", err)
|
||||
}
|
||||
@@ -266,7 +272,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
|
||||
|
||||
start := time.Now()
|
||||
pluginSvc.Shutdown()
|
||||
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
|
||||
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expected reservation shutdown error")
|
||||
}
|
||||
@@ -297,7 +303,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
|
||||
ExecutorReserveBackoff: 5 * time.Millisecond,
|
||||
}
|
||||
|
||||
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 1: %v", err)
|
||||
}
|
||||
@@ -308,7 +314,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
|
||||
}
|
||||
secondReserveCh := make(chan reserveResult, 1)
|
||||
go func() {
|
||||
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
if release2 != nil {
|
||||
release2()
|
||||
}
|
||||
@@ -992,3 +998,50 @@ func TestLaneStatus_LifecycleNextDetectionSurvivesDefaultLanePrune(t *testing.T)
|
||||
status.NextDetectionAt, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveWaterfillsPreferredHosts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
ids := []string{"worker-a", "worker-b", "worker-c", "worker-d"}
|
||||
for _, id := range ids {
|
||||
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: id,
|
||||
Address: "10.0.0.1:9334",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 8,
|
||||
ExecutorReserveBackoff: time.Millisecond,
|
||||
LocalQueuePin: 5,
|
||||
}
|
||||
preferred := []string{"10.0.0.1:8080"}
|
||||
|
||||
seen := map[string]int{}
|
||||
var releases []func()
|
||||
defer func() {
|
||||
for _, release := range releases {
|
||||
release()
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 4; i++ {
|
||||
exec, release, err := pluginSvc.reserveScheduledExecutor(context.Background(), "erasure_coding", policy, preferred)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve %d: %v", i, err)
|
||||
}
|
||||
releases = append(releases, release)
|
||||
seen[exec.WorkerID]++
|
||||
}
|
||||
if len(seen) != 4 {
|
||||
t.Fatalf("want one job per worker, got %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,14 +277,24 @@ func (r *Registry) PickDetector(jobType string) (*WorkerSession, error) {
|
||||
}
|
||||
|
||||
// PickExecutor picks one executor worker for a job type.
|
||||
func (r *Registry) PickExecutor(jobType string) (*WorkerSession, error) {
|
||||
return r.pickByKind(jobType, false)
|
||||
// preferredHosts, when set, ranks workers whose Hello Address host matches a
|
||||
// replica/source host ahead of others (same-machine EC pull). Empty means the
|
||||
// previous slots+round-robin order.
|
||||
func (r *Registry) PickExecutor(jobType string, preferredHosts ...string) (*WorkerSession, error) {
|
||||
list, err := r.ListExecutors(jobType, preferredHosts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list[0], nil
|
||||
}
|
||||
|
||||
// ListExecutors returns sorted executor candidates for one job type.
|
||||
// Ordering is by most available execution slots, then lexical worker ID.
|
||||
// The top tie group is rotated round-robin to prevent sticky assignment.
|
||||
func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
|
||||
// preferredHosts, when any worker matches, are listed first (then the rest);
|
||||
// rotation applies only inside the preferred group so a local worker is not
|
||||
// rotated away in favor of a remote one.
|
||||
func (r *Registry) ListExecutors(jobType string, preferredHosts ...string) ([]*WorkerSession, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -293,8 +303,16 @@ func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
|
||||
return nil, fmt.Errorf("no executor worker available for job_type=%s", jobType)
|
||||
}
|
||||
|
||||
sortByKind(candidates, jobType, false)
|
||||
r.rotateTopCandidatesLocked(candidates, jobType, false)
|
||||
pref, rest := partitionByPreferredHost(candidates, preferredHosts)
|
||||
if len(pref) > 0 {
|
||||
sortByKind(pref, jobType, false)
|
||||
r.rotateTopCandidatesLocked(pref, jobType, false)
|
||||
sortByKind(rest, jobType, false)
|
||||
candidates = append(pref, rest...)
|
||||
} else {
|
||||
sortByKind(candidates, jobType, false)
|
||||
r.rotateTopCandidatesLocked(candidates, jobType, false)
|
||||
}
|
||||
|
||||
out := make([]*WorkerSession, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
|
||||
@@ -401,3 +401,70 @@ func TestRegistryHasCapableWorkerIgnoresNilCapability(t *testing.T) {
|
||||
t.Fatalf("expected HasCapableWorker to return false when capability has no detect or execute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListExecutorsPrefersMatchingReplicaHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
r := NewRegistry()
|
||||
r.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: "worker-remote",
|
||||
Address: "10.87.0.4:33646",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
r.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: "worker-local",
|
||||
Address: "10.87.0.3:33646",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 1},
|
||||
},
|
||||
})
|
||||
r.UpdateHeartbeat("worker-remote", &plugin_pb.WorkerHeartbeat{
|
||||
WorkerId: "worker-remote", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 8,
|
||||
})
|
||||
r.UpdateHeartbeat("worker-local", &plugin_pb.WorkerHeartbeat{
|
||||
WorkerId: "worker-local", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 1,
|
||||
})
|
||||
|
||||
// Without preference the emptier remote worker wins on slots.
|
||||
plain, err := r.ListExecutors("erasure_coding")
|
||||
if err != nil {
|
||||
t.Fatalf("ListExecutors: %v", err)
|
||||
}
|
||||
if plain[0].WorkerID != "worker-remote" {
|
||||
t.Fatalf("without preference want remote first, got %s", plain[0].WorkerID)
|
||||
}
|
||||
|
||||
// Replica on :8080 of the same host as worker-local.
|
||||
pref, err := r.ListExecutors("erasure_coding", "10.87.0.3:8080")
|
||||
if err != nil {
|
||||
t.Fatalf("ListExecutors preferred: %v", err)
|
||||
}
|
||||
if pref[0].WorkerID != "worker-local" {
|
||||
t.Fatalf("preferred host want worker-local first, got %s", pref[0].WorkerID)
|
||||
}
|
||||
if pref[1].WorkerID != "worker-remote" {
|
||||
t.Fatalf("remote should still be fallback, got %s", pref[1].WorkerID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobReplicaHostsFromSourcesAndReplicaServers(t *testing.T) {
|
||||
t.Parallel()
|
||||
job := &plugin_pb.JobSpec{
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "10.87.0.1:8080"},
|
||||
},
|
||||
"replica_servers": {
|
||||
Kind: &plugin_pb.ConfigValue_StringList{
|
||||
StringList: &plugin_pb.StringList{Values: []string{"10.87.0.1:8080", "10.87.0.2:8081"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
got := JobReplicaHosts(job)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
// queuedJob is one scheduled spec parked on a home worker's admin-side queue.
|
||||
type queuedJob struct {
|
||||
job *plugin_pb.JobSpec
|
||||
home string
|
||||
}
|
||||
|
||||
// workerJobQueues holds per-worker FIFOs for one dispatch batch.
|
||||
// Home assignment is done once up front; run goroutines only pop/steal.
|
||||
// leftover is the homeless FIFO: replica hostnames with no currently
|
||||
// registered worker. Orphan grace does not apply here.
|
||||
type workerJobQueues struct {
|
||||
mu sync.Mutex
|
||||
q map[string][]*queuedJob
|
||||
leftover []*queuedJob
|
||||
remaining int
|
||||
missingSince map[string]time.Time
|
||||
homes map[string]struct{}
|
||||
}
|
||||
|
||||
func newWorkerJobQueues() *workerJobQueues {
|
||||
return &workerJobQueues{
|
||||
q: make(map[string][]*queuedJob),
|
||||
missingSince: make(map[string]time.Time),
|
||||
homes: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) hasHome(workerID string) bool {
|
||||
if qs == nil || workerID == "" {
|
||||
return false
|
||||
}
|
||||
_, ok := qs.homes[workerID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) push(home string, job *plugin_pb.JobSpec) {
|
||||
if qs == nil || job == nil {
|
||||
return
|
||||
}
|
||||
item := &queuedJob{job: job, home: home}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
qs.pushLocked(item, false)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) pushFront(item *queuedJob) {
|
||||
if qs == nil || item == nil {
|
||||
return
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
qs.pushLocked(item, true)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) pushLocked(item *queuedJob, front bool) {
|
||||
qs.remaining++
|
||||
if item.home == "" {
|
||||
if front {
|
||||
qs.leftover = append([]*queuedJob{item}, qs.leftover...)
|
||||
return
|
||||
}
|
||||
qs.leftover = append(qs.leftover, item)
|
||||
return
|
||||
}
|
||||
if front {
|
||||
qs.q[item.home] = append([]*queuedJob{item}, qs.q[item.home]...)
|
||||
return
|
||||
}
|
||||
qs.q[item.home] = append(qs.q[item.home], item)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) isEmpty() bool {
|
||||
if qs == nil {
|
||||
return true
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return qs.remaining == 0
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) queueLen(workerID string) int {
|
||||
if qs == nil {
|
||||
return 0
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return len(qs.q[workerID])
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) leftoverLen() int {
|
||||
if qs == nil {
|
||||
return 0
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return len(qs.leftover)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) drain() []*queuedJob {
|
||||
if qs == nil {
|
||||
return nil
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
out := make([]*queuedJob, 0, qs.remaining)
|
||||
for id, qq := range qs.q {
|
||||
out = append(out, qq...)
|
||||
qs.q[id] = nil
|
||||
}
|
||||
out = append(out, qs.leftover...)
|
||||
qs.leftover = nil
|
||||
qs.remaining = 0
|
||||
return out
|
||||
}
|
||||
|
||||
func popFront(qq []*queuedJob) (*queuedJob, []*queuedJob) {
|
||||
item := qq[0]
|
||||
if len(qq) == 1 {
|
||||
return item, nil
|
||||
}
|
||||
return item, qq[1:]
|
||||
}
|
||||
|
||||
// assignJobsToWorkerQueues parks each job on the least-loaded replica-host
|
||||
// worker. A home exists only if a currently registered worker's -address
|
||||
// hostname matches a replica host. Otherwise the job is homeless: global
|
||||
// leftover FIFO, no fake home, no K, no orphan grace. Single-threaded.
|
||||
func (r *Plugin) assignJobsToWorkerQueues(
|
||||
jobType string,
|
||||
jobs []*plugin_pb.JobSpec,
|
||||
_ schedulerPolicy,
|
||||
) *workerJobQueues {
|
||||
qs := newWorkerJobQueues()
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
lane := JobTypeLane(jobType)
|
||||
queued := make(map[string]int, len(executors))
|
||||
|
||||
for _, job := range jobs {
|
||||
if job == nil {
|
||||
continue
|
||||
}
|
||||
if err != nil || len(executors) == 0 {
|
||||
qs.push("", job)
|
||||
continue
|
||||
}
|
||||
pref, _ := partitionByPreferredHost(executors, JobReplicaHosts(job))
|
||||
if len(pref) == 0 {
|
||||
// Never registered for that hostname: homeless, not an orphan.
|
||||
qs.push("", job)
|
||||
continue
|
||||
}
|
||||
for _, e := range pref {
|
||||
if e != nil && e.WorkerID != "" {
|
||||
qs.homes[e.WorkerID] = struct{}{}
|
||||
}
|
||||
}
|
||||
sortExecutorsByLoad(pref, func(e *WorkerSession) int {
|
||||
return r.executorReservationLoad(e, lane) + queued[e.WorkerID]
|
||||
})
|
||||
home := pref[0].WorkerID
|
||||
qs.push(home, job)
|
||||
queued[home]++
|
||||
r.bindQueuedJobWorker(job, home)
|
||||
}
|
||||
return qs
|
||||
}
|
||||
|
||||
// takeQueuedJob pops a job for a free worker.
|
||||
// Home worker (hostname matched a replica / has a home-queue identity):
|
||||
// 1. own home queue
|
||||
// 2. steal from another home if len>=K, or orphan past OrphanRehomeGrace
|
||||
// 3. homeless leftover FIFO
|
||||
// No-home worker (no matching replica, no home-queue identity):
|
||||
// 1. homeless leftover FIFO
|
||||
// 2. only if leftover empty: steal (K / orphan), same rules
|
||||
func (r *Plugin) takeQueuedJob(
|
||||
qs *workerJobQueues,
|
||||
jobType string,
|
||||
policy schedulerPolicy,
|
||||
) (item *queuedJob, targetID string) {
|
||||
if qs == nil {
|
||||
return nil, ""
|
||||
}
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil || len(executors) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
lane := JobTypeLane(jobType)
|
||||
pin := policy.LocalQueuePin
|
||||
defer func() {
|
||||
if item != nil && targetID != "" {
|
||||
r.bindQueuedJobWorker(item.job, targetID)
|
||||
}
|
||||
}()
|
||||
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
if qs.remaining == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
var bestHome *WorkerSession
|
||||
bestLoad := 0
|
||||
for _, exec := range executors {
|
||||
if len(qs.q[exec.WorkerID]) == 0 {
|
||||
continue
|
||||
}
|
||||
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
load := r.executorReservationLoad(exec, lane)
|
||||
if load >= limit {
|
||||
continue
|
||||
}
|
||||
if bestHome == nil || load < bestLoad || (load == bestLoad && exec.WorkerID < bestHome.WorkerID) {
|
||||
bestHome = exec
|
||||
bestLoad = load
|
||||
}
|
||||
}
|
||||
if bestHome != nil {
|
||||
id := bestHome.WorkerID
|
||||
item, qs.q[id] = popFront(qs.q[id])
|
||||
qs.remaining--
|
||||
return item, id
|
||||
}
|
||||
|
||||
var thief *WorkerSession
|
||||
for _, exec := range executors {
|
||||
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
if r.executorReservationLoad(exec, lane) >= limit {
|
||||
continue
|
||||
}
|
||||
if thief == nil || exec.WorkerID < thief.WorkerID {
|
||||
thief = exec
|
||||
}
|
||||
}
|
||||
if thief == nil {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
takeLeftover := func() (*queuedJob, string) {
|
||||
if len(qs.leftover) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
item, qs.leftover = popFront(qs.leftover)
|
||||
qs.remaining--
|
||||
return item, thief.WorkerID
|
||||
}
|
||||
if !qs.hasHome(thief.WorkerID) {
|
||||
if item, id := takeLeftover(); item != nil {
|
||||
return item, id
|
||||
}
|
||||
}
|
||||
|
||||
live := make(map[string]struct{}, len(executors))
|
||||
for _, exec := range executors {
|
||||
live[exec.WorkerID] = struct{}{}
|
||||
delete(qs.missingSince, exec.WorkerID)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
grace := policy.OrphanRehomeGrace
|
||||
victimID := ""
|
||||
victimLen := 0
|
||||
victimOrphan := false
|
||||
for id, qq := range qs.q {
|
||||
if len(qq) == 0 {
|
||||
delete(qs.missingSince, id)
|
||||
continue
|
||||
}
|
||||
_, isLive := live[id]
|
||||
orphan := !isLive
|
||||
if isLive {
|
||||
if pin > 0 && len(qq) < pin {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if _, ok := qs.missingSince[id]; !ok {
|
||||
qs.missingSince[id] = now
|
||||
}
|
||||
if grace > 0 && now.Sub(qs.missingSince[id]) < grace {
|
||||
continue
|
||||
}
|
||||
}
|
||||
better := victimID == ""
|
||||
if !better && orphan != victimOrphan {
|
||||
better = orphan
|
||||
}
|
||||
if !better && orphan == victimOrphan {
|
||||
better = len(qq) > victimLen || (len(qq) == victimLen && id < victimID)
|
||||
}
|
||||
if better {
|
||||
victimID = id
|
||||
victimLen = len(qq)
|
||||
victimOrphan = orphan
|
||||
}
|
||||
}
|
||||
if victimID != "" {
|
||||
item, qs.q[victimID] = popFront(qs.q[victimID])
|
||||
qs.remaining--
|
||||
return item, thief.WorkerID
|
||||
}
|
||||
return takeLeftover()
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
func replicaJob(id, source string) *plugin_pb.JobSpec {
|
||||
return &plugin_pb.JobSpec{
|
||||
JobId: id,
|
||||
JobType: "erasure_coding",
|
||||
Summary: id,
|
||||
DedupeKey: id,
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func registerECWorkers(t *testing.T, pluginSvc *Plugin, workers map[string]string) {
|
||||
t.Helper()
|
||||
for id, addr := range workers {
|
||||
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: id,
|
||||
Address: addr,
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignJobsWaterfillsPreferredWorkers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-a": "10.0.0.1:9334",
|
||||
"worker-b": "10.0.0.2:9334",
|
||||
"worker-c": "10.0.0.3:9334",
|
||||
"worker-d": "10.0.0.4:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("j1", "10.0.0.1:8080"),
|
||||
replicaJob("j2", "10.0.0.1:8080"),
|
||||
replicaJob("j3", "10.0.0.1:8080"),
|
||||
replicaJob("j4", "10.0.0.1:8080"),
|
||||
}
|
||||
// All four replica IPs listed so every worker is preferred.
|
||||
for _, job := range jobs {
|
||||
job.Parameters["replica_servers"] = &plugin_pb.ConfigValue{
|
||||
Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{
|
||||
Values: []string{"10.0.0.1:8080", "10.0.0.2:8080", "10.0.0.3:8080", "10.0.0.4:8080"},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, schedulerPolicy{LocalQueuePin: 5})
|
||||
if qs.queueLen("worker-a")+qs.queueLen("worker-b")+qs.queueLen("worker-c")+qs.queueLen("worker-d") != 4 {
|
||||
t.Fatalf("expected 4 assigned jobs")
|
||||
}
|
||||
for _, id := range []string{"worker-a", "worker-b", "worker-c", "worker-d"} {
|
||||
if qs.queueLen(id) != 1 {
|
||||
t.Fatalf("want 1 job on %s, got %d", id, qs.queueLen(id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignAndTakeDoesNotStarveOtherHomes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("local-1", "10.0.0.1:8080"),
|
||||
replicaJob("local-2", "10.0.0.1:8080"),
|
||||
replicaJob("remote-1", "10.0.0.2:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 1 {
|
||||
t.Fatalf("homes local=%d remote=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"))
|
||||
}
|
||||
|
||||
_, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || target2 != "worker-remote" {
|
||||
t.Fatalf("second take should start remote home, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOnlyWhenHomeQueueReachesK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 {
|
||||
t.Fatalf("all jobs should home on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
// 5 still queued on local (>=K) and local has no slot → steal to remote.
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("expected steal to remote, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 4 {
|
||||
t.Fatalf("after steal want 4 left on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
// 4 < K: do not steal again.
|
||||
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatalf("pin should hold remaining jobs, got target=%s", tgt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsImmediatelyWhenPinDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("a", "10.0.0.1:8080"),
|
||||
replicaJob("b", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 0}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("K=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOrphanedHomeWithoutK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("o1", "10.0.0.1:8080"),
|
||||
replicaJob("o2", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 {
|
||||
t.Fatalf("want 2 on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
|
||||
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatalf("within grace should hold orphan queue, got target=%s", tgt)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 2 {
|
||||
t.Fatalf("held queue changed: %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
qs.mu.Lock()
|
||||
qs.missingSince["worker-local"] = time.Now().Add(-time.Minute)
|
||||
qs.mu.Unlock()
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("after grace should steal without K, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 1 {
|
||||
t.Fatalf("want 1 orphan left, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
stolen2, stealTarget2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen2 == nil || stealTarget2 != "worker-remote" {
|
||||
t.Fatalf("second orphan should also move, got job=%v target=%s", stolen2, stealTarget2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeKeepsOrphanWhenHomeReconnectsDuringGrace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{replicaJob("blink", "10.0.0.1:8080")}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
if item, _ := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatal("expected hold during grace")
|
||||
}
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{"worker-local": "10.0.0.1:9334"})
|
||||
item, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item == nil || target != "worker-local" {
|
||||
t.Fatalf("reconnected home should run its queue, got job=%v target=%s", item, target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOrphanImmediatelyWhenGraceDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{replicaJob("o1", "10.0.0.1:8080")}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5, OrphanRehomeGrace: 0}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("grace=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindQueuedJobWorkerKeepsPendingState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
job := replicaJob("bind-1", "10.0.0.1:8080")
|
||||
pluginSvc.trackExecutionQueued(job)
|
||||
pluginSvc.bindQueuedJobWorker(job, "worker-local")
|
||||
|
||||
pluginSvc.jobsMu.Lock()
|
||||
tracked := pluginSvc.jobs[job.JobId]
|
||||
pluginSvc.jobsMu.Unlock()
|
||||
if tracked == nil {
|
||||
t.Fatal("missing tracked job")
|
||||
}
|
||||
if tracked.State != "job_state_pending" {
|
||||
t.Fatalf("state=%s", tracked.State)
|
||||
}
|
||||
if tracked.Stage != "queued" {
|
||||
t.Fatalf("stage=%s", tracked.Stage)
|
||||
}
|
||||
if tracked.WorkerID != "worker-local" {
|
||||
t.Fatalf("worker=%s", tracked.WorkerID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomelessAllRemoteWorkersUseFIFOWithoutPinOrGrace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
// Workers never registered on the replica hostname — homeless, not orphan.
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-a": "10.0.0.5:9334",
|
||||
"worker-b": "10.0.0.6:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("h1", "10.0.0.1:8080"),
|
||||
replicaJob("h2", "10.0.0.1:8080"),
|
||||
replicaJob("h3", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-a") != 0 || qs.queueLen("worker-b") != 0 {
|
||||
t.Fatalf("homeless jobs must not get a fake home, a=%d b=%d", qs.queueLen("worker-a"), qs.queueLen("worker-b"))
|
||||
}
|
||||
if qs.leftoverLen() != 3 {
|
||||
t.Fatalf("want 3 homeless FIFO, got %d", qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "h1" {
|
||||
t.Fatalf("FIFO first want h1, got %v", item1)
|
||||
}
|
||||
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "h2" {
|
||||
t.Fatalf("FIFO second want h2 immediately, got %v", item2)
|
||||
}
|
||||
if target2 == "" || target2 == target1 {
|
||||
t.Fatalf("second take should hit the other idle worker, t1=%s t2=%s", target1, target2)
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("want 1 homeless left, got %d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMixedReplicaLocalJobsStayHomeBeforeHomeless(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("local-1", "10.0.0.1:8080"),
|
||||
replicaJob("local-2", "10.0.0.1:8080"),
|
||||
replicaJob("homeless-1", "10.0.0.9:8080"),
|
||||
replicaJob("homeless-2", "10.0.0.9:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 0 || qs.leftoverLen() != 2 {
|
||||
t.Fatalf("homes local=%d remote=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"), qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "local-1" || target1 != "worker-local" {
|
||||
t.Fatalf("first take want local-1 on home, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-remote" {
|
||||
t.Fatalf("remote should take homeless, not steal pinned home, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 1 {
|
||||
t.Fatalf("remaining replica-local job must stay home, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("one homeless should remain, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdleHomeWorkerTakesHomelessAfterEmptyHomeQueue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("home-1", "10.0.0.1:8080"),
|
||||
replicaJob("homeless-1", "10.0.0.9:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 1 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "home-1" || target1 != "worker-local" {
|
||||
t.Fatalf("first take want home-1, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-local" {
|
||||
t.Fatalf("idle home with empty queue may take homeless, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
if qs.leftoverLen() != 0 {
|
||||
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomelessWorkerTakesFIFOBeforeStealableHome(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 7)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
|
||||
}
|
||||
if qs.hasHome("worker-remote") {
|
||||
t.Fatal("remote must have no home identity")
|
||||
}
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item, takeTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item == nil || takeTarget != "worker-remote" || item.job.JobId != "homeless-1" {
|
||||
t.Fatalf("remote must take homeless FIFO before K-steal, got job=%v target=%s", item, takeTarget)
|
||||
}
|
||||
if qs.leftoverLen() != 0 {
|
||||
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
if qs.queueLen("worker-local") != 5 {
|
||||
t.Fatalf("stealable home must stay until FIFO empty, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" || stolen.job.JobId == "homeless-1" {
|
||||
t.Fatalf("after FIFO empty remote may steal >=K, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomeWorkerStealsBeforeHomelessAfterOwnQueueEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-other": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 8)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
jobs = append(jobs, replicaJob("other-1", "10.0.0.2:8080"))
|
||||
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 || qs.queueLen("worker-other") != 1 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("local=%d other=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-other"), qs.leftoverLen())
|
||||
}
|
||||
if !qs.hasHome("worker-other") {
|
||||
t.Fatal("other must keep home identity")
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || target1 != "worker-local" {
|
||||
t.Fatalf("first take want local home, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "other-1" || target2 != "worker-other" {
|
||||
t.Fatalf("other should drain own home first, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-other" || stolen.job.JobId == "homeless-1" {
|
||||
t.Fatalf("home worker with empty own queue still steals before homeless, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("homeless should still wait, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
@@ -10,10 +10,13 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
|
||||
// ListExistingPeerUpdates lists the cluster nodes one master currently tracks.
|
||||
// An empty result is only a successful empty answer; a failed RPC returns the
|
||||
// error so callers do not mistake "unknown" for "absent".
|
||||
func ListExistingPeerUpdates(ctx context.Context, master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate, err error) {
|
||||
|
||||
if grpcErr := pb.WithMasterClient(context.Background(), false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
|
||||
err = pb.WithMasterClient(ctx, false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
|
||||
ClientType: clientType,
|
||||
FilerGroup: filerGroup,
|
||||
})
|
||||
@@ -31,8 +34,24 @@ func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOp
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}); grpcErr != nil {
|
||||
glog.V(0).Infof("connect to %s: %v", master, grpcErr)
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(0).Infof("connect to %s: %v", master, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// LookupClusterLeader asks one master for the raft leader it currently sees.
|
||||
// Any reachable master answers, including followers; it returns "" while the
|
||||
// cluster is mid-election.
|
||||
func LookupClusterLeader(ctx context.Context, master pb.ServerAddress, grpcDialOption grpc.DialOption) (leader pb.ServerAddress, err error) {
|
||||
err = pb.WithMasterClient(ctx, false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
leader = pb.ServerAddress(resp.Leader)
|
||||
return nil
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
+15
-3
@@ -382,8 +382,10 @@ func (fo *FilerOptions) startFiler() {
|
||||
}
|
||||
}
|
||||
|
||||
announce := make(chan struct{})
|
||||
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
|
||||
Masters: fo.masters,
|
||||
AnnounceCh: announce,
|
||||
FilerGroup: *fo.filerGroup,
|
||||
Collection: *fo.collection,
|
||||
DefaultReplication: *fo.defaultReplicaPlacement,
|
||||
@@ -500,6 +502,13 @@ func (fo *FilerOptions) startFiler() {
|
||||
go grpcS.Serve(grpcL)
|
||||
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
|
||||
|
||||
// Membership in the master's filer list routes S3 and peer traffic here;
|
||||
// announce once the gRPC port is serving and the metadata replay has
|
||||
// caught up with the peers found at startup, so the filer does not take
|
||||
// lock-ring ownership while a peer's older events can still land on top
|
||||
// of its fresh writes. A cap bounds the wait.
|
||||
go fs.AnnounceOnceCaughtUp(announce, time.Now())
|
||||
|
||||
// Helper to gracefully stop the gRPC server, waiting for active RPCs.
|
||||
gracefulTimeout := fo.gracefulStopTimeout
|
||||
if gracefulTimeout <= 0 {
|
||||
@@ -574,7 +583,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
}
|
||||
httpS := newHttpServer(defaultHandler, tlsConfig)
|
||||
httpServers = append(httpServers, httpS)
|
||||
shutdown := newFilerShutdown(fs.LeaveLockRing, stopGrpcServer, fs.Shutdown, httpServers...)
|
||||
shutdown := newFilerShutdown(fs.LeaveLockRing, fs.StopSubscriptions, stopGrpcServer, fs.Shutdown, httpServers...)
|
||||
|
||||
grace.OnInterrupt(shutdown)
|
||||
|
||||
@@ -603,7 +612,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
}
|
||||
httpS := newHttpServer(defaultHandler, nil)
|
||||
httpServers = append(httpServers, httpS)
|
||||
shutdown := newFilerShutdown(fs.LeaveLockRing, stopGrpcServer, fs.Shutdown, httpServers...)
|
||||
shutdown := newFilerShutdown(fs.LeaveLockRing, fs.StopSubscriptions, stopGrpcServer, fs.Shutdown, httpServers...)
|
||||
|
||||
grace.OnInterrupt(shutdown)
|
||||
|
||||
@@ -625,10 +634,13 @@ func (fo *FilerOptions) startFiler() {
|
||||
// newFilerShutdown joins shutdown callers while gRPC and HTTP drain concurrently.
|
||||
// The filer leaves the lock ring first: peers and S3 gateways route keys to it
|
||||
// until the ring changes, and would hit refused connections once gRPC stops.
|
||||
func newFilerShutdown(leaveLockRing, stopGrpc, shutdownFiler func(), httpServers ...*http.Server) func() {
|
||||
// Then it ends its metadata subscriptions, so gRPC GracefulStop does not wait
|
||||
// out its timeout on streams whose subscribers sit in this same process.
|
||||
func newFilerShutdown(leaveLockRing, stopSubscriptions, stopGrpc, shutdownFiler func(), httpServers ...*http.Server) func() {
|
||||
drain := newGracefulShutdown(stopGrpc, shutdownFiler, httpServers...)
|
||||
return sync.OnceFunc(func() {
|
||||
leaveLockRing()
|
||||
stopSubscriptions()
|
||||
drain()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -287,6 +287,10 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
|
||||
return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
|
||||
}
|
||||
if filer_pb.IsDelete(resp) {
|
||||
if filer.IsMetadataOnlyDelete(message.OldEntry) {
|
||||
glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name)
|
||||
return nil
|
||||
}
|
||||
if resp.Directory == option.bucketsDir {
|
||||
return handleDeleteBucket(message.OldEntry)
|
||||
}
|
||||
@@ -355,7 +359,14 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
|
||||
if message.OldEntry.RemoteEntry != nil && filer.IsSameData(message.OldEntry, message.NewEntry) {
|
||||
glog.V(2).Infof("update meta: %+v", resp)
|
||||
oldDest := toRemoteStorageLocation(oldBucket, util.NewFullPath(resp.Directory, message.OldEntry.Name), oldRemoteStorageMountLocation)
|
||||
return client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry)
|
||||
if err := client.UpdateFileMetadata(oldDest, message.OldEntry, message.NewEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
remoteEntry := message.NewEntry.RemoteEntry
|
||||
if remoteEntry == nil {
|
||||
remoteEntry = message.OldEntry.RemoteEntry
|
||||
}
|
||||
return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
|
||||
} else {
|
||||
newDest := toRemoteStorageLocation(newBucket, util.NewFullPath(message.NewParentPath, message.NewEntry.Name), newRemoteStorageMountLocation)
|
||||
remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, message.NewEntry, newDest)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
|
||||
)
|
||||
|
||||
type recordingRemoteMaker struct{ client *recordingRemote }
|
||||
|
||||
func (m recordingRemoteMaker) Make(*remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
|
||||
return m.client, nil
|
||||
}
|
||||
func (m recordingRemoteMaker) HasBucket() bool { return true }
|
||||
|
||||
func TestGatewayMetadataOnlyDeleteKeepsTheRemoteObject(t *testing.T) {
|
||||
remote := &recordingRemote{}
|
||||
remote_storage.RemoteStorageClientMakers["gatewaytest"] = recordingRemoteMaker{client: remote}
|
||||
t.Cleanup(func() { delete(remote_storage.RemoteStorageClientMakers, "gatewaytest") })
|
||||
option := &RemoteGatewayOptions{
|
||||
bucketsDir: "/buckets",
|
||||
mappings: &remote_pb.RemoteStorageMapping{Mappings: map[string]*remote_pb.RemoteStorageLocation{"/buckets/b": {Name: "gatewaytest", Bucket: "b", Path: "/"}}},
|
||||
remoteConfs: map[string]*remote_pb.RemoteConf{"gatewaytest": {Name: "gatewaytest", Type: "gatewaytest"}},
|
||||
}
|
||||
process, err := option.makeBucketedEventProcessor(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deleteEvent := func(extended map[string][]byte) *filer_pb.SubscribeMetadataResponse {
|
||||
return &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/buckets/b/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "obj.bin", Attributes: &filer_pb.FuseAttributes{}, Extended: extended},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if err := process(deleteEvent(map[string][]byte{filer.ExtKeepRemoteObjectKey: []byte("true")})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Fatalf("deletes = %+v, want none for a metadata-only delete", remote.deletes)
|
||||
}
|
||||
if err := process(deleteEvent(nil)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.deletes) != 1 {
|
||||
t.Fatalf("deletes = %+v, want the ordinary delete", remote.deletes)
|
||||
}
|
||||
}
|
||||
@@ -195,7 +195,7 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
|
||||
return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
|
||||
}
|
||||
if filer_pb.IsDelete(resp) {
|
||||
return processDeleteEvent(client, mountedDir, remoteStorageMountLocation, resp)
|
||||
return processDeleteEvent(filerSource, client, mountedDir, remoteStorageMountLocation, resp)
|
||||
}
|
||||
if message.OldEntry != nil && message.NewEntry != nil {
|
||||
return processUpdateEvent(option, filerSource, *option.storageClass, client, mountedDir, remoteStorageMountLocation, resp)
|
||||
@@ -212,8 +212,11 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
|
||||
// event the inline delete already handled), and GCS reports that case as
|
||||
// ErrRemoteObjectNotFound where S3 and Azure answer an idempotent success.
|
||||
// Returning the error would pin the sync offset on an event that has nothing
|
||||
// left to do.
|
||||
// left to do. A file whose path a remote-only entry has taken since keeps its
|
||||
// object: a rename copied that entry's content to this key, and it exists
|
||||
// nowhere else.
|
||||
func processDeleteEvent(
|
||||
filerSource filer_pb.FilerClient,
|
||||
client remote_storage.RemoteStorageClient,
|
||||
mountedDir string,
|
||||
remoteStorageMountLocation *remote_pb.RemoteStorageLocation,
|
||||
@@ -226,8 +229,42 @@ func processDeleteEvent(
|
||||
glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
|
||||
return nil
|
||||
}
|
||||
if filer.IsMetadataOnlyDelete(message.OldEntry) {
|
||||
glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name)
|
||||
return nil
|
||||
}
|
||||
glog.V(2).Infof("delete: %+v", resp)
|
||||
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
|
||||
if !message.OldEntry.IsDirectory {
|
||||
// a deleted entry whose rename copy was still pending read its
|
||||
// content from the source its marker names, which nothing else
|
||||
// releases
|
||||
if err := releasePendingSource(filerSource, client, mountedDir, remoteStorageMountLocation, message.OldEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
// The object at the key may have been replaced since the delete: a
|
||||
// rename copies another entry's content onto it, a rewrite uploads
|
||||
// over it, and a copy can land before its entry is visible. Delete
|
||||
// only an object the event's stamp still describes.
|
||||
if live, statErr := client.StatFile(dest); statErr == nil {
|
||||
if oldRemote := message.OldEntry.GetRemoteEntry(); oldRemote == nil ||
|
||||
!describes(oldRemote, live) ||
|
||||
oldRemote.RemoteETag != "" && live.RemoteETag == "" {
|
||||
glog.V(0).Infof("keep %s: the object changed since the delete", remote_storage.FormatLocation(dest))
|
||||
return nil
|
||||
}
|
||||
} else if !errors.Is(statErr, remote_storage.ErrRemoteObjectNotFound) {
|
||||
return statErr
|
||||
}
|
||||
current, err := currentEntry(filerSource, resp.Directory, message.OldEntry.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRemoteOnly(current) {
|
||||
glog.V(0).Infof("keep %s: a remote-only entry has taken its path since", remote_storage.FormatLocation(dest))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if message.OldEntry.IsDirectory {
|
||||
glog.V(0).Infof("rmdir %s", remote_storage.FormatLocation(dest))
|
||||
return client.RemoveDirectory(dest)
|
||||
@@ -236,6 +273,41 @@ func processDeleteEvent(
|
||||
return deleteRemoteFile(client, dest)
|
||||
}
|
||||
|
||||
// releasePendingSource deletes the source a pending-rename marker on entry
|
||||
// names, once nothing needs it. The filer never deletes a marked source: a
|
||||
// concurrent rename can copy onto that key before its entry is visible. So
|
||||
// the object goes only while it is still the one entry's stamp describes, and
|
||||
// no remote-only entry holds its path.
|
||||
func releasePendingSource(filerSource filer_pb.FilerClient, client remote_storage.RemoteStorageClient, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) error {
|
||||
src := util.FullPath(entry.GetExtended()[filer.ExtRemoteRenameSourceKey])
|
||||
if src == "" || !strings.HasPrefix(string(src), strings.TrimSuffix(mountedDir, "/")+"/") {
|
||||
return nil
|
||||
}
|
||||
srcDest := toRemoteStorageLocation(util.FullPath(mountedDir), src, remoteStorageMountLocation)
|
||||
srcDir, srcName := src.DirAndName()
|
||||
current, err := currentEntry(filerSource, srcDir, srcName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isRemoteOnly(current) {
|
||||
glog.V(0).Infof("keep %s: a remote-only entry holds the pending rename's source", remote_storage.FormatLocation(srcDest))
|
||||
return nil
|
||||
}
|
||||
live, err := client.StatFile(srcDest)
|
||||
if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stamp := entry.GetRemoteEntry(); !describes(stamp, live) || stamp.RemoteETag != "" && live.RemoteETag == "" {
|
||||
glog.V(0).Infof("keep %s: the pending rename's source changed since", remote_storage.FormatLocation(srcDest))
|
||||
return nil
|
||||
}
|
||||
glog.V(0).Infof("delete %s, the source of a pending rename", remote_storage.FormatLocation(srcDest))
|
||||
return deleteRemoteFile(client, srcDest)
|
||||
}
|
||||
|
||||
// deleteRemoteFile deletes the object and treats an already-absent object as
|
||||
// deleted.
|
||||
func deleteRemoteFile(client remote_storage.RemoteStorageClient, dest *remote_pb.RemoteStorageLocation) error {
|
||||
@@ -264,6 +336,17 @@ func processUpdateEvent(
|
||||
}
|
||||
oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
|
||||
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
|
||||
eventSource := renameSource{path: util.NewFullPath(resp.Directory, message.OldEntry.Name), loc: oldDest, stamp: message.OldEntry.GetRemoteEntry()}
|
||||
// sourceOf is where a remote-only entry's content is: the source its
|
||||
// pending-rename marker names, which may be an earlier rename's old key,
|
||||
// or else the event's old key
|
||||
sourceOf := func(entry *filer_pb.Entry, fallback renameSource) renameSource {
|
||||
if src := string(entry.Extended[filer.ExtRemoteRenameSourceKey]); strings.HasPrefix(src, strings.TrimSuffix(mountedDir, "/")+"/") {
|
||||
// a pending entry carries the stamp of the source it reads
|
||||
return renameSource{path: util.FullPath(src), loc: toRemoteStorageLocation(util.FullPath(mountedDir), util.FullPath(src), remoteStorageMountLocation), stamp: entry.GetRemoteEntry()}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
if proto.Equal(oldDest, dest) && !shouldSendToRemote(message.NewEntry) {
|
||||
glog.V(2).Infof("skipping updating: %+v", resp)
|
||||
return nil
|
||||
@@ -282,7 +365,10 @@ func processUpdateEvent(
|
||||
}
|
||||
if remoteEntry != nil {
|
||||
glog.V(2).Infof("update meta: %+v", resp)
|
||||
return client.UpdateFileMetadata(dest, message.OldEntry, remoteWriteEntry(message.NewEntry, storageClass))
|
||||
if err := client.UpdateFileMetadata(dest, message.OldEntry, remoteWriteEntry(message.NewEntry, storageClass)); err != nil {
|
||||
return err
|
||||
}
|
||||
return updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry)
|
||||
}
|
||||
glog.V(0).Infof("never replicated, uploading %s", remote_storage.FormatLocation(dest))
|
||||
}
|
||||
@@ -300,7 +386,23 @@ func processUpdateEvent(
|
||||
return err
|
||||
}
|
||||
if isRemoteOnly(current) {
|
||||
return completeRemoteOnlyRename(filerClient, client, message.NewParentPath, current, oldDest, dest, storageClass)
|
||||
source := sourceOf(current, eventSource)
|
||||
if err := completeRemoteOnlyRename(filerClient, client, message.NewParentPath, current, source, dest, storageClass); err != nil {
|
||||
return err
|
||||
}
|
||||
// a pending entry the filer copied on this rename leaves the
|
||||
// source its marker named
|
||||
if pending := sourceOf(message.OldEntry, renameSource{}); pending.loc != nil && !proto.Equal(pending.loc, source.loc) {
|
||||
return releasePendingSource(filerSource, client, mountedDir, remoteStorageMountLocation, message.OldEntry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if current == nil && len(message.NewEntry.Extended[filer.ExtRemoteRenameSourceKey]) > 0 {
|
||||
// the rename's copy was pending and the entry has moved on: a
|
||||
// later rename carries the source in its own marker and finishes
|
||||
// or releases it, so this one leaves it
|
||||
glog.V(0).Infof("keep %s: the pending rename moved on", remote_storage.FormatLocation(oldDest))
|
||||
return nil
|
||||
}
|
||||
glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
|
||||
if err := client.DeleteFile(oldDest); err != nil {
|
||||
@@ -316,14 +418,43 @@ func processUpdateEvent(
|
||||
if errors.Is(writeErr, errSuperseded) {
|
||||
glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
|
||||
if !proto.Equal(oldDest, dest) {
|
||||
return uploadCurrentEntry(filerClient, filerSource, client, message.NewParentPath, message.NewEntry.Name, oldDest, dest, storageClass)
|
||||
if err := uploadCurrentEntry(filerClient, filerSource, client, message.NewParentPath, message.NewEntry.Name, eventSource, dest, storageClass, sourceOf); err != nil {
|
||||
return err
|
||||
}
|
||||
return releaseRenamedSource(filerSource, client, mountedDir, remoteStorageMountLocation, message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if writeErr != nil {
|
||||
return writeErr
|
||||
}
|
||||
return updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry)
|
||||
if err := updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
if !proto.Equal(oldDest, dest) {
|
||||
return releaseRenamedSource(filerSource, client, mountedDir, remoteStorageMountLocation, message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// releaseRenamedSource releases a pending source the renamed entry's old
|
||||
// version read, once the destination holds the content on the remote under
|
||||
// its own key: only after the upload and its stamp, which clears the marker,
|
||||
// and only if the entry does not read through a marker any more. Released
|
||||
// earlier, an upload that fails followed by remote.uncache would leave the
|
||||
// entry reading a deleted source.
|
||||
func releaseRenamedSource(filerSource filer_pb.FilerClient, client remote_storage.RemoteStorageClient, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, message *filer_pb.EventNotification) error {
|
||||
if len(message.OldEntry.GetExtended()[filer.ExtRemoteRenameSourceKey]) == 0 {
|
||||
return nil
|
||||
}
|
||||
current, err := currentEntry(filerSource, message.NewParentPath, message.NewEntry.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current != nil && len(current.Extended[filer.ExtRemoteRenameSourceKey]) > 0 {
|
||||
return nil
|
||||
}
|
||||
return releasePendingSource(filerSource, client, mountedDir, remoteStorageMountLocation, message.OldEntry)
|
||||
}
|
||||
|
||||
// currentEntry returns what the filer holds at dir/name now, nil when the
|
||||
@@ -352,11 +483,24 @@ func isRemoteOnly(entry *filer_pb.Entry) bool {
|
||||
// copied over the destination, the entry is stamped with the copy, and the
|
||||
// old key goes. With neither object present the content is lost: the event
|
||||
// fails and holds the offset for recovery.
|
||||
func completeRemoteOnlyRename(filerClient filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir string, current *filer_pb.Entry, oldDest, dest *remote_pb.RemoteStorageLocation, storageClass string) error {
|
||||
func completeRemoteOnlyRename(filerClient filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir string, current *filer_pb.Entry, source renameSource, dest *remote_pb.RemoteStorageLocation, storageClass string) error {
|
||||
oldDest := source.loc
|
||||
// the source's recorded stamp; a retry can find the entry already
|
||||
// stamped with the copy, whose ETag a multipart source does not share
|
||||
sourceStamp := source.stamp
|
||||
if sourceStamp == nil {
|
||||
sourceStamp = proto.Clone(current.RemoteEntry).(*filer_pb.RemoteEntry)
|
||||
}
|
||||
if existing, err := client.StatFile(dest); err == nil {
|
||||
if describes(current.RemoteEntry, existing) {
|
||||
glog.V(0).Infof("%s already holds the renamed content", remote_storage.FormatLocation(dest))
|
||||
return deleteRemoteFile(client, oldDest)
|
||||
if len(current.Extended[filer.ExtRemoteRenameSourceKey]) > 0 {
|
||||
// reads still go to the source until the stamp clears the marker
|
||||
if err := updateLocalEntry(filerClient, dir, current, existing); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return deleteRenameSource(filerClient, client, dir, current.Name, source, sourceStamp)
|
||||
}
|
||||
glog.V(0).Infof("%s holds an object the entry does not describe (size %d, want %d); replacing it from %s", remote_storage.FormatLocation(dest), existing.RemoteSize, current.RemoteEntry.RemoteSize, remote_storage.FormatLocation(oldDest))
|
||||
} else if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
|
||||
@@ -382,7 +526,42 @@ func completeRemoteOnlyRename(filerClient filer_pb.FilerClient, client remote_st
|
||||
if err := updateLocalEntry(filerClient, dir, current, remoteEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteRemoteFile(client, oldDest)
|
||||
return deleteRenameSource(filerClient, client, dir, current.Name, source, sourceStamp)
|
||||
}
|
||||
|
||||
// renameSource is where a renamed remote-only entry's content was read
|
||||
// from: the path, the object it maps to, and the stamp describing it.
|
||||
type renameSource struct {
|
||||
path util.FullPath
|
||||
loc *remote_pb.RemoteStorageLocation
|
||||
stamp *filer_pb.RemoteEntry
|
||||
}
|
||||
|
||||
// deleteRenameSource deletes a rename's old key once the renamed entry no
|
||||
// longer reads it: an entry still carrying the pending-source marker (its
|
||||
// stamp lost a race) keeps the source, and the event is retried. The key is
|
||||
// left alone when the object there is no longer the one stamp describes, as
|
||||
// when another rename copied onto it, visible yet or not.
|
||||
func deleteRenameSource(filerClient filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir, name string, source renameSource, stamp *filer_pb.RemoteEntry) error {
|
||||
current, err := currentEntry(filerClient, dir, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current != nil && len(current.Extended[filer.ExtRemoteRenameSourceKey]) > 0 {
|
||||
return fmt.Errorf("%s still reads %s", util.NewFullPath(dir, name), remote_storage.FormatLocation(source.loc))
|
||||
}
|
||||
live, err := client.StatFile(source.loc)
|
||||
if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !describes(stamp, live) || stamp.GetRemoteETag() != "" && live.RemoteETag == "" {
|
||||
glog.V(0).Infof("keep %s: it is no longer the renamed content", remote_storage.FormatLocation(source.loc))
|
||||
return nil
|
||||
}
|
||||
return deleteRemoteFile(client, source.loc)
|
||||
}
|
||||
|
||||
// describes reports whether the object a stat returned is the one the entry's
|
||||
@@ -418,7 +597,7 @@ func openRemoteObject(client remote_storage.RemoteStorageClient, loc *remote_pb.
|
||||
// up once. A remote-only entry is finished the way completeRemoteOnlyRename
|
||||
// finishes a rename: its stamp can already describe the destination (a sync
|
||||
// plus remote.uncache in the meantime), and only then is it complete.
|
||||
func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir, name string, oldDest, dest *remote_pb.RemoteStorageLocation, storageClass string) error {
|
||||
func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir, name string, oldSource renameSource, dest *remote_pb.RemoteStorageLocation, storageClass string, sourceOf func(*filer_pb.Entry, renameSource) renameSource) error {
|
||||
current, _, _, err := filer_pb.GetEntry(context.Background(), filerSource, util.NewFullPath(dir, name))
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil
|
||||
@@ -430,7 +609,7 @@ func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.F
|
||||
return nil
|
||||
}
|
||||
if isRemoteOnly(current) {
|
||||
return completeRemoteOnlyRename(filerClient, client, dir, current, oldDest, dest, storageClass)
|
||||
return completeRemoteOnlyRename(filerClient, client, dir, current, sourceOf(current, oldSource), dest, storageClass)
|
||||
}
|
||||
if shouldSendToRemote(current) {
|
||||
glog.V(0).Infof("leaving %s to the rewrite that superseded the rename", remote_storage.FormatLocation(dest))
|
||||
@@ -589,10 +768,12 @@ func shouldSendToRemote(entry *filer_pb.Entry) bool {
|
||||
if entry.RemoteEntry == nil {
|
||||
return true
|
||||
}
|
||||
if entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime {
|
||||
return true
|
||||
if entry.RemoteEntry.RemoteMtime != entry.Attributes.Mtime {
|
||||
return entry.RemoteEntry.RemoteMtime < entry.Attributes.Mtime
|
||||
}
|
||||
return false
|
||||
// Equal seconds cannot tell a rewrite from the version already
|
||||
// uploaded; the sync stamp can, the same check remote.uncache uses.
|
||||
return entry.RemoteEntry.LastLocalSyncTsNs < entry.Attributes.Mtime*1e9+int64(entry.Attributes.MtimeNs)
|
||||
}
|
||||
|
||||
// remoteWriteEntry returns the entry as remote storage should see it: the
|
||||
@@ -621,9 +802,11 @@ func remoteWriteEntry(entry *filer_pb.Entry, storageClass string) *filer_pb.Entr
|
||||
// in the log and stamps the current entry, so the stale stamp is skipped the
|
||||
// same way a superseded upload is.
|
||||
func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
|
||||
remoteEntry = proto.Clone(remoteEntry).(*filer_pb.RemoteEntry)
|
||||
remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
|
||||
expected := proto.Clone(entry).(*filer_pb.Entry)
|
||||
entry.RemoteEntry = remoteEntry
|
||||
delete(entry.Extended, filer.ExtRemoteRenameSourceKey)
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -420,6 +421,7 @@ type stubFilerClient struct {
|
||||
updateErr error
|
||||
lookups int
|
||||
updates int
|
||||
updated *filer_pb.UpdateEntryRequest
|
||||
}
|
||||
|
||||
func (c *stubFilerClient) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest, ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
@@ -430,8 +432,9 @@ func (c *stubFilerClient) LookupDirectoryEntry(context.Context, *filer_pb.Lookup
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: c.entry}, nil
|
||||
}
|
||||
|
||||
func (c *stubFilerClient) UpdateEntry(context.Context, *filer_pb.UpdateEntryRequest, ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
|
||||
func (c *stubFilerClient) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
|
||||
c.updates++
|
||||
c.updated = req
|
||||
if c.updateErr != nil {
|
||||
return &filer_pb.UpdateEntryResponse{}, c.updateErr
|
||||
}
|
||||
@@ -677,18 +680,27 @@ func TestRetriedWriteFileStopsWhenSuperseded(t *testing.T) {
|
||||
|
||||
type recordingRemote struct {
|
||||
remote_storage.RemoteStorageClient
|
||||
deletes []*remote_pb.RemoteStorageLocation
|
||||
writes []*remote_pb.RemoteStorageLocation
|
||||
written [][]byte
|
||||
deletes []*remote_pb.RemoteStorageLocation
|
||||
writes []*remote_pb.RemoteStorageLocation
|
||||
metadataUpdates []*remote_pb.RemoteStorageLocation
|
||||
written [][]byte
|
||||
// chunk ids of each written entry: which version of the file went up
|
||||
writtenChunks [][]string
|
||||
deleteErr error
|
||||
writeErr error
|
||||
// objects present on the remote, by path: StatFile and ReadFile answer
|
||||
// from it, everything else is ErrRemoteObjectNotFound.
|
||||
objects map[string][]byte
|
||||
// statETag is the ETag StatFile reports for a present object, unless
|
||||
// statETags names one for its path.
|
||||
statETag string
|
||||
statETags map[string]string
|
||||
}
|
||||
|
||||
func (r *recordingRemote) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (*filer_pb.RemoteEntry, error) {
|
||||
if r.writeErr != nil {
|
||||
return nil, r.writeErr
|
||||
}
|
||||
r.writes = append(r.writes, loc)
|
||||
var ids []string
|
||||
for _, c := range entry.GetChunks() {
|
||||
@@ -711,7 +723,11 @@ func (r *recordingRemote) StatFile(loc *remote_pb.RemoteStorageLocation) (*filer
|
||||
if !ok {
|
||||
return nil, remote_storage.ErrRemoteObjectNotFound
|
||||
}
|
||||
return &filer_pb.RemoteEntry{StorageName: loc.Name, RemoteSize: int64(len(body))}, nil
|
||||
etag := r.statETag
|
||||
if e, ok := r.statETags[loc.Path]; ok {
|
||||
etag = e
|
||||
}
|
||||
return &filer_pb.RemoteEntry{StorageName: loc.Name, RemoteSize: int64(len(body)), RemoteETag: etag}, nil
|
||||
}
|
||||
|
||||
func (r *recordingRemote) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) ([]byte, error) {
|
||||
@@ -727,16 +743,22 @@ func (r *recordingRemote) DeleteFile(loc *remote_pb.RemoteStorageLocation) error
|
||||
return r.deleteErr
|
||||
}
|
||||
|
||||
func (r *recordingRemote) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry, newEntry *filer_pb.Entry) error {
|
||||
r.metadataUpdates = append(r.metadataUpdates, loc)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestRenameWithInheritedRemoteEntryWritesNewKey reproduces #11261: a rename
|
||||
// arrives as an update whose NewEntry carries the RemoteEntry inherited from
|
||||
// the source. shouldSendToRemote returns false for it (RemoteMtime >= Mtime),
|
||||
// so the old code skipped the event and never wrote the new key, while the
|
||||
// filer had already deleted the old object. A path change must always write.
|
||||
// the source. shouldSendToRemote returns false for it (the sync stamp covers
|
||||
// the entry's mtime), so the old code skipped the event and never wrote the
|
||||
// new key, while the filer had already deleted the old object. A path change
|
||||
// must always write.
|
||||
func TestRenameWithInheritedRemoteEntryWritesNewKey(t *testing.T) {
|
||||
const mountedDir = "/buckets"
|
||||
mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"}
|
||||
|
||||
replicated := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 2048, RemoteMtime: 1786096669}
|
||||
replicated := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 2048, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1}
|
||||
oldEntry := &filer_pb.Entry{Name: "probe.bin", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: replicated}
|
||||
newEntry := &filer_pb.Entry{
|
||||
Name: "probe.bin",
|
||||
@@ -999,7 +1021,7 @@ func TestDeleteEventAbsentRemoteObjectIsSuccess(t *testing.T) {
|
||||
|
||||
t.Run("absent object", func(t *testing.T) {
|
||||
remote := &recordingRemote{deleteErr: remote_storage.ErrRemoteObjectNotFound}
|
||||
if err := processDeleteEvent(remote, mountedDir, mountLoc, resp); err != nil {
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, resp); err != nil {
|
||||
t.Fatalf("err = %v, want nil: deleting an absent object is complete", err)
|
||||
}
|
||||
if len(remote.deletes) != 1 || !proto.Equal(remote.deletes[0], wantDelete) {
|
||||
@@ -1007,9 +1029,120 @@ func TestDeleteEventAbsentRemoteObjectIsSuccess(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("metadata-only delete keeps the remote object", func(t *testing.T) {
|
||||
marked := proto.Clone(resp).(*filer_pb.SubscribeMetadataResponse)
|
||||
marked.EventNotification.OldEntry.Extended = map[string][]byte{filer.ExtKeepRemoteObjectKey: []byte("true")}
|
||||
remote := &recordingRemote{}
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, marked); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want none", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a remote-only entry that took the path keeps the object", func(t *testing.T) {
|
||||
// deleted, then a remote-only file renamed onto the path: the rename
|
||||
// copied its content to this key before the sync got to the delete
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: &filer_pb.Entry{
|
||||
Name: "state.db.tmp",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: 10},
|
||||
RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 10},
|
||||
}}
|
||||
if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, resp); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want none", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a replaced object is kept", func(t *testing.T) {
|
||||
// the delete recorded the old object's etag; a rename copied another
|
||||
// entry's content onto the key since, and it must not go with it
|
||||
deleted := proto.Clone(resp).(*filer_pb.SubscribeMetadataResponse)
|
||||
deleted.EventNotification.OldEntry.RemoteEntry = &filer_pb.RemoteEntry{RemoteETag: `"orig"`, RemoteSize: 10}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/vol/state.db.tmp": []byte("copied")}, statETag: `"copy"`}
|
||||
filerClient := &stubFilerClient{entry: &filer_pb.Entry{Name: "state.db.tmp", Content: []byte("copied")}}
|
||||
if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, deleted); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want none", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a changed object with no entry at the path yet is kept", func(t *testing.T) {
|
||||
// a directory rename copies a child onto the key before its new entry
|
||||
// becomes visible; the copy may be the content's only remaining home
|
||||
deleted := proto.Clone(resp).(*filer_pb.SubscribeMetadataResponse)
|
||||
deleted.EventNotification.OldEntry.RemoteEntry = &filer_pb.RemoteEntry{RemoteETag: `"orig"`, RemoteSize: 10}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/vol/state.db.tmp": []byte("copied")}, statETag: `"copy"`}
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, deleted); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want none", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an unchanged object still goes", func(t *testing.T) {
|
||||
deleted := proto.Clone(resp).(*filer_pb.SubscribeMetadataResponse)
|
||||
deleted.EventNotification.OldEntry.RemoteEntry = &filer_pb.RemoteEntry{RemoteETag: `"orig"`, RemoteSize: 3}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/vol/state.db.tmp": []byte("old")}, statETag: `"orig"`}
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, deleted); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 1 {
|
||||
t.Errorf("deletes = %+v, want one", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a copy that landed before its entry is kept", func(t *testing.T) {
|
||||
// the deleted entry was never replicated, so its event records no
|
||||
// stamp to match the fresh rename copy waiting for its filer entry
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/vol/state.db.tmp": []byte("copied")}, statETag: `"copy"`}
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, resp); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want none", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an unverifiable object is kept", func(t *testing.T) {
|
||||
// the delete recorded an etag the live stat does not report; the
|
||||
// object cannot be confirmed as the one the delete meant
|
||||
deleted := proto.Clone(resp).(*filer_pb.SubscribeMetadataResponse)
|
||||
deleted.EventNotification.OldEntry.RemoteEntry = &filer_pb.RemoteEntry{RemoteETag: `"orig"`, RemoteSize: 3}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/vol/state.db.tmp": []byte("old")}}
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, deleted); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want none", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an entry with local data at the path does not stop the delete", func(t *testing.T) {
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: &filer_pb.Entry{
|
||||
Name: "state.db.tmp",
|
||||
Content: []byte("local"),
|
||||
RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 10},
|
||||
}}
|
||||
if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, resp); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(remote.deletes) != 1 {
|
||||
t.Errorf("deletes = %+v, want one", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("other failure still fails", func(t *testing.T) {
|
||||
remote := &recordingRemote{deleteErr: errors.New("AccessDenied: Access Denied")}
|
||||
if err := processDeleteEvent(remote, mountedDir, mountLoc, resp); err == nil {
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, resp); err == nil {
|
||||
t.Fatal("err = nil, want the delete failure")
|
||||
}
|
||||
})
|
||||
@@ -1082,9 +1215,9 @@ func TestUpdateLocalEntrySkipsDeletedEntry(t *testing.T) {
|
||||
// TestSupersededRenameUploadsCurrentEntry: a rename A -> B queued behind a
|
||||
// rewrite of B. The rename's snapshot is superseded (B's chunks changed) and
|
||||
// the old key A is already deleted. The rewrite's own event uploads B unless
|
||||
// shouldSendToRemote skips it on the inherited RemoteEntry (same-second
|
||||
// mtimes); only then does the rename upload what the filer holds now, so the
|
||||
// content goes up exactly once.
|
||||
// the entry already reads synced (the stamp covers its mtime, or the remote
|
||||
// is newer); only then does the rename upload what the filer holds now, so
|
||||
// the content goes up exactly once.
|
||||
func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
const mountedDir = "/buckets"
|
||||
mountLoc := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/"}
|
||||
@@ -1100,9 +1233,9 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
wantDelete := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/b/dir/a.txt"}
|
||||
wantWrite := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/b/dir/b.txt"}
|
||||
|
||||
t.Run("rewrite hidden by the inherited stamp: upload once here", func(t *testing.T) {
|
||||
// RemoteMtime inherited from a.txt equals the rewrite's mtime.
|
||||
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 1024}, chunk("3,02", "e2"))
|
||||
t.Run("rewrite already synced: upload once here", func(t *testing.T) {
|
||||
// The stamp covers the rewrite's mtime, so its own event skips.
|
||||
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 1024, LastLocalSyncTsNs: 1786096669*1e9 + 1}, chunk("3,02", "e2"))
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp); err != nil {
|
||||
@@ -1206,7 +1339,7 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("snapshot remote-only but rewritten since: the rewrite's bytes go up, nothing is copied", func(t *testing.T) {
|
||||
t.Run("snapshot remote-only but rewritten since: the rewrite's own event uploads", func(t *testing.T) {
|
||||
remoteOnly := &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 7}
|
||||
snapshot := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/buckets/b/dir",
|
||||
@@ -1216,21 +1349,17 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
NewEntry: entryWith("b.txt", remoteOnly),
|
||||
},
|
||||
}
|
||||
// rewritten with the inherited stamp still hiding it from shouldSendToRemote
|
||||
current := entryWith("b.txt", remoteOnly, chunk("3,09", "e9"))
|
||||
// rewritten since the stamp; its own event uploads, not this one
|
||||
rewritten := proto.Clone(remoteOnly).(*filer_pb.RemoteEntry)
|
||||
rewritten.LastLocalSyncTsNs = 1
|
||||
current := entryWith("b.txt", rewritten, chunk("3,09", "e9"))
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/dir/a.txt": []byte("payload")}}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, snapshot); err != nil {
|
||||
t.Fatalf("err = %v, want nil", err)
|
||||
}
|
||||
if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) {
|
||||
t.Fatalf("writes = %+v, want one upload of the rewritten b.txt", remote.writes)
|
||||
}
|
||||
if len(remote.writtenChunks[0]) != 1 || remote.writtenChunks[0][0] != "3,09" {
|
||||
t.Errorf("uploaded chunks = %v, want the rewrite's chunk 3,09, not the chunkless snapshot", remote.writtenChunks[0])
|
||||
}
|
||||
if string(remote.written[0]) == "payload" {
|
||||
t.Error("the old object's bytes were copied over a rewritten entry")
|
||||
if len(remote.writes) != 0 {
|
||||
t.Errorf("writes = %+v, want none: the rewrite event behind this one uploads b.txt", remote.writes)
|
||||
}
|
||||
if len(remote.deletes) != 1 {
|
||||
t.Errorf("deletes = %+v, want the old key deleted", remote.deletes)
|
||||
@@ -1264,7 +1393,7 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 1024})
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", wantDelete, wantWrite, "")
|
||||
err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", renameSource{loc: wantDelete}, wantWrite, "", eventSource)
|
||||
if err == nil || !strings.Contains(err.Error(), "neither") {
|
||||
t.Errorf("err = %v, want the lost-content failure", err)
|
||||
}
|
||||
@@ -1275,9 +1404,9 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
|
||||
t.Run("uncached with the destination already synced: complete without a write", func(t *testing.T) {
|
||||
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 7})
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/dir/b.txt": []byte("payload")}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/dir/a.txt": []byte("payload"), "/b/dir/b.txt": []byte("payload")}}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", wantDelete, wantWrite, ""); err != nil {
|
||||
if err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", renameSource{loc: wantDelete}, wantWrite, "", eventSource); err != nil {
|
||||
t.Fatalf("err = %v, want nil: the destination object matches the entry's stamp", err)
|
||||
}
|
||||
if len(remote.writes) != 0 {
|
||||
@@ -1288,3 +1417,337 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCompleteRemoteOnlyRenameClearsThePendingSourceFirst(t *testing.T) {
|
||||
oldDest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/src/a.jpg"}
|
||||
dest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/dst/a.jpg"}
|
||||
pending := func() *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: "a.jpg",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: 4},
|
||||
RemoteEntry: &filer_pb.RemoteEntry{RemoteETag: `"copy"`, RemoteSize: 4},
|
||||
Extended: map[string][]byte{filer.ExtRemoteRenameSourceKey: []byte("/buckets/b/src/a.jpg")},
|
||||
}
|
||||
}
|
||||
objects := func() map[string][]byte {
|
||||
return map[string][]byte{"/b/src/a.jpg": []byte("data"), "/b/dst/a.jpg": []byte("data")}
|
||||
}
|
||||
|
||||
t.Run("stamped, then the source goes", func(t *testing.T) {
|
||||
// the destination already holds the copy; the entry still reads the
|
||||
// source until its stamp clears the marker
|
||||
current := pending()
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
remote := &recordingRemote{objects: objects(), statETag: `"copy"`}
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b/dst", current, renameSource{path: "/buckets/b/src/a.jpg", loc: oldDest}, dest, ""); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if filerClient.updates != 1 {
|
||||
t.Errorf("updates = %d, want the stamp that clears the marker", filerClient.updates)
|
||||
}
|
||||
if len(remote.deletes) != 1 || remote.deletes[0].Path != "/b/src/a.jpg" {
|
||||
t.Errorf("deletes = %+v, want the source", remote.deletes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a marker still in place keeps the source", func(t *testing.T) {
|
||||
// the stamp did not land: the filer still has the marker
|
||||
filerClient := &stubFilerClient{entry: pending()}
|
||||
remote := &recordingRemote{objects: objects(), statETag: `"copy"`}
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b/dst", pending(), renameSource{path: "/buckets/b/src/a.jpg", loc: oldDest}, dest, ""); err == nil {
|
||||
t.Fatal("err = nil, want the event retried")
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want none while the entry reads the source", remote.deletes)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// eventSource resolves a rename's source to the event's old key.
|
||||
func eventSource(_ *filer_pb.Entry, source renameSource) renameSource {
|
||||
return source
|
||||
}
|
||||
|
||||
func TestRenameOfAPendingEntryFollowsItsSource(t *testing.T) {
|
||||
// A was renamed over B and B over C, both copies failing: C reads A's
|
||||
// object through the marker, and B's key holds the old target's bytes
|
||||
const mountedDir = "/buckets"
|
||||
mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"}
|
||||
remoteOnly := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: `"a"`, RemoteSize: 6}
|
||||
pendingC := func() *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: "c.jpg",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: 6},
|
||||
RemoteEntry: remoteOnly,
|
||||
Extended: map[string][]byte{filer.ExtRemoteRenameSourceKey: []byte("/buckets/b/a.jpg")},
|
||||
}
|
||||
}
|
||||
event := func(oldName, newName string) *filer_pb.SubscribeMetadataResponse {
|
||||
newEntry := pendingC()
|
||||
newEntry.Name = newName
|
||||
return &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/buckets/b",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: oldName, RemoteEntry: remoteOnly},
|
||||
NewParentPath: "/buckets/b",
|
||||
NewEntry: newEntry,
|
||||
},
|
||||
}
|
||||
}
|
||||
objects := func() map[string][]byte {
|
||||
return map[string][]byte{"/b/a.jpg": []byte("data-a"), "/b/b.jpg": []byte("old-b"), "/b/c.jpg": []byte("old-c")}
|
||||
}
|
||||
|
||||
t.Run("the first rename leaves the source to the entry that moved on", func(t *testing.T) {
|
||||
remote := &recordingRemote{objects: objects()}
|
||||
filerClient := &stubFilerClient{} // nothing at b.jpg any more
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, event("a.jpg", "b.jpg")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.deletes) != 0 || len(remote.writes) != 0 {
|
||||
t.Errorf("deletes = %+v, writes = %+v, want neither: c.jpg still reads a.jpg", remote.deletes, remote.writes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the second rename copies from the marker's source", func(t *testing.T) {
|
||||
current := pendingC()
|
||||
remote := &recordingRemote{objects: objects(), statETag: `"a"`}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, event("b.jpg", "c.jpg")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.written) != 1 || string(remote.written[0]) != "data-a" {
|
||||
t.Errorf("written = %q, want a.jpg's content", remote.written)
|
||||
}
|
||||
if len(remote.deletes) != 1 || remote.deletes[0].Path != "/b/a.jpg" {
|
||||
t.Errorf("deletes = %+v, want the source once the stamp cleared the marker", remote.deletes)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPendingRenameSourceIsReleasedOnlyWhileUnchanged(t *testing.T) {
|
||||
const mountedDir = "/buckets"
|
||||
mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"}
|
||||
// b.jpg's copy failed after it replaced a target: it read a.jpg's object
|
||||
pendingB := func() *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: "b.jpg",
|
||||
RemoteEntry: &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: `"a"`, RemoteSize: 6},
|
||||
Extended: map[string][]byte{filer.ExtRemoteRenameSourceKey: []byte("/buckets/b/a.jpg")},
|
||||
}
|
||||
}
|
||||
deleted := func() *filer_pb.SubscribeMetadataResponse {
|
||||
return &filer_pb.SubscribeMetadataResponse{Directory: "/buckets/b", EventNotification: &filer_pb.EventNotification{OldEntry: pendingB()}}
|
||||
}
|
||||
deletedPaths := func(r *recordingRemote) (paths []string) {
|
||||
for _, d := range r.deletes {
|
||||
paths = append(paths, d.Path)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("deleting the pending entry releases its source", func(t *testing.T) {
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-a")}, statETag: `"a"`}
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, deleted()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := deletedPaths(remote); !slices.Contains(got, "/b/a.jpg") {
|
||||
t.Errorf("deletes = %v, want the source", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a source replaced since is kept", func(t *testing.T) {
|
||||
// another rename copied onto a.jpg before its entry became visible
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-d")}, statETag: `"d"`}
|
||||
if err := processDeleteEvent(&stubFilerClient{}, remote, mountedDir, mountLoc, deleted()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := deletedPaths(remote); slices.Contains(got, "/b/a.jpg") {
|
||||
t.Errorf("deletes = %v, want the replaced source kept", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a destination cached since releases the source too", func(t *testing.T) {
|
||||
// c.jpg was cached before the sync reached the rename: it is no
|
||||
// longer remote-only, and a.jpg is still left over
|
||||
cached := &filer_pb.Entry{Name: "c.jpg", Content: []byte("data-a"), Attributes: &filer_pb.FuseAttributes{FileSize: 6}, RemoteEntry: &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: `"a"`, RemoteSize: 6}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-a"), "/b/b.jpg": []byte("old-b"), "/b/c.jpg": []byte("data-a")}, statETag: `"a"`}
|
||||
event := &filer_pb.SubscribeMetadataResponse{Directory: "/buckets/b", EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: pendingB(), NewParentPath: "/buckets/b", NewEntry: cached,
|
||||
}}
|
||||
filerClient := &pathFilerClient{entries: map[string]*filer_pb.Entry{"/buckets/b/c.jpg": cached}}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := deletedPaths(remote); !slices.Contains(got, "/b/a.jpg") {
|
||||
t.Errorf("deletes = %v, want the pending source released", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a cached entry still reading the source keeps it until its upload lands", func(t *testing.T) {
|
||||
// c.jpg was cached through its marker; until the upload is stamped
|
||||
// and the marker cleared, a.jpg is what remote.uncache would fall
|
||||
// back to
|
||||
marked := func() *filer_pb.Entry {
|
||||
return &filer_pb.Entry{Name: "c.jpg", Content: []byte("data-a"), Attributes: &filer_pb.FuseAttributes{FileSize: 6},
|
||||
RemoteEntry: &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: `"a"`, RemoteSize: 6},
|
||||
Extended: map[string][]byte{filer.ExtRemoteRenameSourceKey: []byte("/buckets/b/a.jpg")}}
|
||||
}
|
||||
for name, writeErr := range map[string]error{"upload fails": errors.New("upload failed"), "stamp does not land": nil} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-a"), "/b/b.jpg": []byte("old-b")}, statETag: `"a"`, writeErr: writeErr}
|
||||
event := &filer_pb.SubscribeMetadataResponse{Directory: "/buckets/b", EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: pendingB(), NewParentPath: "/buckets/b", NewEntry: marked(),
|
||||
}}
|
||||
// the filer keeps the marker: the stub does not store updates
|
||||
filerClient := &pathFilerClient{entries: map[string]*filer_pb.Entry{"/buckets/b/c.jpg": marked()}}
|
||||
_ = processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, event)
|
||||
if got := deletedPaths(remote); slices.Contains(got, "/b/a.jpg") {
|
||||
t.Errorf("deletes = %v, want the source kept while the entry reads it", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a later rename the filer copied releases the source", func(t *testing.T) {
|
||||
// b.jpg was renamed to c.jpg and the filer copied a.jpg there at once
|
||||
copied := &filer_pb.Entry{Name: "c.jpg", Attributes: &filer_pb.FuseAttributes{FileSize: 6}, RemoteEntry: &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: `"a"`, RemoteSize: 6}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-a"), "/b/b.jpg": []byte("old-b"), "/b/c.jpg": []byte("data-a")}, statETag: `"a"`}
|
||||
event := &filer_pb.SubscribeMetadataResponse{Directory: "/buckets/b", EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: pendingB(), NewParentPath: "/buckets/b", NewEntry: copied,
|
||||
}}
|
||||
filerClient := &pathFilerClient{entries: map[string]*filer_pb.Entry{"/buckets/b/c.jpg": copied}}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, event); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := deletedPaths(remote); !slices.Contains(got, "/b/a.jpg") {
|
||||
t.Errorf("deletes = %v, want the pending source released", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// pathFilerClient serves entries by path; a path it lacks is not found.
|
||||
type pathFilerClient struct {
|
||||
stubFilerClient
|
||||
entries map[string]*filer_pb.Entry
|
||||
}
|
||||
|
||||
func (c *pathFilerClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: c.entries[req.Directory+"/"+req.Name]}, nil
|
||||
}
|
||||
|
||||
func (c *pathFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
return fn(c)
|
||||
}
|
||||
|
||||
func TestShouldSendToRemote(t *testing.T) {
|
||||
const mtime = 1786096669
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mtime int64
|
||||
mtimeNs int32
|
||||
remote *filer_pb.RemoteEntry
|
||||
want bool
|
||||
}{
|
||||
{name: "no RemoteEntry", mtime: mtime, want: true},
|
||||
{name: "remote older", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime - 1}, want: true},
|
||||
{name: "remote newer", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime + 1}, want: false},
|
||||
{name: "same second, no sync stamp", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime}, want: true},
|
||||
{name: "same second, synced at the second", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime * 1e9}, want: false},
|
||||
{name: "same second, synced after", mtime: mtime, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime*1e9 + 1}, want: false},
|
||||
{name: "same second, rewritten after the sync", mtime: mtime, mtimeNs: 337, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime*1e9 + 100}, want: true},
|
||||
{name: "same second, rewrite synced", mtime: mtime, mtimeNs: 337, remote: &filer_pb.RemoteEntry{RemoteMtime: mtime, LastLocalSyncTsNs: mtime*1e9 + 400}, want: false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
entry := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: tt.mtime, MtimeNs: tt.mtimeNs}, RemoteEntry: tt.remote}
|
||||
if got := shouldSendToRemote(entry); got != tt.want {
|
||||
t.Errorf("shouldSendToRemote = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A metadata-only update pushed to the remote must advance the sync stamp.
|
||||
// Without the restamp a same-second edit stays dirty forever and every
|
||||
// following event re-sends the metadata update.
|
||||
func TestMetadataOnlyUpdateRestampsSyncTs(t *testing.T) {
|
||||
const mountedDir = "/buckets"
|
||||
mountLoc := &remote_pb.RemoteStorageLocation{Name: "gcs", Bucket: "bucket", Path: "/"}
|
||||
|
||||
replicated := &filer_pb.RemoteEntry{StorageName: "gcs", RemoteETag: "abc", RemoteSize: 2048, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1}
|
||||
oldEntry := entryWith("a.txt", replicated, chunk("3,01", "e1"))
|
||||
newEntry := entryWith("a.txt", replicated, chunk("3,01", "e1"))
|
||||
resp := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/buckets/b/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: oldEntry,
|
||||
NewParentPath: "/buckets/b/dir",
|
||||
NewEntry: newEntry,
|
||||
},
|
||||
}
|
||||
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: newEntry}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.metadataUpdates) != 1 {
|
||||
t.Fatalf("metadata updates = %+v, want 1", remote.metadataUpdates)
|
||||
}
|
||||
if len(remote.writes) != 0 {
|
||||
t.Errorf("writes = %+v, want none: the content did not change", remote.writes)
|
||||
}
|
||||
if filerClient.updates != 1 {
|
||||
t.Fatalf("stamped %d times, want 1: the entry is restamped after the metadata update", filerClient.updates)
|
||||
}
|
||||
stamp := filerClient.updated.Entry.GetRemoteEntry().GetLastLocalSyncTsNs()
|
||||
if stamp <= 1 {
|
||||
t.Errorf("LastLocalSyncTsNs = %d, want it advanced past the previous stamp", stamp)
|
||||
}
|
||||
expected := filerClient.updated.Condition.Clauses[0].ExpectedEntry
|
||||
if got := expected.GetRemoteEntry().GetLastLocalSyncTsNs(); got != 1 {
|
||||
t.Errorf("precondition stamp = %d, want the stored stamp 1 or IF_ENTRY_EQUAL rejects the write", got)
|
||||
}
|
||||
if got := replicated.GetLastLocalSyncTsNs(); got != 1 {
|
||||
t.Errorf("shared RemoteEntry stamp = %d, want 1: stamping must not mutate the caller's RemoteEntry", got)
|
||||
}
|
||||
if shouldSendToRemote(filerClient.updated.Entry) {
|
||||
t.Error("the restamped entry should read synced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteRemoteOnlyRenameKeepsAReusedOldKey(t *testing.T) {
|
||||
// A was renamed to B, then D to A before the sync got to the first
|
||||
// rename: the object at A is now D's, and the only copy of it
|
||||
oldDest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/a.jpg"}
|
||||
dest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/b.jpg"}
|
||||
current := &filer_pb.Entry{Name: "b.jpg", RemoteEntry: &filer_pb.RemoteEntry{RemoteETag: `"a"`, RemoteSize: 6}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-d"), "/b/b.jpg": []byte("data-a")}, statETag: `"a"`}
|
||||
remote.statETags = map[string]string{"/b/a.jpg": `"d"`}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b", current, renameSource{path: "/buckets/b/a.jpg", loc: oldDest}, dest, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want the reused key kept", remote.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteRemoteOnlyRenameRetryUsesTheSourceStamp(t *testing.T) {
|
||||
// an earlier run copied a multipart source and stamped the entry with
|
||||
// the copy's ETag, then failed to delete the source
|
||||
oldDest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/a.jpg"}
|
||||
dest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/b.jpg"}
|
||||
current := &filer_pb.Entry{Name: "b.jpg", RemoteEntry: &filer_pb.RemoteEntry{RemoteETag: `"copy"`, RemoteSize: 6}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-a"), "/b/b.jpg": []byte("data-a")},
|
||||
statETags: map[string]string{"/b/a.jpg": `"a-2"`, "/b/b.jpg": `"copy"`}}
|
||||
source := renameSource{path: "/buckets/b/a.jpg", loc: oldDest, stamp: &filer_pb.RemoteEntry{RemoteETag: `"a-2"`, RemoteSize: 6}}
|
||||
if err := completeRemoteOnlyRename(&stubFilerClient{entry: current}, remote, "/buckets/b", current, source, dest, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.deletes) != 1 || remote.deletes[0].Path != "/b/a.jpg" {
|
||||
t.Errorf("deletes = %+v, want the unchanged source deleted on the retry", remote.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@ func TestFilerShutdownJoinsServeDuringParallelDrain(t *testing.T) {
|
||||
grpcStopped := make(chan struct{})
|
||||
filerClosed := make(chan struct{})
|
||||
var closes atomic.Int32
|
||||
shutdown := newFilerShutdown(func() {}, func() {
|
||||
shutdown := newFilerShutdown(func() {}, func() {}, func() {
|
||||
close(grpcStarted)
|
||||
<-grpcRelease
|
||||
close(grpcStopped)
|
||||
@@ -138,7 +138,7 @@ func TestFilerShutdownWaitsForHTTPAfterGrpcStops(t *testing.T) {
|
||||
server.Config.RegisterOnShutdown(func() { close(httpClosing) })
|
||||
grpcStopped := make(chan struct{})
|
||||
filerClosed := make(chan struct{})
|
||||
shutdown := newFilerShutdown(func() {}, func() { close(grpcStopped) }, func() { close(filerClosed) }, server.Config)
|
||||
shutdown := newFilerShutdown(func() {}, func() {}, func() { close(grpcStopped) }, func() { close(filerClosed) }, server.Config)
|
||||
joined := make(chan struct{})
|
||||
go func() { shutdown(); close(joined) }()
|
||||
<-httpClosing
|
||||
@@ -172,7 +172,7 @@ func TestFilerShutdownLeavesLockRingBeforeDraining(t *testing.T) {
|
||||
shutdown := newFilerShutdown(func() {
|
||||
close(leaving)
|
||||
<-releaseLeave
|
||||
}, func() { close(grpcStopping) }, func() {}, server.Config)
|
||||
}, func() {}, func() { close(grpcStopping) }, func() {}, server.Config)
|
||||
joined := make(chan struct{})
|
||||
go func() { shutdown(); close(joined) }()
|
||||
|
||||
@@ -196,3 +196,31 @@ func TestFilerShutdownLeavesLockRingBeforeDraining(t *testing.T) {
|
||||
t.Error("gRPC was not stopped after leaving the lock ring")
|
||||
}
|
||||
}
|
||||
|
||||
// The filer's own MetaAggregator and an in-process S3 gateway keep metadata
|
||||
// subscription streams open until the filer is shut down, which happens after
|
||||
// gRPC GracefulStop - so the subscriptions have to end before it, or every
|
||||
// shutdown sits out the full graceful-stop timeout.
|
||||
func TestFilerShutdownStopsSubscriptionsBeforeGrpc(t *testing.T) {
|
||||
var order []string
|
||||
var mu sync.Mutex
|
||||
record := func(step string) func() {
|
||||
return func() {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
order = append(order, step)
|
||||
}
|
||||
}
|
||||
shutdown := newFilerShutdown(record("leave"), record("subscriptions"), record("grpc"), record("filer"))
|
||||
shutdown()
|
||||
|
||||
want := []string{"leave", "subscriptions", "grpc", "filer"}
|
||||
if len(order) != len(want) {
|
||||
t.Fatalf("shutdown steps %v, want %v", order, want)
|
||||
}
|
||||
for i := range want {
|
||||
if order[i] != want[i] {
|
||||
t.Fatalf("shutdown steps %v, want %v", order, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
@@ -671,7 +672,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
|
||||
return nil
|
||||
}
|
||||
key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
|
||||
return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
|
||||
return sink.DeleteEntry(dataSink, key, message.OldEntry.IsDirectory, message.DeleteChunks, filer.IsMetadataOnlyDelete(message.OldEntry), message.Signatures)
|
||||
}
|
||||
|
||||
// handle new entries
|
||||
|
||||
@@ -82,7 +82,7 @@ func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir)
|
||||
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir, options.Address)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to build plugin worker handlers: %v", err)
|
||||
return false
|
||||
@@ -141,7 +141,7 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
|
||||
// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
|
||||
// (which may contain category names like "all", "default", "heavy" and/or
|
||||
// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
|
||||
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
|
||||
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string, workerAddress ...string) ([]pluginworker.JobHandler, error) {
|
||||
jobTypes = strings.TrimSpace(jobTypes)
|
||||
if jobTypes == "" {
|
||||
jobTypes = defaultPluginWorkerJobTypes
|
||||
@@ -152,10 +152,16 @@ func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxE
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addr := ""
|
||||
if len(workerAddress) > 0 {
|
||||
addr = strings.TrimSpace(workerAddress[0])
|
||||
}
|
||||
|
||||
opts := pluginworker.HandlerBuildOptions{
|
||||
GrpcDialOption: dialOption,
|
||||
MaxExecute: maxExecute,
|
||||
WorkingDir: workingDir,
|
||||
WorkerAddress: addr,
|
||||
}
|
||||
|
||||
handlers := make([]pluginworker.JobHandler, 0, len(factories))
|
||||
|
||||
@@ -458,13 +458,13 @@ func retriedStreamFetchChunkData(ctx context.Context, writer io.Writer, urlStrin
|
||||
localProcessed += writtenCount
|
||||
totalWritten += writtenCount
|
||||
})
|
||||
if !shouldRetry {
|
||||
break
|
||||
}
|
||||
if writeErr != nil {
|
||||
err = writeErr
|
||||
break
|
||||
}
|
||||
if !shouldRetry {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
failed = true
|
||||
glog.V(0).InfofCtx(ctx, "read %s failed, err: %v", urlString, err)
|
||||
|
||||
@@ -338,7 +338,10 @@ func TestCompactResolvedOverlappingManifests(t *testing.T) {
|
||||
}
|
||||
|
||||
// CompactFileChunks on resolved sub-chunks (nil lookupFn is fine for non-manifest chunks)
|
||||
compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
|
||||
compacted, garbage, err := CompactFileChunks(context.Background(), nil, dataChunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(compacted) != 3 {
|
||||
t.Fatalf("expected 3 compacted chunks, got %d", len(compacted))
|
||||
@@ -487,7 +490,10 @@ func TestCompactMultipleOverlappingManifestGenerations(t *testing.T) {
|
||||
}
|
||||
|
||||
// Compact: only generation 5 (the newest 3 chunks) should survive
|
||||
compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
|
||||
compacted, garbage, err := CompactFileChunks(context.Background(), nil, dataChunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(compacted) != 3 {
|
||||
t.Errorf("expected 3 compacted (gen 5), got %d", len(compacted))
|
||||
}
|
||||
@@ -781,3 +787,23 @@ func TestFetchWholeChunkUsesProvidedFilerJwt(t *testing.T) {
|
||||
return ""
|
||||
}))
|
||||
}
|
||||
|
||||
// A completed fetch whose destination write failed is a failed copy, not a
|
||||
// short success: the caller must see the write error, or a temp disk filling
|
||||
// mid-copy would still upload the truncated file.
|
||||
func TestRetriedStreamFetchChunkDataReportsWriteError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("0123456789"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
written, err := retriedStreamFetchChunkData(context.Background(), &failOnWrite{}, []string{srv.URL + "/5,abc"}, "", nil, false, true, 0, 0, nil)
|
||||
assert.Error(t, err)
|
||||
assert.Zero(t, written)
|
||||
}
|
||||
|
||||
type failOnWrite struct{}
|
||||
|
||||
func (w *failOnWrite) Write(p []byte) (int, error) {
|
||||
return 0, fmt.Errorf("no space left")
|
||||
}
|
||||
|
||||
@@ -63,9 +63,15 @@ func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
|
||||
return finalETag
|
||||
}
|
||||
|
||||
func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
|
||||
// CompactFileChunks splits chunks into those still visible and those fully
|
||||
// covered by newer writes. It fails if the chunk manifests cannot be resolved,
|
||||
// including when ctx is cancelled.
|
||||
func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk, err error) {
|
||||
|
||||
visibles, _ := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
|
||||
visibles, err := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
compacted, garbage = SeparateGarbageChunks(visibles, chunks)
|
||||
|
||||
@@ -73,6 +79,9 @@ func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileId
|
||||
}
|
||||
|
||||
func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*filer_pb.FileChunk) (compacted []*filer_pb.FileChunk, garbage []*filer_pb.FileChunk) {
|
||||
if visibles == nil {
|
||||
return chunks, nil
|
||||
}
|
||||
fileIds := make(map[string]bool)
|
||||
for x := visibles.Front(); x != nil; x = x.Next {
|
||||
interval := x.Value
|
||||
@@ -194,11 +203,17 @@ func (cv *ChunkView) CanRangeFetch() bool {
|
||||
return cv.CipherKey == nil && !cv.IsGzipped
|
||||
}
|
||||
|
||||
func ViewFromChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
|
||||
// ViewFromChunks returns the views of chunks that cover [offset, offset+size).
|
||||
// It fails if the chunk manifests cannot be resolved, including when ctx is
|
||||
// cancelled.
|
||||
func ViewFromChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView], err error) {
|
||||
|
||||
visibles, _ := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, offset, offset+size)
|
||||
visibles, err := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, offset, offset+size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ViewFromVisibleIntervals(visibles, offset, size)
|
||||
return ViewFromVisibleIntervals(visibles, offset, size), nil
|
||||
|
||||
}
|
||||
|
||||
@@ -213,6 +228,9 @@ func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset i
|
||||
}
|
||||
|
||||
chunkViews = NewIntervalList[*ChunkView]()
|
||||
if visibles == nil {
|
||||
return chunkViews
|
||||
}
|
||||
for x := visibles.Front(); x != nil; x = x.Next {
|
||||
chunk := x.Value
|
||||
|
||||
|
||||
@@ -66,7 +66,10 @@ func TestCompactFileChunksRealCase(t *testing.T) {
|
||||
|
||||
printChunks("before", chunks)
|
||||
|
||||
compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
|
||||
compacted, garbage, err := CompactFileChunks(context.Background(), nil, chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
printChunks("compacted", compacted)
|
||||
printChunks("garbage", garbage)
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// cancelledChunks returns a cancelled context and plain data chunks. Resolving
|
||||
// chunks under a cancelled context fails even without manifests, so these
|
||||
// exercise the error path with no lookup function.
|
||||
func cancelledChunks() (context.Context, []*filer_pb.FileChunk) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
return ctx, []*filer_pb.FileChunk{
|
||||
{FileId: "1,01", Offset: 0, Size: 100, ModifiedTsNs: 1},
|
||||
{FileId: "1,02", Offset: 0, Size: 100, ModifiedTsNs: 2},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactFileChunksCancelledContext(t *testing.T) {
|
||||
ctx, chunks := cancelledChunks()
|
||||
|
||||
compacted, garbage, err := CompactFileChunks(ctx, nil, chunks)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err = %v, want context.Canceled", err)
|
||||
}
|
||||
if compacted != nil || garbage != nil {
|
||||
t.Fatalf("compacted = %v, garbage = %v, want both nil on error", compacted, garbage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewFromChunksCancelledContext(t *testing.T) {
|
||||
ctx, chunks := cancelledChunks()
|
||||
|
||||
views, err := ViewFromChunks(ctx, nil, chunks, 0, 100)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err = %v, want context.Canceled", err)
|
||||
}
|
||||
if views != nil {
|
||||
t.Fatalf("views = %v, want nil on error", views)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkStreamReaderCancelledContext(t *testing.T) {
|
||||
ctx, chunks := cancelledChunks()
|
||||
|
||||
reader := NewChunkStreamReaderFromLookup(ctx, nil, chunks)
|
||||
n, err := reader.Read(make([]byte, 10))
|
||||
if n != 0 || !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("Read = %d, %v, want 0, context.Canceled", n, err)
|
||||
}
|
||||
if !errors.Is(reader.SourceError(), context.Canceled) {
|
||||
t.Fatalf("SourceError = %v, want context.Canceled", reader.SourceError())
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,10 @@ func TestCompactFileChunks(t *testing.T) {
|
||||
{Offset: 110, Size: 200, FileId: "jkl", ModifiedTsNs: 300},
|
||||
}
|
||||
|
||||
compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
|
||||
compacted, garbage, err := CompactFileChunks(context.Background(), nil, chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(compacted) != 3 {
|
||||
t.Fatalf("unexpected compacted: %d", len(compacted))
|
||||
@@ -55,7 +58,10 @@ func TestCompactFileChunks2(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
|
||||
compacted, garbage, err := CompactFileChunks(context.Background(), nil, chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(compacted) != 4 {
|
||||
t.Fatalf("unexpected compacted: %d", len(compacted))
|
||||
@@ -427,7 +433,10 @@ func TestChunksReading(t *testing.T) {
|
||||
// continue
|
||||
}
|
||||
log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i)
|
||||
chunks := ViewFromChunks(context.Background(), nil, testcase.Chunks, testcase.Offset, testcase.Size)
|
||||
chunks, err := ViewFromChunks(context.Background(), nil, testcase.Chunks, testcase.Offset, testcase.Size)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
x := -1
|
||||
for c := chunks.Front(); c != nil; c = c.Next {
|
||||
x++
|
||||
@@ -474,7 +483,9 @@ func BenchmarkCompactFileChunks(b *testing.B) {
|
||||
}
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
CompactFileChunks(context.Background(), nil, chunks)
|
||||
if _, _, err := CompactFileChunks(context.Background(), nil, chunks); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -563,9 +574,45 @@ func TestCompactFileChunks3(t *testing.T) {
|
||||
{Offset: 300, Size: 100, FileId: "def", ModifiedTsNs: 200},
|
||||
}
|
||||
|
||||
compacted, _ := CompactFileChunks(context.Background(), nil, chunks)
|
||||
compacted, _, err := CompactFileChunks(context.Background(), nil, chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(compacted) != 4 {
|
||||
t.Fatalf("unexpected compacted: %d", len(compacted))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateGarbageChunksNilVisibles(t *testing.T) {
|
||||
chunks := []*filer_pb.FileChunk{
|
||||
{Offset: 10, Size: 100, FileId: "abc", ModifiedTsNs: 50},
|
||||
}
|
||||
compacted, garbage := SeparateGarbageChunks(nil, chunks)
|
||||
if len(compacted) != 1 || len(garbage) != 0 {
|
||||
t.Fatalf("expected fallback to original chunks with nil garbage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewFromVisibleIntervalsNilVisibles(t *testing.T) {
|
||||
views := ViewFromVisibleIntervals(nil, 0, 100)
|
||||
if views == nil || views.Front() != nil {
|
||||
t.Fatalf("expected empty interval list")
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewFromChunksPropagatesError(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
chunks := []*filer_pb.FileChunk{
|
||||
{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 50, IsChunkManifest: true},
|
||||
}
|
||||
views, err := ViewFromChunks(ctx, nil, chunks, 0, math.MaxInt64)
|
||||
if err == nil {
|
||||
t.Fatal("expected error on cancelled context")
|
||||
}
|
||||
if views != nil {
|
||||
t.Fatalf("expected nil views on error, got %v", views)
|
||||
}
|
||||
}
|
||||
|
||||
+111
-5
@@ -82,6 +82,11 @@ type Filer struct {
|
||||
persistedLogCache *persistedLogCache
|
||||
metaLogInflight metaLogInflight
|
||||
remoteTombstones *remoteDeletionTombstones
|
||||
// EntryLockTable serializes mutations per entry path. The filer server
|
||||
// holds it across local read-modify-write mutations and the meta
|
||||
// aggregator across a replayed peer change's check-and-apply, so the two
|
||||
// cannot interleave on the same path.
|
||||
EntryLockTable *util.LockTable[util.FullPath]
|
||||
// remoteTombstonesDone, when non-nil, is closed once the startup tombstone
|
||||
// rebuild finishes; lazy remote reads wait on it so a pending delete
|
||||
// cannot resurrect in the gap.
|
||||
@@ -110,6 +115,12 @@ type Filer struct {
|
||||
// overwriting the unread ledger with a partial set.
|
||||
deletionLedgerBlocked atomic.Bool
|
||||
deletionLedgerFlush chan struct{}
|
||||
// metaTsPending holds per-path commit records whose KV write failed,
|
||||
// keyed by path; ordering reads consult it until the retry lands them
|
||||
// (see meta_path_ts.go). metaTsRepairing marks paths with a live
|
||||
// repair goroutine so failures neither strand a record nor pile on.
|
||||
metaTsPending sync.Map
|
||||
metaTsRepairing sync.Map
|
||||
// flushCtx is the context metadata-log flushes append under. It stays
|
||||
// live for the process lifetime and is cancelled once during Shutdown,
|
||||
// bounding every retry - in-flight and queued alike - to one deadline.
|
||||
@@ -133,6 +144,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
|
||||
DeletionRetryQueue: NewDeletionRetryQueue(),
|
||||
persistedLogCache: newPersistedLogCache(persistedLogCacheMaxBytes),
|
||||
remoteTombstones: newRemoteDeletionTombstones(),
|
||||
EntryLockTable: util.NewLockTable[util.FullPath](),
|
||||
}
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
if f.UniqueFilerId < 0 {
|
||||
@@ -254,8 +266,96 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
|
||||
|
||||
}
|
||||
|
||||
func (f *Filer) ListExistingPeerUpdates(ctx context.Context) (existingNodes []*master_pb.ClusterNodeUpdate) {
|
||||
return cluster.ListExistingPeerUpdates(f.GetMaster(ctx), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
|
||||
// peerLookupTimeout bounds each master query so one unresponsive master
|
||||
// cannot stall the peer scan.
|
||||
const peerLookupTimeout = 10 * time.Second
|
||||
|
||||
// leaderSettleTimeout bounds how long an empty peer list from the leader is
|
||||
// re-polled: a freshly elected leader has not yet received the filers'
|
||||
// reconnects, so an immediate empty answer can just mean membership has not
|
||||
// migrated yet. A persistent empty answer is then checked against the other
|
||||
// masters below before it is accepted as an empty cluster.
|
||||
const leaderSettleTimeout = 10 * time.Second
|
||||
const leaderSettleInterval = time.Second
|
||||
|
||||
// ListExistingPeerUpdates returns the peer list to bootstrap from. The
|
||||
// leader's list is authoritative: after a leader change an old master keeps
|
||||
// its registrations until its streams end, so the longest answer is not
|
||||
// necessarily the current one. The leader's answer wins whenever it can be
|
||||
// found and queried; only when the leader is unreachable does the scan fall
|
||||
// back to the fullest answer from the other masters. It returns an error when
|
||||
// no master answers, so a failed lookup is not mistaken for an empty cluster.
|
||||
func (f *Filer) ListExistingPeerUpdates(ctx context.Context) (existingNodes []*master_pb.ClusterNodeUpdate, err error) {
|
||||
masters := f.MasterClient.ListMasters()
|
||||
var queryErrs []error
|
||||
|
||||
var leader pb.ServerAddress
|
||||
for _, master := range masters {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
lookupCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
|
||||
l, lookErr := cluster.LookupClusterLeader(lookupCtx, master, f.GrpcDialOption)
|
||||
cancel()
|
||||
if lookErr != nil {
|
||||
queryErrs = append(queryErrs, lookErr)
|
||||
continue
|
||||
}
|
||||
if l != "" {
|
||||
leader = l
|
||||
break
|
||||
}
|
||||
}
|
||||
if leader != "" {
|
||||
settleDeadline := time.Now().Add(leaderSettleTimeout)
|
||||
for {
|
||||
lookupCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
|
||||
nodes, listErr := cluster.ListExistingPeerUpdates(lookupCtx, leader, f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
|
||||
cancel()
|
||||
if listErr != nil {
|
||||
queryErrs = append(queryErrs, fmt.Errorf("list peers from leader %s: %w", leader, listErr))
|
||||
break
|
||||
}
|
||||
if len(nodes) > 0 {
|
||||
return nodes, nil
|
||||
}
|
||||
if ctx.Err() != nil || !time.Now().Before(settleDeadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(leaderSettleInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
answered := false
|
||||
for _, master := range masters {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
lookupCtx, cancel := context.WithTimeout(ctx, peerLookupTimeout)
|
||||
nodes, listErr := cluster.ListExistingPeerUpdates(lookupCtx, master, f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
|
||||
cancel()
|
||||
if listErr != nil {
|
||||
queryErrs = append(queryErrs, listErr)
|
||||
continue
|
||||
}
|
||||
answered = true
|
||||
if len(nodes) > len(existingNodes) {
|
||||
existingNodes = nodes
|
||||
}
|
||||
}
|
||||
if !answered {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
if err = errors.Join(queryErrs...); err == nil {
|
||||
err = fmt.Errorf("no masters configured")
|
||||
}
|
||||
return nil, fmt.Errorf("list existing peers: %w", err)
|
||||
}
|
||||
return existingNodes, nil
|
||||
}
|
||||
|
||||
func (f *Filer) SetStore(store FilerStore) (isFresh bool) {
|
||||
@@ -339,9 +439,9 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
|
||||
if oldEntry == nil {
|
||||
var findErr error
|
||||
oldEntry, findErr = f.FindEntry(ctx, entry.FullPath)
|
||||
if o_excl && findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
|
||||
// An exclusive create cannot decide whether the path exists when
|
||||
// the lookup itself failed; proceeding would upsert over it.
|
||||
if findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
|
||||
// A create cannot decide whether the path exists when the lookup
|
||||
// itself failed; proceeding would upsert over it.
|
||||
return fmt.Errorf("find entry %s: %w", entry.FullPath, findErr)
|
||||
}
|
||||
}
|
||||
@@ -825,6 +925,12 @@ func (f *Filer) Shutdown() {
|
||||
f.LocalMetaLogBuffer.ShutdownLogBuffer()
|
||||
// The final metadata-log flush still needs the store to append its entry.
|
||||
f.LocalMetaLogBuffer.WaitForShutdown()
|
||||
// Stop peer subscriptions and let each persist its last applied offset;
|
||||
// the replay on the next start otherwise resumes from a stale offset and
|
||||
// applies those old events over newer local state.
|
||||
if f.MetaAggregator != nil {
|
||||
f.MetaAggregator.Shutdown()
|
||||
}
|
||||
// Persist the deletion ledger one last time before the store closes, so a
|
||||
// clean shutdown leaves the recovery set exactly consistent with reality.
|
||||
f.snapshotDeletionLedger()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -52,6 +53,46 @@ type deleteEntryError struct {
|
||||
func (e *deleteEntryError) Error() string { return e.msg }
|
||||
func (e *deleteEntryError) Unwrap() error { return e.cause }
|
||||
|
||||
// ExtKeepRemoteObjectKey marks the entry in the event of a metadata-only
|
||||
// delete, so the remote write-back daemons leave the remote object alone too.
|
||||
// It is set on the event only; the store drops it from every write, so a
|
||||
// client cannot plant it on an entry.
|
||||
const ExtKeepRemoteObjectKey = "Seaweed-X-Keep-Remote-Object"
|
||||
|
||||
// IsMetadataOnlyDelete reports a delete event's entry marked by
|
||||
// WithKeepRemoteObject.
|
||||
func IsMetadataOnlyDelete(oldEntry *filer_pb.Entry) bool {
|
||||
_, marked := oldEntry.GetExtended()[ExtKeepRemoteObjectKey]
|
||||
return marked
|
||||
}
|
||||
|
||||
type keepRemoteObjectKey struct{}
|
||||
|
||||
// WithKeepRemoteObject marks a delete as metadata-only: neither the filer nor
|
||||
// filer.remote.sync deletes the entry's object on the mounted remote storage.
|
||||
// Unlike isFromOtherCluster, the metadata event stays an ordinary local
|
||||
// delete, so replication still carries it.
|
||||
func WithKeepRemoteObject(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, keepRemoteObjectKey{}, true)
|
||||
}
|
||||
|
||||
func keepsRemoteObject(ctx context.Context) bool {
|
||||
return ctx.Value(keepRemoteObjectKey{}) != nil
|
||||
}
|
||||
|
||||
func deleteEventEntry(ctx context.Context, entry *Entry) *Entry {
|
||||
if !keepsRemoteObject(ctx) {
|
||||
return entry
|
||||
}
|
||||
marked := entry.ShallowClone()
|
||||
marked.Extended = maps.Clone(entry.Extended)
|
||||
if marked.Extended == nil {
|
||||
marked.Extended = map[string][]byte{}
|
||||
}
|
||||
marked.Extended[ExtKeepRemoteObjectKey] = []byte("true")
|
||||
return marked
|
||||
}
|
||||
|
||||
type OnChunksFunc func([]*filer_pb.FileChunk) error
|
||||
type OnHardLinkIdsFunc func([]HardLinkId) error
|
||||
|
||||
@@ -165,7 +206,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
subIsDeletingBucket := f.IsBucket(sub)
|
||||
err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, isFromOtherCluster, nil, onHardLinkIdsFn)
|
||||
} else {
|
||||
if !isFromOtherCluster {
|
||||
if !isFromOtherCluster && !keepsRemoteObject(ctx) {
|
||||
if _, remoteErr := f.maybeDeleteFromRemote(ctx, sub); remoteErr != nil {
|
||||
glog.Warningf("remote delete child %s: %v", sub.FullPath, remoteErr)
|
||||
if !ignoreRecursiveError {
|
||||
@@ -176,7 +217,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
if err != nil && !ignoreRecursiveError {
|
||||
break
|
||||
}
|
||||
f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster, nil)
|
||||
f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, sub), nil, shouldDeleteChunks, isFromOtherCluster, nil)
|
||||
if len(sub.HardLinkId) != 0 {
|
||||
// hard link chunk data are deleted separately
|
||||
err = onHardLinkIdsFn([]HardLinkId{sub.HardLinkId})
|
||||
@@ -207,7 +248,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
}
|
||||
}
|
||||
|
||||
f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster, signatures)
|
||||
f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, entry), nil, shouldDeleteChunks, isFromOtherCluster, signatures)
|
||||
f.DeleteChunks(ctx, entry.FullPath, chunksToDelete)
|
||||
|
||||
return nil
|
||||
@@ -217,7 +258,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
|
||||
|
||||
glog.V(3).InfofCtx(ctx, "deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks)
|
||||
|
||||
if !isFromOtherCluster {
|
||||
if !isFromOtherCluster && !keepsRemoteObject(ctx) {
|
||||
if _, remoteDeletionErr := f.maybeDeleteFromRemote(ctx, entry); remoteDeletionErr != nil {
|
||||
return remoteDeletionErr
|
||||
}
|
||||
@@ -230,7 +271,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
|
||||
}
|
||||
|
||||
if !entry.IsDirectory() {
|
||||
f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster, signatures)
|
||||
f.NotifyUpdateEvent(ctx, deleteEventEntry(ctx, entry), nil, shouldDeleteChunks, isFromOtherCluster, signatures)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The path in a non-empty-folder failure is the client's, so the marker has to
|
||||
@@ -27,3 +31,33 @@ func TestNonEmptyFolderClassification(t *testing.T) {
|
||||
t.Errorf("expected no forgery from the entry name: %v", spoofed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeepRemoteObjectMarkerOnlyLivesOnDeleteEvents(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newTestFiler(t, store, NewFilerRemoteStorage())
|
||||
ctx := context.Background()
|
||||
planted := func() map[string][]byte {
|
||||
return map[string][]byte{ExtKeepRemoteObjectKey: []byte("true"), "Seaweed-Other": []byte("kept")}
|
||||
}
|
||||
dir := &Entry{FullPath: "/buckets/b/dir", Attr: Attr{Mode: os.ModeDir | 0755}}
|
||||
file := &Entry{FullPath: "/buckets/b/dir/obj.bin", Attr: Attr{Mode: 0644}, Extended: planted()}
|
||||
require.NoError(t, f.CreateEntry(ctx, dir, nil, false, false, nil, true, 255))
|
||||
require.NoError(t, f.CreateEntry(ctx, file, nil, false, false, nil, true, 255))
|
||||
updated := file.ShallowClone()
|
||||
updated.Extended = planted()
|
||||
require.NoError(t, f.UpdateEntry(ctx, file, updated, false))
|
||||
|
||||
stored := store.entries[string(file.FullPath)]
|
||||
require.NotContains(t, stored.Extended, ExtKeepRemoteObjectKey, "a client must not be able to store the marker")
|
||||
require.Contains(t, stored.Extended, "Seaweed-Other")
|
||||
|
||||
ordinaryCtx, ordinary := WithMetadataEventSink(ctx)
|
||||
require.NoError(t, f.DeleteEntryMetaAndData(ordinaryCtx, file.FullPath, false, false, false, false, nil, 0))
|
||||
require.False(t, IsMetadataOnlyDelete(ordinary.Last().EventNotification.OldEntry))
|
||||
|
||||
require.NoError(t, f.CreateEntry(ctx, &Entry{FullPath: file.FullPath, Attr: Attr{Mode: 0644}}, nil, false, false, nil, true, 255))
|
||||
keepCtx, keep := WithMetadataEventSink(WithKeepRemoteObject(ctx))
|
||||
require.NoError(t, f.DeleteEntryMetaAndData(keepCtx, dir.FullPath, true, false, false, false, nil, 0))
|
||||
require.Equal(t, "dir", keep.Last().EventNotification.OldEntry.Name)
|
||||
require.True(t, IsMetadataOnlyDelete(keep.Last().EventNotification.OldEntry))
|
||||
}
|
||||
|
||||
@@ -265,3 +265,28 @@ func TestUpdateEntryBackfillsMissingLegacyInode(t *testing.T) {
|
||||
assert.NotZero(t, stored.Attr.Inode)
|
||||
assert.NotEqual(t, uint64(1), stored.Attr.Inode)
|
||||
}
|
||||
|
||||
// A transient store error must fail the write instead of being mistaken for a
|
||||
// missing entry: on upsert stores proceeding would replace the entry without
|
||||
// loading the old chunks, orphaning them from cleanup.
|
||||
func TestCreateEntryFailsWhenLookupErrors(t *testing.T) {
|
||||
f, store := newTestFilerWithStubStore()
|
||||
path := util.FullPath("/dir/file.txt")
|
||||
require.NoError(t, store.InsertEntry(context.Background(), &Entry{
|
||||
FullPath: path,
|
||||
Content: []byte("existing"),
|
||||
}))
|
||||
store.findErr = errors.New("transient store failure")
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mode: 0o644},
|
||||
Content: []byte("overwrite"),
|
||||
}
|
||||
require.Error(t, f.CreateEntry(context.Background(), entry, nil, false, false, nil, false, f.MaxFilenameLength))
|
||||
|
||||
store.findErr = nil
|
||||
stored, findErr := store.FindEntry(context.Background(), path)
|
||||
require.NoError(t, findErr)
|
||||
assert.Equal(t, "existing", string(stored.Content))
|
||||
}
|
||||
|
||||
@@ -225,3 +225,97 @@ func (f *Filer) maybeDeleteFromRemote(ctx context.Context, entry *Entry) (bool,
|
||||
glog.V(3).InfofCtx(ctx, "maybeDeleteFromRemote: deleted %s from remote", entry.FullPath)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// DeleteFromRemote deletes the object, or for a directory every object, the
|
||||
// entry maps to on its mounted remote storage.
|
||||
func (f *Filer) DeleteFromRemote(ctx context.Context, entry *Entry) error {
|
||||
_, err := f.maybeDeleteFromRemote(ctx, entry)
|
||||
return err
|
||||
}
|
||||
|
||||
// ExtRemoteRenameSourceKey marks a remote-only entry whose rename committed
|
||||
// while its copy to the new key was still pending: it holds the path whose
|
||||
// object remote-only reads must serve until the copy stamps the entry.
|
||||
const ExtRemoteRenameSourceKey = "Seaweed-X-Remote-Rename-Source"
|
||||
|
||||
// RemoteSourcePath returns the path whose remote object holds the content of
|
||||
// the remote-only entry at p: the renamed-from path a pending deferred copy
|
||||
// still reads, or p itself. A marker leading out of p's mount is ignored, so
|
||||
// whoever can set extended attributes cannot point reads at another mount.
|
||||
func (f *Filer) RemoteSourcePath(entry *Entry, p util.FullPath) util.FullPath {
|
||||
src := util.FullPath(entry.Extended[ExtRemoteRenameSourceKey])
|
||||
if src == "" || f.RemoteStorage == nil {
|
||||
return p
|
||||
}
|
||||
mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(p)
|
||||
if remoteLoc == nil {
|
||||
return p
|
||||
}
|
||||
if srcMountDir, _ := f.RemoteStorage.FindMountDirectory(src); srcMountDir != mountDir {
|
||||
return p
|
||||
}
|
||||
return src
|
||||
}
|
||||
|
||||
// RemoteReadPath returns the dir and name whose object a remote-only read of
|
||||
// the entry at dir/name serves.
|
||||
func (f *Filer) RemoteReadPath(entry *Entry, dir, name string) (string, string) {
|
||||
return f.RemoteSourcePath(entry, util.NewFullPath(dir, name)).DirAndName()
|
||||
}
|
||||
|
||||
// CopyRemoteOnlyEntry copies the remote object of a renamed entry whose
|
||||
// content exists only on the remote to the key of newPath, and returns the
|
||||
// RemoteEntry of the copy. The rename deletes the old key along with the old
|
||||
// entry, and with no local data nothing would write the new key. It returns
|
||||
// nil when no copy is needed: the entry has local data, which the remote sync
|
||||
// uploads to the new key, it is empty, which the sync writes as an empty
|
||||
// object, or it is not under a remote mount. A rename that
|
||||
// moves such an entry out of its mount, or onto a remote that cannot copy, is
|
||||
// refused, since it would delete the only copy of the content.
|
||||
func (f *Filer) CopyRemoteOnlyEntry(ctx context.Context, entry *Entry, oldPath, newPath util.FullPath) (*filer_pb.RemoteEntry, error) {
|
||||
src, dst, copier, err := f.RemoteCopyLocations(ctx, entry, oldPath, newPath)
|
||||
if err != nil || copier == nil {
|
||||
return nil, err
|
||||
}
|
||||
remoteEntry, err := copier.CopyFile(src, dst)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("copy remote %s to %s: %w", remote_storage.FormatLocation(src), remote_storage.FormatLocation(dst), err)
|
||||
}
|
||||
glog.V(3).InfofCtx(ctx, "CopyRemoteOnlyEntry: copied %s to %s", remote_storage.FormatLocation(src), remote_storage.FormatLocation(dst))
|
||||
return remoteEntry, nil
|
||||
}
|
||||
|
||||
// RemoteCopyLocations resolves the object locations a remote-only rename copy
|
||||
// of entry between oldPath and newPath would use, and the client able to copy
|
||||
// it. A nil copier means the entry needs no copy; an error means it needs one
|
||||
// that cannot be served, which a rename must refuse before committing.
|
||||
func (f *Filer) RemoteCopyLocations(ctx context.Context, entry *Entry, oldPath, newPath util.FullPath) (src, dst *remote_pb.RemoteStorageLocation, copier remote_storage.RemoteStorageObjectCopier, err error) {
|
||||
if f.RemoteStorage == nil || entry.IsDirectory() || !entry.IsInRemoteOnly() || len(entry.Content) > 0 {
|
||||
return nil, nil, nil, nil
|
||||
}
|
||||
|
||||
mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(oldPath)
|
||||
if remoteLoc == nil {
|
||||
return nil, nil, nil, nil
|
||||
}
|
||||
if newMountDir, _ := f.RemoteStorage.FindMountDirectory(newPath); newMountDir != mountDir {
|
||||
return nil, nil, nil, fmt.Errorf("%s exists only on the remote of mount %s and cannot be renamed out of it; cache it first", oldPath, mountDir)
|
||||
}
|
||||
|
||||
remoteConf, found := f.RemoteStorage.GetRemoteStorageConf(remoteLoc.Name)
|
||||
if !found {
|
||||
return nil, nil, nil, fmt.Errorf("resolve remote storage client for %s: not found", oldPath)
|
||||
}
|
||||
client, clientErr := f.buildRemoteStorageClient(ctx, remoteConf)
|
||||
if clientErr != nil {
|
||||
return nil, nil, nil, fmt.Errorf("resolve remote storage client for %s: %w", oldPath, clientErr)
|
||||
}
|
||||
copier, ok := client.(remote_storage.RemoteStorageObjectCopier)
|
||||
if !ok {
|
||||
return nil, nil, nil, fmt.Errorf("%s exists only on the remote and remote storage type %s cannot copy it to the new name; cache it first", oldPath, remoteConf.Type)
|
||||
}
|
||||
|
||||
src = MapFullPathToRemoteStorageLocation(mountDir, remoteLoc, f.RemoteSourcePath(entry, oldPath))
|
||||
dst = MapFullPathToRemoteStorageLocation(mountDir, remoteLoc, newPath)
|
||||
return src, dst, copier, nil
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ type stubFilerStore struct {
|
||||
kv map[string][]byte
|
||||
insertErr error
|
||||
findErr error
|
||||
kvPutErr error
|
||||
kvGetErr error
|
||||
kvDeleteErr error
|
||||
kvGetHook func(key []byte)
|
||||
@@ -49,6 +50,12 @@ func newStubFilerStore() *stubFilerStore {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *stubFilerStore) setKvPutErr(err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.kvPutErr = err
|
||||
}
|
||||
|
||||
func (s *stubFilerStore) GetName() string { return "stub" }
|
||||
func (s *stubFilerStore) Initialize(util.Configuration, string) error { return nil }
|
||||
func (s *stubFilerStore) Shutdown() {}
|
||||
@@ -60,6 +67,9 @@ func (s *stubFilerStore) RollbackTransaction(context.Context) error { return nil
|
||||
func (s *stubFilerStore) KvPut(_ context.Context, key []byte, value []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.kvPutErr != nil {
|
||||
return s.kvPutErr
|
||||
}
|
||||
s.kv[string(key)] = append([]byte(nil), value...)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -71,6 +71,10 @@ func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry
|
||||
}
|
||||
|
||||
f.logMetaEvent(ctx, event)
|
||||
// Record the commit timestamp so a peer replay ordered by the entry lock
|
||||
// this mutation may still be under sees the local change, including as a
|
||||
// tombstone after the entry itself is gone.
|
||||
f.noteCommittedEvent(event)
|
||||
if sink := metadataEventSinkFromContext(ctx); sink != nil {
|
||||
sink.Record(event)
|
||||
}
|
||||
|
||||
@@ -141,6 +141,7 @@ func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) err
|
||||
|
||||
filer_pb.BeforeEntrySerialization(entry.GetChunks())
|
||||
normalizeEntryMimeForStore(entry)
|
||||
delete(entry.Extended, ExtKeepRemoteObjectKey)
|
||||
|
||||
if len(entry.HardLinkId) > 0 {
|
||||
glog.V(4).InfofCtx(ctx, "InsertEntry %s has HardLinkId %x counter=%d",
|
||||
@@ -170,6 +171,7 @@ func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry
|
||||
|
||||
filer_pb.BeforeEntrySerialization(entry.GetChunks())
|
||||
normalizeEntryMimeForStore(entry)
|
||||
delete(entry.Extended, ExtKeepRemoteObjectKey)
|
||||
|
||||
if len(entry.HardLinkId) > 0 {
|
||||
glog.V(4).InfofCtx(ctx, "InsertEntryKnownAbsent %s has HardLinkId %x counter=%d",
|
||||
@@ -196,6 +198,7 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
|
||||
|
||||
filer_pb.BeforeEntrySerialization(entry.GetChunks())
|
||||
normalizeEntryMimeForStore(entry)
|
||||
delete(entry.Extended, ExtKeepRemoteObjectKey)
|
||||
|
||||
if len(entry.HardLinkId) > 0 {
|
||||
glog.V(4).InfofCtx(ctx, "UpdateEntry %s has HardLinkId %x counter=%d",
|
||||
|
||||
+211
-54
@@ -63,6 +63,11 @@ type MetaAggregator struct {
|
||||
flushAdvanced chan struct{}
|
||||
remotePeerArrived chan struct{}
|
||||
peerWatermarksLock sync.Mutex
|
||||
peerWg sync.WaitGroup
|
||||
// stopped is set under peerChansLock by Shutdown; OnPeerUpdate then
|
||||
// rejects adds so no subscription can outlive the shutdown wait and run
|
||||
// against a store Filer.Shutdown has already closed.
|
||||
stopped bool
|
||||
}
|
||||
|
||||
// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
|
||||
@@ -87,6 +92,13 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
|
||||
ma.peerChansLock.Lock()
|
||||
defer ma.peerChansLock.Unlock()
|
||||
|
||||
// After Shutdown the master connection may still deliver an update; a
|
||||
// subscription started now would have no stop signal and could run past
|
||||
// the store close.
|
||||
if ma.stopped {
|
||||
return
|
||||
}
|
||||
|
||||
address := pb.ServerAddress(update.Address)
|
||||
if update.IsAdd {
|
||||
// the peer is already followed, restarting would only lose the events
|
||||
@@ -102,7 +114,11 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
|
||||
// Account for the peer before its stream signals; keep prior values
|
||||
// on reconnect.
|
||||
ma.initPeerWatermark(address)
|
||||
go ma.loopSubscribeToOneFiler(ma.filer, ma.self, address, startFrom, stopChan)
|
||||
ma.peerWg.Add(1)
|
||||
go func() {
|
||||
defer ma.peerWg.Done()
|
||||
ma.loopSubscribeToOneFiler(ma.filer, ma.self, address, startFrom, stopChan)
|
||||
}()
|
||||
} else {
|
||||
if prevChan, found := ma.peerChans[address]; found {
|
||||
close(prevChan)
|
||||
@@ -273,6 +289,21 @@ func lowWatermarkOf(watermarks map[pb.ServerAddress]int64) int64 {
|
||||
return low
|
||||
}
|
||||
|
||||
// PeersCaughtUpTo reports whether every listed peer's stream has delivered
|
||||
// an event or idle heartbeat at or past tsNs, i.e. no pre-tsNs backlog can
|
||||
// still arrive from it. A peer no longer tracked counts as caught up: it
|
||||
// cannot deliver a backlog anymore.
|
||||
func (ma *MetaAggregator) PeersCaughtUpTo(peers []pb.ServerAddress, tsNs int64) bool {
|
||||
ma.peerWatermarksLock.Lock()
|
||||
defer ma.peerWatermarksLock.Unlock()
|
||||
for _, peer := range peers {
|
||||
if ts, found := ma.peerWatermarks[peer]; found && ts < tsNs {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (ma *MetaAggregator) HasRemotePeers() bool {
|
||||
ma.peerChansLock.Lock()
|
||||
defer ma.peerChansLock.Unlock()
|
||||
@@ -329,6 +360,42 @@ func (ma *MetaAggregator) HasPeer(address pb.ServerAddress) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Shutdown stops every peer subscription and waits for each to return,
|
||||
// letting the deferred offset save persist the last applied position before
|
||||
// the store closes. Without it a graceful restart resumes from the last
|
||||
// periodically-saved offset — up to a minute stale — and replays those
|
||||
// events over newer local state.
|
||||
func (ma *MetaAggregator) Shutdown() {
|
||||
ma.peerChansLock.Lock()
|
||||
ma.stopped = true
|
||||
chans := ma.peerChans
|
||||
ma.peerChans = make(map[pb.ServerAddress]chan struct{})
|
||||
for _, stopChan := range chans {
|
||||
close(stopChan)
|
||||
}
|
||||
ma.peerChansLock.Unlock()
|
||||
|
||||
// Wait without a cap: every blocking point in the subscribe loop is
|
||||
// stop-aware, so a peer still running here is genuinely wedged and
|
||||
// returning anyway would let Filer.Shutdown close the store under it and
|
||||
// skip its deferred offset save.
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
ma.peerWg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
glog.Warningf("meta aggregator shutdown: peer subscriptions still stopping")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time, stopChan chan struct{}) {
|
||||
lastTsNs := startFrom.UnixNano()
|
||||
for {
|
||||
@@ -353,7 +420,12 @@ func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddres
|
||||
if lastTsNs < nextLastTsNs {
|
||||
lastTsNs = nextLastTsNs
|
||||
}
|
||||
time.Sleep(1733 * time.Millisecond)
|
||||
select {
|
||||
case <-stopChan:
|
||||
glog.V(0).Infof("stop subscribing peer %s meta change", peer)
|
||||
return
|
||||
case <-time.After(1733 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -369,11 +441,34 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
|
||||
*/
|
||||
|
||||
var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
|
||||
var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse) error
|
||||
lastPersistTime := time.Now()
|
||||
lastTsNs := startFrom
|
||||
// replayFloorTsNs is the earliest event this session gave up replaying.
|
||||
// The persisted offset and the reconnect resume point must stay behind
|
||||
// it so the event is redelivered on the next subscribe instead of being
|
||||
// skipped forever.
|
||||
var replayFloorTsNs int64
|
||||
persistableTsNs := func(tsNs int64) int64 {
|
||||
if replayFloorTsNs > 0 && replayFloorTsNs-1 < tsNs {
|
||||
return replayFloorTsNs - 1
|
||||
}
|
||||
return tsNs
|
||||
}
|
||||
|
||||
peerSignature, err := ma.readFilerStoreSignature(peer)
|
||||
// Everything below derives from ctx, cancelled when the peer is removed
|
||||
// or the aggregator shuts down, so no stage can block past its stop.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-stopChan:
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
peerSignature, err := ma.readFilerStoreSignature(ctx, peer)
|
||||
if err != nil {
|
||||
return lastTsNs, fmt.Errorf("connecting to peer filer %s: %v", peer, err)
|
||||
}
|
||||
@@ -393,7 +488,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
// different concept and can be set to arbitrary values.
|
||||
preTraverseTime := time.Now()
|
||||
glog.V(0).Infof("no previous offset for peer %s, starting full metadata sync", peer)
|
||||
if traverseErr := ma.traversePeerMetadata(f, peer); traverseErr != nil {
|
||||
if traverseErr := ma.traversePeerMetadata(ctx, f, peer); traverseErr != nil {
|
||||
return lastTsNs, fmt.Errorf("initial metadata sync from %s: %v", peer, traverseErr)
|
||||
}
|
||||
lastTsNs = preTraverseTime.Add(-time.Minute).UnixNano()
|
||||
@@ -405,11 +500,12 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
return lastTsNs, fmt.Errorf("read offset for peer %s: %w", peer, err)
|
||||
}
|
||||
defer func(prevTsNs int64) {
|
||||
if lastTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
|
||||
if err := ma.updateOffset(f, peer, peerSignature, lastTsNs); err == nil {
|
||||
glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
|
||||
saveTsNs := persistableTsNs(lastTsNs)
|
||||
if saveTsNs != prevTsNs && lastTsNs != lastPersistTime.UnixNano() {
|
||||
if err := ma.updateOffset(f, peer, peerSignature, saveTsNs); err == nil {
|
||||
glog.V(0).Infof("last sync time with %s at %v (%d)", peer, time.Unix(0, saveTsNs), saveTsNs)
|
||||
} else {
|
||||
glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
|
||||
glog.Errorf("failed to save last sync time with %s at %v (%d)", peer, time.Unix(0, saveTsNs), saveTsNs)
|
||||
}
|
||||
}
|
||||
}(lastTsNs)
|
||||
@@ -417,11 +513,23 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
|
||||
var counter int64
|
||||
var synced bool
|
||||
maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
|
||||
replicateMetadataChange(f.Store, peer, event)
|
||||
maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) error {
|
||||
if err := replicateMetadataChange(f, peer, event, stopChan); err != nil {
|
||||
if errors.Is(err, errPeerSubscriptionStopped) {
|
||||
return err
|
||||
}
|
||||
// A failed event becomes a barrier the persisted offset and
|
||||
// the reconnect resume point cannot pass, while the stream
|
||||
// keeps flowing: blocking on an event that can never replay
|
||||
// would stall every later event from this peer.
|
||||
if replayFloorTsNs == 0 || event.TsNs < replayFloorTsNs {
|
||||
replayFloorTsNs = event.TsNs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
counter++
|
||||
if lastPersistTime.Add(time.Minute).Before(time.Now()) {
|
||||
if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
|
||||
if err := ma.updateOffset(f, peer, peerSignature, persistableTsNs(event.TsNs)); err == nil {
|
||||
if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
|
||||
glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
|
||||
} else if !synced {
|
||||
@@ -434,6 +542,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -450,7 +559,9 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
return err
|
||||
}
|
||||
if maybeReplicateMetadataChange != nil {
|
||||
maybeReplicateMetadataChange(event)
|
||||
if err := maybeReplicateMetadataChange(event); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -461,17 +572,8 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
|
||||
glog.V(0).Infof("subscribing remote %s meta change: %v, clientId:%d", peer, time.Unix(0, lastTsNs), ma.filer.UniqueFilerId)
|
||||
err = pb.WithFilerClient(true, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
// Stop the stream promptly on removal; the blocking Recv below would
|
||||
// otherwise only notice when the stream errors on its own.
|
||||
go func() {
|
||||
select {
|
||||
case <-stopChan:
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
// ctx is cancelled on stopChan by the watch above; the blocking Recv
|
||||
// below stops promptly on removal or shutdown.
|
||||
atomic.AddInt32(&ma.filer.UniqueFilerEpoch, 1)
|
||||
// Construct a log file reader that reads chunks via the peer filer's LookupVolume.
|
||||
lookupFn := LookupFn(filerClient{client})
|
||||
@@ -496,6 +598,14 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
}
|
||||
|
||||
processOne := func(event *filer_pb.SubscribeMetadataResponse) error {
|
||||
// Bail promptly on stop; a cancelled stream may still deliver
|
||||
// queued events while unwinding, and replaying them now only
|
||||
// delays the offset save the shutdown is waiting on.
|
||||
select {
|
||||
case <-stopChan:
|
||||
return errPeerSubscriptionStopped
|
||||
default:
|
||||
}
|
||||
if err := processEventFn(event); err != nil {
|
||||
glog.V(0).Infof("SubscribeLocalMetadata process %v: %v", event, err)
|
||||
return fmt.Errorf("process %v: %w", event, err)
|
||||
@@ -572,19 +682,37 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
}
|
||||
}
|
||||
})
|
||||
return lastTsNs, err
|
||||
return persistableTsNs(lastTsNs), err
|
||||
}
|
||||
|
||||
// replicateMetadataChange retries transient Replay failures with bounded
|
||||
// backoff. A failure that outlives the retry budget is counted and logged,
|
||||
// then skipped: blocking on an event that can never replay would stall every
|
||||
// later event from this peer, which is worse than one entry staying stale.
|
||||
func replicateMetadataChange(store FilerStore, peer pb.ServerAddress, event *filer_pb.SubscribeMetadataResponse) {
|
||||
err := util.Retry("replicate metadata change from "+string(peer), func() error {
|
||||
return Replay(store, event)
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
// backoff and reports the outcome. errPeerSubscriptionStopped is returned
|
||||
// when the subscription stops mid-retry so the caller aborts and the offset
|
||||
// save keeps the last completed event. A failure that outlives the retry
|
||||
// budget is counted and logged, then returned: the caller pins the peer's
|
||||
// offset at the event so a restart replays it rather than skipping it.
|
||||
func replicateMetadataChange(f *Filer, peer pb.ServerAddress, event *filer_pb.SubscribeMetadataResponse, stopChan <-chan struct{}) error {
|
||||
waitTime := time.Second
|
||||
var err error
|
||||
for waitTime < util.RetryWaitTime {
|
||||
select {
|
||||
case <-stopChan:
|
||||
return errPeerSubscriptionStopped
|
||||
default:
|
||||
}
|
||||
if err = f.Replay(event); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !util.IsTransientError(err) {
|
||||
break
|
||||
}
|
||||
glog.V(0).Infof("retry replicate metadata change from %s: %v", peer, err)
|
||||
select {
|
||||
case <-stopChan:
|
||||
return errPeerSubscriptionStopped
|
||||
case <-time.After(waitTime):
|
||||
}
|
||||
waitTime += waitTime / 2
|
||||
}
|
||||
stats.FilerMetaAggregatorReplayFailures.WithLabelValues(string(peer)).Inc()
|
||||
name := event.GetEventNotification().GetNewEntry().GetName()
|
||||
@@ -592,15 +720,20 @@ func replicateMetadataChange(store FilerStore, peer pb.ServerAddress, event *fil
|
||||
name = event.GetEventNotification().GetOldEntry().GetName()
|
||||
}
|
||||
glog.Errorf("giving up replicating metadata change from %s for %s/%s (ts=%d): %v", peer, event.Directory, name, event.TsNs, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// errPeerSubscriptionStopped ends event processing once the subscription's
|
||||
// stop channel closed.
|
||||
var errPeerSubscriptionStopped = errors.New("peer subscription stopped")
|
||||
|
||||
// traversePeerMetadata does a full BFS traversal of a peer filer's metadata
|
||||
// and inserts all entries into the local store. This is used when a filer
|
||||
// connects to a peer for the first time and needs to bootstrap pre-existing data.
|
||||
func (ma *MetaAggregator) traversePeerMetadata(f *Filer, peer pb.ServerAddress) error {
|
||||
// The ctx cancels on the peer's stop channel so a shutdown does not have to
|
||||
// out-wait a long traversal.
|
||||
func (ma *MetaAggregator) traversePeerMetadata(ctx context.Context, f *Filer, peer pb.ServerAddress) error {
|
||||
return pb.WithFilerClient(true, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
stream, err := client.TraverseBfsMetadata(ctx, &filer_pb.TraverseBfsMetadataRequest{
|
||||
Directory: "/",
|
||||
ExcludedPrefixes: []string{SystemLogDir},
|
||||
@@ -620,22 +753,17 @@ func (ma *MetaAggregator) traversePeerMetadata(f *Filer, peer pb.ServerAddress)
|
||||
if resp.Entry == nil {
|
||||
continue
|
||||
}
|
||||
fullpath := util.Join(resp.Directory, resp.Entry.Name)
|
||||
entry := FromPbEntry(resp.Directory, resp.Entry)
|
||||
if insertErr := f.Store.InsertEntry(context.Background(), entry); insertErr != nil {
|
||||
// Entry may already exist (root dir, or partial previous bootstrap).
|
||||
existing, findErr := f.Store.FindEntry(context.Background(), entry.FullPath)
|
||||
if findErr != nil {
|
||||
return fmt.Errorf("insert entry %s: %w", fullpath, insertErr)
|
||||
}
|
||||
// Only overwrite if the peer's entry is newer.
|
||||
if entry.Attr.Mtime.After(existing.Attr.Mtime) {
|
||||
if updateErr := f.Store.UpdateEntry(context.Background(), entry); updateErr != nil {
|
||||
return fmt.Errorf("update entry %s: %w", fullpath, updateErr)
|
||||
}
|
||||
} else {
|
||||
glog.V(1).Infof("skip older peer entry %s (peer mtime %v <= local mtime %v)", fullpath, entry.Attr.Mtime, existing.Attr.Mtime)
|
||||
}
|
||||
// Order on the sender's per-path commit timestamp, not the
|
||||
// entry's mtime — mtime is client-settable and does not track
|
||||
// when the peer committed the change. Peers that predate the
|
||||
// field send 0 and fall back to mtime.
|
||||
peerTsNs := resp.EntryTsNs
|
||||
if peerTsNs == 0 {
|
||||
peerTsNs = entry.Attr.Mtime.UnixNano()
|
||||
}
|
||||
if applyErr := f.applyTraversedEntry(ctx, entry, peerTsNs); applyErr != nil {
|
||||
return applyErr
|
||||
}
|
||||
count++
|
||||
if count%10000 == 0 {
|
||||
@@ -647,9 +775,38 @@ func (ma *MetaAggregator) traversePeerMetadata(f *Filer, peer pb.ServerAddress)
|
||||
})
|
||||
}
|
||||
|
||||
func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
|
||||
// applyTraversedEntry merges one bootstrap entry under the path's replay
|
||||
// locks, keeping local state when its commit is newer than the sender's —
|
||||
// most importantly a delete, whose tombstone must not let the bootstrap
|
||||
// resurrect the path. An applied entry records the sender's commit time so a
|
||||
// later peer replay orders against it rather than the entry's mtime.
|
||||
func (f *Filer) applyTraversedEntry(ctx context.Context, entry *Entry, peerTsNs int64) error {
|
||||
unlock := f.lockReplayPaths(entry.FullPath, "")
|
||||
defer unlock()
|
||||
localTsNs, err := f.localCommitTsNs(ctx, entry.FullPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check local state for %s: %w", entry.FullPath, err)
|
||||
}
|
||||
if localTsNs >= peerTsNs {
|
||||
return nil
|
||||
}
|
||||
if insertErr := f.Store.InsertEntry(ctx, entry); insertErr != nil {
|
||||
// Entry already exists (root dir, or partial previous bootstrap);
|
||||
// the lock kept the check above current, so the peer commit is newer.
|
||||
if _, findErr := f.Store.FindEntry(ctx, entry.FullPath); findErr != nil {
|
||||
return fmt.Errorf("insert entry %s: %w", entry.FullPath, insertErr)
|
||||
}
|
||||
if updateErr := f.Store.UpdateEntry(ctx, entry); updateErr != nil {
|
||||
return fmt.Errorf("update entry %s: %w", entry.FullPath, updateErr)
|
||||
}
|
||||
}
|
||||
f.notePathChange(entry.FullPath, peerTsNs, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ma *MetaAggregator) readFilerStoreSignature(ctx context.Context, peer pb.ServerAddress) (sig int32, err error) {
|
||||
err = pb.WithFilerClient(false, 0, peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -15,6 +15,14 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// newReplayTestFiler builds the minimum Filer Replay needs over a stub store.
|
||||
func newReplayTestFiler(store FilerStore) *Filer {
|
||||
return &Filer{
|
||||
Store: NewFilerStoreWrapper(store),
|
||||
EntryLockTable: util.NewLockTable[util.FullPath](),
|
||||
}
|
||||
}
|
||||
|
||||
// flakyReplayStore fails InsertEntry for its first failUntil calls, then
|
||||
// delegates to the embedded stubFilerStore.
|
||||
type flakyReplayStore struct {
|
||||
@@ -73,8 +81,11 @@ func TestReplicateMetadataChangeRetriesTransientFailure(t *testing.T) {
|
||||
err: errors.New("i/o timeout talking to store"),
|
||||
}
|
||||
peer := pb.ServerAddress("peer-transient:1")
|
||||
f := newReplayTestFiler(store)
|
||||
|
||||
replicateMetadataChange(store, peer, quotaChangeEvent("my-bucket", wantQuota))
|
||||
if err := replicateMetadataChange(f, peer, quotaChangeEvent("my-bucket", wantQuota), nil); err != nil {
|
||||
t.Fatalf("replicateMetadataChange: %v", err)
|
||||
}
|
||||
|
||||
inserted, err := store.FindEntry(context.Background(), util.NewFullPath("/buckets", "my-bucket"))
|
||||
if err != nil {
|
||||
@@ -97,10 +108,14 @@ func TestReplicateMetadataChangeGivesUpLoudlyOnPermanentFailure(t *testing.T) {
|
||||
err: errors.New("entry checksum mismatch"),
|
||||
}
|
||||
peer := pb.ServerAddress("peer-permanent:1")
|
||||
f := newReplayTestFiler(store)
|
||||
|
||||
before := replayFailureCount(t, peer)
|
||||
|
||||
replicateMetadataChange(store, peer, quotaChangeEvent("poison-bucket", 65536<<20))
|
||||
err := replicateMetadataChange(f, peer, quotaChangeEvent("poison-bucket", 65536<<20), nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected the exhausted replay error so the caller can pin the peer offset at the failed event")
|
||||
}
|
||||
|
||||
if got := store.attemptCount(); got != 1 {
|
||||
t.Fatalf("attempts = %d, want exactly 1: a non-transient error must fail fast, not retry", got)
|
||||
@@ -109,3 +124,587 @@ func TestReplicateMetadataChangeGivesUpLoudlyOnPermanentFailure(t *testing.T) {
|
||||
t.Fatalf("FilerMetaAggregatorReplayFailures[%s] = %v, want %v", peer, got, before+1)
|
||||
}
|
||||
}
|
||||
|
||||
func overwriteEvent(dir, name string, tsNs int64, content string) *filer_pb.SubscribeMetadataResponse {
|
||||
return &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: dir,
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: name},
|
||||
NewEntry: &filer_pb.Entry{Name: name, Content: []byte(content)},
|
||||
},
|
||||
TsNs: tsNs,
|
||||
}
|
||||
}
|
||||
|
||||
func deleteEvent(dir, name string, tsNs int64) *filer_pb.SubscribeMetadataResponse {
|
||||
return &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: dir,
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: name},
|
||||
},
|
||||
TsNs: tsNs,
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplaySkipsEventOlderThanLocalEntry(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
store.entries[string(path)] = &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now},
|
||||
Content: []byte("newer local write"),
|
||||
}
|
||||
|
||||
if err := f.Replay(overwriteEvent("/dir", "file", now.Add(-time.Minute).UnixNano(), "old replayed version")); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
entry, err := store.FindEntry(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("FindEntry: %v", err)
|
||||
}
|
||||
if string(entry.Content) != "newer local write" {
|
||||
t.Fatalf("content = %q, want local write kept", entry.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayAppliesEventNewerThanLocalEntry(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
store.entries[string(path)] = &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(-time.Minute)},
|
||||
Content: []byte("old local write"),
|
||||
}
|
||||
|
||||
if err := f.Replay(overwriteEvent("/dir", "file", now.UnixNano(), "newer peer write")); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
entry, err := store.FindEntry(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("FindEntry: %v", err)
|
||||
}
|
||||
if string(entry.Content) != "newer peer write" {
|
||||
t.Fatalf("content = %q, want peer write applied", entry.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplaySkipsDeleteOlderThanLocalEntry(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
store.entries[string(path)] = &Entry{FullPath: path, Attr: Attr{Mtime: now}}
|
||||
|
||||
if err := f.Replay(deleteEvent("/dir", "file", now.Add(-time.Minute).UnixNano())); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err != nil {
|
||||
t.Fatalf("newer local entry must survive a stale delete: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayAppliesNewerDelete(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
store.entries[string(path)] = &Entry{FullPath: path, Attr: Attr{Mtime: now.Add(-time.Minute)}}
|
||||
|
||||
if err := f.Replay(deleteEvent("/dir", "file", now.UnixNano())); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err == nil {
|
||||
t.Fatal("newer delete must remove the local entry")
|
||||
}
|
||||
}
|
||||
|
||||
// A move event touches two paths; the local check applies to each side
|
||||
// independently, so a newer destination is kept while the source is removed.
|
||||
func TestReplayMoveChecksEachPathSeparately(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
oldPath := util.NewFullPath("/a", "file")
|
||||
newPath := util.NewFullPath("/b", "file")
|
||||
store.entries[string(oldPath)] = &Entry{FullPath: oldPath, Attr: Attr{Mtime: now.Add(-time.Minute)}}
|
||||
store.entries[string(newPath)] = &Entry{FullPath: newPath, Attr: Attr{Mtime: now}, Content: []byte("kept")}
|
||||
|
||||
event := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/a",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "file"},
|
||||
NewEntry: &filer_pb.Entry{Name: "file"},
|
||||
NewParentPath: "/b",
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(event); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), oldPath); err == nil {
|
||||
t.Fatal("old source entry should be deleted")
|
||||
}
|
||||
entry, err := store.FindEntry(context.Background(), newPath)
|
||||
if err != nil {
|
||||
t.Fatalf("FindEntry: %v", err)
|
||||
}
|
||||
if string(entry.Content) != "kept" {
|
||||
t.Fatalf("content = %q, want newer destination kept", entry.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// A local delete leaves a recorded tombstone, so a replayed create older than
|
||||
// the delete must not resurrect the path even though no entry remains to
|
||||
// compare against.
|
||||
func TestReplaySkipsCreateOlderThanLocalDelete(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
// The delete removed the entry; its commit record survives.
|
||||
f.notePathChange(path, now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(createEvent); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err == nil {
|
||||
t.Fatal("stale replayed create resurrected a locally deleted entry")
|
||||
}
|
||||
}
|
||||
|
||||
// A deleted directory covers its children: a bulk drop emits no per-child
|
||||
// events, so the tombstone on the ancestor is what a replayed child create
|
||||
// must check.
|
||||
func TestReplaySkipsCreateUnderDeletedAncestor(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
|
||||
f.notePathChange("/dir", now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "child", Content: []byte("stale")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(createEvent); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), "/dir/child"); err == nil {
|
||||
t.Fatal("stale replayed create resurrected a child of a deleted directory")
|
||||
}
|
||||
}
|
||||
|
||||
// A recorded commit timestamp orders replay, not the entry's mtime: a rename
|
||||
// keeps the source's mtime on the destination, and a mounted client can set
|
||||
// mtime arbitrarily, so an entry whose mtime postdates the event must still
|
||||
// be overwritten when its commit record is older.
|
||||
func TestReplayAppliesWhenCommitRecordIsOlderThanFutureMtime(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/b", "x")
|
||||
store.entries[string(path)] = &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(time.Hour)}, // future mtime, e.g. client-set or kept from a rename source
|
||||
Content: []byte("old destination"),
|
||||
}
|
||||
// The last committed change predates the event.
|
||||
f.notePathChange(path, now.Add(-time.Minute).UnixNano(), false)
|
||||
|
||||
event := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/a",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "x"},
|
||||
NewEntry: &filer_pb.Entry{Name: "x", Content: []byte("renamed in")},
|
||||
NewParentPath: "/b",
|
||||
},
|
||||
TsNs: now.UnixNano(),
|
||||
}
|
||||
if err := f.Replay(event); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
entry, err := store.FindEntry(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("FindEntry: %v", err)
|
||||
}
|
||||
if string(entry.Content) != "renamed in" {
|
||||
t.Fatalf("content = %q, want the rename destination applied", entry.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// A store read failure is not a not-found: the replayed change must not be
|
||||
// applied on the guess that no newer local entry exists.
|
||||
func TestReplayPropagatesLookupError(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
store.findErr = errors.New("store read failed")
|
||||
f := newReplayTestFiler(store)
|
||||
|
||||
err := f.Replay(overwriteEvent("/dir", "file", time.Now().UnixNano(), "v"))
|
||||
if err == nil {
|
||||
t.Fatal("expected Replay to propagate the FindEntry failure")
|
||||
}
|
||||
}
|
||||
|
||||
// The check and the mutation share the per-path entry lock, so a local write
|
||||
// holding that lock cannot be split by a replayed change.
|
||||
func TestReplaySerializesWithEntryLock(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
store.entries[string(path)] = &Entry{FullPath: path, Attr: Attr{Mtime: now}}
|
||||
|
||||
// A local mutation holding the path lock; a replay started behind it
|
||||
// sees the recorded commit and skips.
|
||||
held := f.EntryLockTable.AcquireLock("test local write", path, util.ExclusiveLock)
|
||||
f.notePathChange(path, now.UnixNano(), false)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- f.Replay(overwriteEvent("/dir", "file", now.Add(-time.Minute).UnixNano(), "stale"))
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("replay finished while the path lock was still held")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
f.EntryLockTable.ReleaseLock(path, held)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
entry, err := store.FindEntry(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("FindEntry: %v", err)
|
||||
}
|
||||
if entry.Content != nil {
|
||||
t.Fatalf("content = %q, want local entry kept", entry.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// A closed stop channel must abort the retry loop with
|
||||
// errPeerSubscriptionStopped so the subscription tears down and the offset
|
||||
// save keeps the last completed event instead of skipping this one.
|
||||
func TestReplicateMetadataChangeAbortsOnStop(t *testing.T) {
|
||||
store := &flakyReplayStore{
|
||||
stubFilerStore: newStubFilerStore(),
|
||||
failUntil: 1 << 30,
|
||||
err: errors.New("i/o timeout talking to store"),
|
||||
}
|
||||
peer := pb.ServerAddress("peer-stopped:1")
|
||||
f := newReplayTestFiler(store)
|
||||
|
||||
stopChan := make(chan struct{})
|
||||
close(stopChan)
|
||||
err := replicateMetadataChange(f, peer, quotaChangeEvent("stuck-bucket", 1<<20), stopChan)
|
||||
if !errors.Is(err, errPeerSubscriptionStopped) {
|
||||
t.Fatalf("err = %v, want errPeerSubscriptionStopped", err)
|
||||
}
|
||||
if got := store.attemptCount(); got != 0 {
|
||||
t.Fatalf("attempts = %d, want 0: a stopped subscription must not replay", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Commit-record keys must stay small: SQL KV stores keep the key suffix in a
|
||||
// VARCHAR(766) column, so a raw path over ~578 bytes silently drops the
|
||||
// record (the insert fails and the fallback update touches no row).
|
||||
func TestMetaPathKeyFixedSizeForLongPaths(t *testing.T) {
|
||||
path := util.NewFullPath("/", string(make([]rune, 600)))
|
||||
for _, p := range []util.FullPath{"/a", path} {
|
||||
if got := len(metaPathKey(metaPathTsPrefix, p)); got != len(metaPathTsPrefix)+16 {
|
||||
t.Fatalf("metaPathKey(%d-byte path) = %d bytes, want %d", len(p), got, len(metaPathTsPrefix)+16)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Exact-path locking alone lets a replayed child create slip between a local
|
||||
// ancestor delete's tombstone check and the sweep: replay locks every
|
||||
// ancestor too, so it waits for the delete's lock and then sees the
|
||||
// tombstone.
|
||||
func TestReplaySerializesWithAncestorLock(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
|
||||
now := time.Now()
|
||||
child := util.NewFullPath("/dir", "file")
|
||||
held := f.EntryLockTable.AcquireLock("test local delete", "/dir", util.ExclusiveLock)
|
||||
f.notePathChange("/dir", now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- f.Replay(createEvent)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("replay finished while the ancestor lock was still held")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
f.EntryLockTable.ReleaseLock("/dir", held)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), child); err == nil {
|
||||
t.Fatal("stale replayed create resurrected a child of a locally deleted directory")
|
||||
}
|
||||
}
|
||||
|
||||
// Bootstrap ordering uses the sender's commit timestamp: a local commit
|
||||
// newer than the traversed entry keeps its state even when the peer entry's
|
||||
// mtime is far in the future.
|
||||
func TestBootstrapSkipsPeerEntryBehindLocalDelete(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
f.notePathChange(path, now.UnixNano(), true)
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(time.Hour)}, // client-settable, must not win
|
||||
Content: []byte("stale peer entry"),
|
||||
}
|
||||
if err := f.applyTraversedEntry(context.Background(), entry, now.Add(-time.Minute).UnixNano()); err != nil {
|
||||
t.Fatalf("applyTraversedEntry: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err == nil {
|
||||
t.Fatal("bootstrap resurrected a path newer local delete covered")
|
||||
}
|
||||
}
|
||||
|
||||
// The bug this guards: a peer entry committed recently but stamped with an
|
||||
// old client mtime must still apply over an absent-or-older local commit.
|
||||
func TestBootstrapAppliesPeerCommitNewerThanLocal(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(-time.Hour)}, // old client-set mtime
|
||||
Content: []byte("fresh peer write"),
|
||||
}
|
||||
if err := f.applyTraversedEntry(context.Background(), entry, now.UnixNano()); err != nil {
|
||||
t.Fatalf("applyTraversedEntry: %v", err)
|
||||
}
|
||||
got, err := store.FindEntry(context.Background(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("peer entry with newer commit must bootstrap: %v", err)
|
||||
}
|
||||
if string(got.Content) != "fresh peer write" {
|
||||
t.Fatalf("content = %q, want peer entry applied", got.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// A bootstrap-applied entry records the sender's commit timestamp, so a
|
||||
// replayed event older than it — but newer than the entry's mtime — is
|
||||
// still skipped.
|
||||
func TestBootstrapAppliedCommitBlocksStaleReplay(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(-time.Hour)},
|
||||
Content: []byte("fresh peer write"),
|
||||
}
|
||||
if err := f.applyTraversedEntry(context.Background(), entry, now.UnixNano()); err != nil {
|
||||
t.Fatalf("applyTraversedEntry: %v", err)
|
||||
}
|
||||
if err := f.Replay(overwriteEvent("/dir", "file", now.Add(-time.Minute).UnixNano(), "stale replay")); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
got, _ := store.FindEntry(context.Background(), path)
|
||||
if string(got.Content) != "fresh peer write" {
|
||||
t.Fatalf("content = %q, stale replay must not overwrite the bootstrap entry", got.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// An existing local entry is replaced only when the peer commit is newer —
|
||||
// not when its mtime happens to be.
|
||||
func TestBootstrapUpdatesExistingByCommitTime(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
store.entries[string(path)] = &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(time.Hour)}, // future mtime, e.g. client-set
|
||||
Content: []byte("old local write"),
|
||||
}
|
||||
f.notePathChange(path, now.Add(-time.Minute).UnixNano(), false)
|
||||
|
||||
entry := &Entry{
|
||||
FullPath: path,
|
||||
Attr: Attr{Mtime: now.Add(-time.Hour)},
|
||||
Content: []byte("newer peer commit"),
|
||||
}
|
||||
if err := f.applyTraversedEntry(context.Background(), entry, now.UnixNano()); err != nil {
|
||||
t.Fatalf("applyTraversedEntry: %v", err)
|
||||
}
|
||||
got, _ := store.FindEntry(context.Background(), path)
|
||||
if string(got.Content) != "newer peer commit" {
|
||||
t.Fatalf("content = %q, newer peer commit must update despite the future local mtime", got.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// A delete whose commit record failed to persist still blocks a stale
|
||||
// replayed create: the pending record stands in until the retry lands it.
|
||||
func TestPendingDeleteRecordBlocksStaleReplay(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
now := time.Now()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
// The delete already committed; only its ordering record failed.
|
||||
f.notePathChange(path, now.UnixNano(), true)
|
||||
|
||||
createEvent := &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/dir",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
NewEntry: &filer_pb.Entry{Name: "file", Content: []byte("stale resurrect")},
|
||||
},
|
||||
TsNs: now.Add(-time.Minute).UnixNano(),
|
||||
}
|
||||
if err := f.Replay(createEvent); err != nil {
|
||||
t.Fatalf("Replay: %v", err)
|
||||
}
|
||||
if _, err := store.FindEntry(context.Background(), path); err == nil {
|
||||
t.Fatal("stale create resurrected a path whose delete record is still pending")
|
||||
}
|
||||
}
|
||||
|
||||
// The failed record write retries until it lands, then leaves the pending set.
|
||||
func TestPendingCommitRecordRepairs(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f.notePathChange(path, 100, true)
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
t.Fatal("failed record write must stay pending")
|
||||
}
|
||||
store.setKvPutErr(nil)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathDelPrefix, path))
|
||||
if err != nil || !found || ts != 100 {
|
||||
t.Fatalf("delete record after repair: ts=%v found=%v err=%v", ts, found, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("pending record was not repaired")
|
||||
}
|
||||
|
||||
// CommitTsNs reports the recorded commit time over the entry mtime, and a
|
||||
// still-pending record over both.
|
||||
func TestCommitTsNsPrefersRecordAndPending(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 7 {
|
||||
t.Fatalf("no record: CommitTsNs = %d, %v; want fallback 7", got, err)
|
||||
}
|
||||
f.notePathChange(path, 50, false)
|
||||
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 50 {
|
||||
t.Fatalf("recorded: CommitTsNs = %d, %v; want 50", got, err)
|
||||
}
|
||||
pending := &metaTsPendingRec{}
|
||||
pending.changeTsNs.Store(80)
|
||||
f.metaTsPending.Store(string(path), pending)
|
||||
if got, err := f.CommitTsNs(context.Background(), path, 7); err != nil || got != 80 {
|
||||
t.Fatalf("pending: CommitTsNs = %d, %v; want 80", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A successful write discharges what the pending record owed before it, so a
|
||||
// repair pass can never put an older timestamp back over it.
|
||||
func TestPendingCommitRecordDischargedByNewerWrite(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f.notePathChange(path, 100, false)
|
||||
store.setKvPutErr(nil)
|
||||
f.notePathChange(path, 200, false)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathTsPrefix, path))
|
||||
if err != nil || !found || ts != 200 {
|
||||
t.Fatalf("MpTs after repair = %v, %v, %v; want 200", ts, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed delete record keeps its own time even when a later failed change
|
||||
// merges into the pending record: the repair must not stamp the delete at
|
||||
// the newer change's time.
|
||||
func TestPendingDeleteRecordKeepsItsTime(t *testing.T) {
|
||||
store := newStubFilerStore()
|
||||
store.setKvPutErr(errors.New("kv store unreachable"))
|
||||
f := newReplayTestFiler(store)
|
||||
f.flushCtx, f.flushCancel = context.WithCancel(context.Background())
|
||||
defer f.flushCancel()
|
||||
path := util.NewFullPath("/dir", "file")
|
||||
|
||||
f.notePathChange(path, 100, true)
|
||||
f.notePathChange(path, 200, false)
|
||||
store.setKvPutErr(nil)
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, loaded := f.metaTsPending.Load(string(path)); !loaded {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathTsPrefix, path)); err != nil || !found || ts != 200 {
|
||||
t.Fatalf("MpTs = %v, %v, %v; want 200", ts, found, err)
|
||||
}
|
||||
if ts, found, err := f.kvReadTsNs(context.Background(), metaPathKey(metaPathDelPrefix, path)); err != nil || !found || ts != 100 {
|
||||
t.Fatalf("MpDl = %v, %v, %v; want 100", ts, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,3 +34,55 @@ func TestOnPeerUpdateRepeatedAdd(t *testing.T) {
|
||||
t.Fatal("expecting the subscription to be removed")
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown must close every peer's stopChan and wait for the loops, which is
|
||||
// what lets each loop's deferred offset save run before the store closes.
|
||||
func TestMetaAggregatorShutdownStopsPeerLoops(t *testing.T) {
|
||||
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
|
||||
|
||||
stopChan := make(chan struct{})
|
||||
peer := pb.ServerAddress("127.0.0.1:1")
|
||||
ma.peerChans[peer] = stopChan
|
||||
ma.peerWg.Add(1)
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
defer ma.peerWg.Done()
|
||||
<-stopChan
|
||||
close(exited)
|
||||
}()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
ma.Shutdown()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("Shutdown did not return")
|
||||
}
|
||||
select {
|
||||
case <-exited:
|
||||
default:
|
||||
t.Fatal("peer loop was not stopped")
|
||||
}
|
||||
}
|
||||
|
||||
// After Shutdown begins, a peer-add update arriving on the still-connected
|
||||
// master watch must not start a subscription: it would have no stop signal
|
||||
// and could run past the store close.
|
||||
func TestOnPeerUpdateAfterShutdownRejected(t *testing.T) {
|
||||
ma := NewMetaAggregator(nil, pb.ServerAddress("127.0.0.1:2"), nil)
|
||||
ma.Shutdown()
|
||||
|
||||
peer := pb.ServerAddress("127.0.0.1:1")
|
||||
ma.OnPeerUpdate(&master_pb.ClusterNodeUpdate{Address: string(peer), IsAdd: true}, time.Now())
|
||||
|
||||
ma.peerChansLock.Lock()
|
||||
_, found := ma.peerChans[peer]
|
||||
ma.peerChansLock.Unlock()
|
||||
if found {
|
||||
t.Fatal("peer subscription started after shutdown")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
// metaPathTsPrefix keys per-path commit timestamps in the filer store KV:
|
||||
// the newest event timestamp this filer committed or applied for the
|
||||
// path. Written for every emitted mutation event and kept after the entry
|
||||
// is deleted, so a replayed peer event can still be ordered against a
|
||||
// change that left no entry behind.
|
||||
metaPathTsPrefix = "MpTs"
|
||||
// metaPathDelPrefix keys delete-only timestamps. A replayed event checks
|
||||
// the path's ancestors too, so a directory delete also covers children
|
||||
// swept without per-child events (a dropped bucket's whole-store delete,
|
||||
// or a walk aborted partway).
|
||||
metaPathDelPrefix = "MpDl"
|
||||
)
|
||||
|
||||
// metaPathKey hashes the path into a fixed-size key. SQL-backed KV stores
|
||||
// split the key at byte 8 and store the remainder base64-encoded in a
|
||||
// VARCHAR(766) name column: a raw path over ~578 bytes overflows it, the
|
||||
// KvPut insert fails, and the fallback update touches no row yet returns no
|
||||
// error — the commit record would never land and stale replay could restore
|
||||
// the path.
|
||||
func metaPathKey(prefix string, path util.FullPath) []byte {
|
||||
sum := sha256.Sum256([]byte(path))
|
||||
key := make([]byte, len(prefix)+16)
|
||||
copy(key, prefix)
|
||||
copy(key[len(prefix):], sum[:16])
|
||||
return key
|
||||
}
|
||||
|
||||
// kvReadTsNs returns the timestamp recorded for key, or found=false when the
|
||||
// key is absent or malformed. Any other store read failure is an error: a
|
||||
// caller deciding whether a replayed change is stale must not guess.
|
||||
func (f *Filer) kvReadTsNs(ctx context.Context, key []byte) (tsNs int64, found bool, err error) {
|
||||
value, err := f.Store.KvGet(ctx, key)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrKvNotFound) {
|
||||
return 0, false, nil
|
||||
}
|
||||
return 0, false, err
|
||||
}
|
||||
if len(value) < 8 {
|
||||
return 0, false, nil
|
||||
}
|
||||
return int64(util.BytesToUint64(value)), true, nil
|
||||
}
|
||||
|
||||
// notePathChange persists tsNs as path's newest committed change. Local event
|
||||
// timestamps are stamped monotonically (metaLogInflight.stamp), so a plain
|
||||
// put cannot regress an earlier local record; a higher peer record it may
|
||||
// overwrite was already newer under last-writer-wins ordering anyway. A
|
||||
// delete additionally writes a delete record the ancestor lookup consults.
|
||||
//
|
||||
// The write uses a fresh context: callers notify after CommitTransaction,
|
||||
// and SQL stores keep the already-committed *sql.Tx in the request context —
|
||||
// a record written through it would fail and the path would lose ordering.
|
||||
//
|
||||
// A failed write is kept in metaTsPending and retried in the background: the
|
||||
// mutation the record orders has already committed, so dropping it would let
|
||||
// a stale replayed change overwrite or resurrect the path while the store is
|
||||
// unreachable. Ordering reads consult the pending record until the retry
|
||||
// lands it.
|
||||
func (f *Filer) notePathChange(path util.FullPath, tsNs int64, deleted bool) {
|
||||
if f.Store == nil {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
key := string(path)
|
||||
rec := f.pendingRec(key)
|
||||
defer rec.mu.Unlock()
|
||||
|
||||
value := make([]byte, 8)
|
||||
util.Uint64toBytes(value, uint64(tsNs))
|
||||
if err := f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); err != nil {
|
||||
glog.WarningfCtx(ctx, "record meta change time %s: %v", path, err)
|
||||
if tsNs > rec.changeTsNs.Load() {
|
||||
rec.changeTsNs.Store(tsNs)
|
||||
}
|
||||
} else if rec.changeTsNs.Load() <= tsNs {
|
||||
// This write covers every change record owed so far.
|
||||
rec.changeTsNs.Store(0)
|
||||
}
|
||||
if deleted {
|
||||
if err := f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); err != nil {
|
||||
glog.WarningfCtx(ctx, "record meta delete time %s: %v", path, err)
|
||||
if tsNs > rec.deleteTsNs.Load() {
|
||||
rec.deleteTsNs.Store(tsNs)
|
||||
}
|
||||
} else if rec.deleteTsNs.Load() <= tsNs {
|
||||
rec.deleteTsNs.Store(0)
|
||||
}
|
||||
}
|
||||
if rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0 {
|
||||
f.metaTsPending.CompareAndDelete(key, rec)
|
||||
return
|
||||
}
|
||||
if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
|
||||
go f.repairMetaPathTs(path)
|
||||
}
|
||||
}
|
||||
|
||||
// metaTsPendingRec is a commit record still on its way to the store, holding
|
||||
// only work not yet written: changeTsNs is the newest committed change the
|
||||
// MpTs record lacks and deleteTsNs the newest delete the MpDl record lacks —
|
||||
// the two keep their own times so a later update cannot masquerade as the
|
||||
// delete. Store writes and the bookkeeping they discharge share mu, so a
|
||||
// repair pass can never write an older record over a newer one that already
|
||||
// landed; readers take the atomic fields directly.
|
||||
type metaTsPendingRec struct {
|
||||
mu sync.Mutex
|
||||
changeTsNs atomic.Int64
|
||||
deleteTsNs atomic.Int64
|
||||
}
|
||||
|
||||
// pendingRec returns the live pending record for key, locked for mutation,
|
||||
// recreating it when a repair already dropped the previous one.
|
||||
func (f *Filer) pendingRec(key string) *metaTsPendingRec {
|
||||
for {
|
||||
cur, _ := f.metaTsPending.LoadOrStore(key, &metaTsPendingRec{})
|
||||
rec := cur.(*metaTsPendingRec)
|
||||
rec.mu.Lock()
|
||||
if live, ok := f.metaTsPending.Load(key); ok && live == cur {
|
||||
return rec
|
||||
}
|
||||
rec.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// repairMetaPathTs retries a pending commit record until it lands. Each pass
|
||||
// writes only what the record still owes, so a merge by a later failure is
|
||||
// picked up without another goroutine and a landed write is never rewritten.
|
||||
// The retry is unbounded — the mutation is already committed and only the
|
||||
// record preserves its ordering.
|
||||
func (f *Filer) repairMetaPathTs(path util.FullPath) {
|
||||
key := string(path)
|
||||
ctx := f.flushCtx
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
defer func() {
|
||||
f.metaTsRepairing.Delete(key)
|
||||
// A failure landing between the last pass and the flag release
|
||||
// still needs a repair pass — unless the retry context is done.
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if _, ok := f.metaTsPending.Load(key); ok {
|
||||
if _, running := f.metaTsRepairing.LoadOrStore(key, struct{}{}); !running {
|
||||
go f.repairMetaPathTs(path)
|
||||
}
|
||||
}
|
||||
}()
|
||||
for {
|
||||
if _, ok := f.metaTsPending.Load(key); !ok {
|
||||
return
|
||||
}
|
||||
rec := f.pendingRec(key)
|
||||
var tsErr, delErr error
|
||||
if tsNs := rec.changeTsNs.Load(); tsNs > 0 {
|
||||
value := make([]byte, 8)
|
||||
util.Uint64toBytes(value, uint64(tsNs))
|
||||
if tsErr = f.Store.KvPut(ctx, metaPathKey(metaPathTsPrefix, path), value); tsErr == nil {
|
||||
rec.changeTsNs.Store(0)
|
||||
}
|
||||
}
|
||||
if tsNs := rec.deleteTsNs.Load(); tsNs > 0 {
|
||||
value := make([]byte, 8)
|
||||
util.Uint64toBytes(value, uint64(tsNs))
|
||||
if delErr = f.Store.KvPut(ctx, metaPathKey(metaPathDelPrefix, path), value); delErr == nil {
|
||||
rec.deleteTsNs.Store(0)
|
||||
}
|
||||
}
|
||||
done := rec.changeTsNs.Load() == 0 && rec.deleteTsNs.Load() == 0
|
||||
if done {
|
||||
f.metaTsPending.CompareAndDelete(key, rec)
|
||||
}
|
||||
rec.mu.Unlock()
|
||||
if !done {
|
||||
glog.V(0).InfofCtx(ctx, "retry record meta change time %s: %v %v", path, tsErr, delErr)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pendingPathTsNs returns the newest pending commit timestamp for path and
|
||||
// the newest pending delete timestamp.
|
||||
func (f *Filer) pendingPathTsNs(path util.FullPath) (changeTsNs, deleteTsNs int64) {
|
||||
cur, ok := f.metaTsPending.Load(string(path))
|
||||
if !ok {
|
||||
return 0, 0
|
||||
}
|
||||
rec := cur.(*metaTsPendingRec)
|
||||
return rec.changeTsNs.Load(), rec.deleteTsNs.Load()
|
||||
}
|
||||
|
||||
// CommitTsNs returns the newest commit timestamp known for path — the
|
||||
// recorded event time, or a still-pending one — and fallbackTsNs when none
|
||||
// exists. Peers stamp traversal entries with it so a bootstrap subscriber
|
||||
// orders each entry on the sender's commit clock rather than the entry's
|
||||
// client-settable mtime. A store read failure is an error, not a fallback:
|
||||
// the mtime it would mint could outdate real commits.
|
||||
func (f *Filer) CommitTsNs(ctx context.Context, path util.FullPath, fallbackTsNs int64) (int64, error) {
|
||||
if f.Store == nil {
|
||||
return fallbackTsNs, nil
|
||||
}
|
||||
tsNs := fallbackTsNs
|
||||
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if found {
|
||||
tsNs = recTs
|
||||
}
|
||||
if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
|
||||
tsNs = pendTs
|
||||
}
|
||||
return tsNs, nil
|
||||
}
|
||||
|
||||
// noteCommittedEvent records the event's commit timestamp for every path it
|
||||
// touches: the source of a delete or rename, and the target of a create,
|
||||
// update, or rename. It runs after the local mutation commits while the
|
||||
// caller still holds the path's entry lock (for filer-server mutations), so
|
||||
// a peer replay ordered by the same lock sees either the whole local change
|
||||
// or none of it.
|
||||
func (f *Filer) noteCommittedEvent(event *filer_pb.SubscribeMetadataResponse) {
|
||||
message := event.EventNotification
|
||||
if message == nil {
|
||||
return
|
||||
}
|
||||
var source, target util.FullPath
|
||||
if message.OldEntry != nil {
|
||||
source = util.FullPath(filer_pb.MetadataEventSourceFullPath(event))
|
||||
}
|
||||
if message.NewEntry != nil {
|
||||
target = util.FullPath(filer_pb.MetadataEventTargetFullPath(event))
|
||||
}
|
||||
// A delete or a rename removes the source path; an update reports the
|
||||
// same path on both sides and counts once.
|
||||
if source != "" && source != target {
|
||||
f.notePathChange(source, event.TsNs, true)
|
||||
}
|
||||
if target != "" {
|
||||
f.notePathChange(target, event.TsNs, false)
|
||||
}
|
||||
}
|
||||
|
||||
// deletedAfterTsNs returns the newest recorded delete timestamp among path
|
||||
// and its ancestors.
|
||||
func (f *Filer) deletedAfterTsNs(ctx context.Context, path util.FullPath) (int64, error) {
|
||||
var tsNs int64
|
||||
for p := string(path); p != "" && p != "/"; {
|
||||
ts, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathDelPrefix, util.FullPath(p)))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if found && ts > tsNs {
|
||||
tsNs = ts
|
||||
}
|
||||
if _, pendDel := f.pendingPathTsNs(util.FullPath(p)); pendDel > tsNs {
|
||||
tsNs = pendDel
|
||||
}
|
||||
i := strings.LastIndexByte(p, '/')
|
||||
if i <= 0 {
|
||||
break
|
||||
}
|
||||
p = p[:i]
|
||||
}
|
||||
return tsNs, nil
|
||||
}
|
||||
|
||||
// localCommitTsNs returns the newest timestamp this filer is known to have
|
||||
// committed for path: the recorded event time when one exists — it is not
|
||||
// client-controllable the way an entry's mtime is — else the entry's mtime
|
||||
// for entries written before recording existed, plus any recorded delete of
|
||||
// the path or an ancestor.
|
||||
func (f *Filer) localCommitTsNs(ctx context.Context, path util.FullPath) (int64, error) {
|
||||
if f.Store == nil {
|
||||
return 0, nil
|
||||
}
|
||||
var tsNs int64
|
||||
recTs, found, err := f.kvReadTsNs(ctx, metaPathKey(metaPathTsPrefix, path))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if found {
|
||||
tsNs = recTs
|
||||
} else if entry, findErr := f.Store.FindEntry(ctx, path); findErr != nil {
|
||||
if !errors.Is(findErr, filer_pb.ErrNotFound) {
|
||||
return 0, findErr
|
||||
}
|
||||
} else if entry != nil {
|
||||
tsNs = entry.Mtime.UnixNano()
|
||||
}
|
||||
if pendTs, _ := f.pendingPathTsNs(path); pendTs > tsNs {
|
||||
tsNs = pendTs
|
||||
}
|
||||
delTs, err := f.deletedAfterTsNs(ctx, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if delTs > tsNs {
|
||||
tsNs = delTs
|
||||
}
|
||||
return tsNs, nil
|
||||
}
|
||||
+151
-14
@@ -2,6 +2,8 @@ package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -9,31 +11,166 @@ import (
|
||||
)
|
||||
|
||||
// Replay applies the delete and the insert as separate, non-transactional
|
||||
// store calls, so retrying it after a partial failure is not atomic.
|
||||
func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
// store calls, so retrying it after a partial failure is not atomic; both
|
||||
// operations are idempotent and the timestamp check tolerates re-delivery.
|
||||
//
|
||||
// A replayed event can be older than local state: a peer resuming from a
|
||||
// stale offset replays everything since the offset, and two writes through
|
||||
// different filers race within the replication lag. Apply each path's change
|
||||
// only when no newer local change is recorded, so a late event cannot revert
|
||||
// or resurrect entries.
|
||||
//
|
||||
// Ordering uses the per-path commit timestamp recorded for every emitted
|
||||
// mutation (meta_path_ts.go), which survives deletes as a tombstone and is
|
||||
// not client-controllable the way entry mtime is — a mounted client can set
|
||||
// mtime into the future, and a rename keeps the source file's mtime on the
|
||||
// destination entry. Entries written before recording existed fall back to
|
||||
// their mtime.
|
||||
//
|
||||
// The check and the mutation run under the same per-path entry lock the
|
||||
// filer server's local writes hold, so they cannot be split by a concurrent
|
||||
// local update or by another peer's replay.
|
||||
func (f *Filer) Replay(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
message := resp.EventNotification
|
||||
var oldPath util.FullPath
|
||||
var newEntry *Entry
|
||||
dir := resp.Directory
|
||||
if message.NewParentPath != "" {
|
||||
dir = message.NewParentPath
|
||||
}
|
||||
var oldPath, newPath util.FullPath
|
||||
if message.OldEntry != nil {
|
||||
oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name)
|
||||
glog.V(4).Infof("deleting %v", oldPath)
|
||||
if err := filerStore.DeleteEntry(context.Background(), oldPath); err != nil {
|
||||
}
|
||||
if message.NewEntry != nil {
|
||||
newPath = util.NewFullPath(dir, message.NewEntry.Name)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
unlock := f.lockReplayPaths(oldPath, newPath)
|
||||
defer unlock()
|
||||
|
||||
samePath := oldPath != "" && oldPath == newPath
|
||||
if samePath {
|
||||
newer, err := f.replayBlockedByLocal(ctx, oldPath, resp.TsNs)
|
||||
if err != nil || newer {
|
||||
return err
|
||||
}
|
||||
glog.V(4).Infof("deleting %v", oldPath)
|
||||
if err := f.Store.DeleteEntry(ctx, oldPath); err != nil {
|
||||
return err
|
||||
}
|
||||
// Record the commit before the insert too: if the insert fails and
|
||||
// the event is retried later, the path sits deleted in between —
|
||||
// without the record a stale peer create could resurrect it in the
|
||||
// gap. The entry still exists afterward, so no delete record.
|
||||
f.notePathChange(oldPath, resp.TsNs, false)
|
||||
glog.V(4).Infof("creating %v", newPath)
|
||||
newEntry := FromPbEntry(dir, message.NewEntry)
|
||||
if err := f.Store.InsertEntry(ctx, newEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
f.notePathChange(oldPath, resp.TsNs, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
if message.OldEntry != nil {
|
||||
newer, err := f.replayBlockedByLocal(ctx, oldPath, resp.TsNs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !newer {
|
||||
glog.V(4).Infof("deleting %v", oldPath)
|
||||
if err := f.Store.DeleteEntry(ctx, oldPath); err != nil {
|
||||
return err
|
||||
}
|
||||
f.notePathChange(oldPath, resp.TsNs, true)
|
||||
}
|
||||
}
|
||||
|
||||
if message.NewEntry != nil {
|
||||
dir := resp.Directory
|
||||
if message.NewParentPath != "" {
|
||||
dir = message.NewParentPath
|
||||
}
|
||||
key := util.NewFullPath(dir, message.NewEntry.Name)
|
||||
glog.V(4).Infof("creating %v", key)
|
||||
newEntry = FromPbEntry(dir, message.NewEntry)
|
||||
if err := filerStore.InsertEntry(context.Background(), newEntry); err != nil {
|
||||
newer, err := f.replayBlockedByLocal(ctx, newPath, resp.TsNs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !newer {
|
||||
glog.V(4).Infof("creating %v", newPath)
|
||||
newEntry := FromPbEntry(dir, message.NewEntry)
|
||||
if err := f.Store.InsertEntry(ctx, newEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
f.notePathChange(newPath, resp.TsNs, false)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// replayBlockedByLocal reports whether the local store already holds a
|
||||
// version of path committed after the event's timestamp. Lookup failures are
|
||||
// propagated: a replayed change must not be applied on the guess that the
|
||||
// path was simply absent, or a transient read error could let a stale event
|
||||
// overwrite a newer entry past the point of no return.
|
||||
func (f *Filer) replayBlockedByLocal(ctx context.Context, path util.FullPath, tsNs int64) (bool, error) {
|
||||
if tsNs <= 0 {
|
||||
return false, nil
|
||||
}
|
||||
localTsNs, err := f.localCommitTsNs(ctx, path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return localTsNs > tsNs, nil
|
||||
}
|
||||
|
||||
// lockReplayPaths holds the shared entry locks for every path the event
|
||||
// touches, in the same sorted order the rename handler takes them.
|
||||
// lockReplayPaths takes the per-path entry locks for the event's paths and
|
||||
// every ancestor of them, in lexicographic order — parents sort before
|
||||
// children, the same ordering the local rename handler's two-path lock uses,
|
||||
// so lock acquisition is deadlock-free across operations. Ancestors are
|
||||
// included because the replay check consults ancestor delete records: a
|
||||
// local recursive delete or directory rename holds only the ancestor's lock,
|
||||
// and without contending on it a replay could reinsert a stale child after
|
||||
// its check. Ancestors are locked shared — they only need to exclude an
|
||||
// exclusive holder, not each other, or every replay under a busy parent
|
||||
// would serialize — while a path the event actually changes stays exclusive
|
||||
// even when it is also the other path's ancestor. The root is never locked:
|
||||
// it cannot be deleted, and a shared root lock would still be excluded by
|
||||
// every top-level delete.
|
||||
func (f *Filer) lockReplayPaths(oldPath, newPath util.FullPath) func() {
|
||||
lt := f.EntryLockTable
|
||||
if lt == nil {
|
||||
return func() {}
|
||||
}
|
||||
var paths []util.FullPath
|
||||
addAncestors := func(p util.FullPath) {
|
||||
for i := strings.LastIndexByte(string(p), '/'); i > 0; i = strings.LastIndexByte(string(p), '/') {
|
||||
p = p[:i]
|
||||
paths = append(paths, p)
|
||||
}
|
||||
}
|
||||
addAncestors(oldPath)
|
||||
addAncestors(newPath)
|
||||
if oldPath != "" {
|
||||
paths = append(paths, oldPath)
|
||||
}
|
||||
if newPath != "" {
|
||||
paths = append(paths, newPath)
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
return func() {}
|
||||
}
|
||||
slices.Sort(paths)
|
||||
paths = slices.Compact(paths)
|
||||
locks := make([]*util.ActiveLock, len(paths))
|
||||
for i, p := range paths {
|
||||
lockType := util.SharedLock
|
||||
if p == oldPath || p == newPath {
|
||||
lockType = util.ExclusiveLock
|
||||
}
|
||||
locks[i] = lt.AcquireLock("MetaReplay", p, lockType)
|
||||
}
|
||||
return func() {
|
||||
for i := len(paths) - 1; i >= 0; i-- {
|
||||
lt.ReleaseLock(paths[i], locks[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
// A sequential reader that consumes a chunk in several small ReadAt calls, as
|
||||
// the S3 GET path does with its 256 KiB copy buffer over 4 MiB chunks, must
|
||||
// keep its read-ahead at prefetchCount chunks. Each ReadAt triggers a
|
||||
// prefetch; if chunks already being fetched are not counted toward the
|
||||
// window, every call starts prefetchCount more downloads and the read-ahead
|
||||
// runs to the ReaderCache downloader limit, buffering most of the object.
|
||||
func TestChunkReadAtSequentialPrefetchStaysWithinWindow(t *testing.T) {
|
||||
const chunkSize = 64 << 10
|
||||
const chunkCount = 64
|
||||
const sliceSize = 16 << 10
|
||||
|
||||
var readerChunk atomic.Int64
|
||||
var maxLead atomic.Int64
|
||||
rc := NewReaderCache(256, (*chunk_cache.TieredChunkCache)(nil), func(context.Context, string) ([]string, error) {
|
||||
return []string{"unused"}, nil
|
||||
}, nil)
|
||||
defer rc.destroy()
|
||||
rc.fetchChunkDataFn = func(_ context.Context, buffer []byte, _ []string, _ []byte, _ bool, _ bool, _ int64, fileId string, _ util_http.RefreshUrlsFunc) (int, error) {
|
||||
idx, err := strconv.Atoi(strings.TrimPrefix(fileId, "chunk"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
lead := int64(idx) - readerChunk.Load()
|
||||
for {
|
||||
cur := maxLead.Load()
|
||||
if lead <= cur || maxLead.CompareAndSwap(cur, lead) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return len(buffer), nil
|
||||
}
|
||||
|
||||
views := NewIntervalList[*ChunkView]()
|
||||
for i := 0; i < chunkCount; i++ {
|
||||
views.AppendInterval(&Interval[*ChunkView]{
|
||||
StartOffset: int64(i * chunkSize),
|
||||
StopOffset: int64((i + 1) * chunkSize),
|
||||
Value: &ChunkView{
|
||||
FileId: fmt.Sprintf("chunk%d", i),
|
||||
ViewSize: chunkSize,
|
||||
ViewOffset: int64(i * chunkSize),
|
||||
ChunkSize: chunkSize,
|
||||
},
|
||||
})
|
||||
}
|
||||
reader := NewChunkReaderAtFromClient(context.Background(), rc, views, chunkSize*chunkCount, DefaultPrefetchCount)
|
||||
defer reader.ReleaseStream()
|
||||
|
||||
buf := make([]byte, sliceSize)
|
||||
for off := int64(0); off < chunkSize*chunkCount; off += sliceSize {
|
||||
readerChunk.Store(off / chunkSize)
|
||||
if _, err := reader.ReadAt(buf, off); err != nil && off+sliceSize < chunkSize*chunkCount {
|
||||
t.Fatalf("ReadAt(%d): %v", off, err)
|
||||
}
|
||||
}
|
||||
|
||||
if got := maxLead.Load(); got > DefaultPrefetchCount {
|
||||
t.Fatalf("read-ahead reached %d chunks past the reader, want at most %d", got, DefaultPrefetchCount)
|
||||
}
|
||||
}
|
||||
@@ -92,8 +92,10 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
|
||||
return
|
||||
}
|
||||
|
||||
cached := 0
|
||||
for x := chunkViews; x != nil && cached < count; x = x.Next {
|
||||
// The window is the next count chunks, whether or not a download for them
|
||||
// is already running: counting only newly started downloads would push the
|
||||
// window count chunks further on every call.
|
||||
for x, i := chunkViews, 0; x != nil && i < count; x, i = x.Next, i+1 {
|
||||
chunkView := x.Value
|
||||
if _, found := rc.downloaders[chunkView.FileId]; found {
|
||||
continue
|
||||
@@ -124,7 +126,6 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
|
||||
rc.downloaders[chunkView.FileId] = cacher
|
||||
go cacher.startCaching()
|
||||
started = append(started, cacher)
|
||||
cached++
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
+14
-1
@@ -386,9 +386,18 @@ var _ = io.ReadSeeker(&ChunkStreamReader{})
|
||||
var _ = io.ReaderAt(&ChunkStreamReader{})
|
||||
var _ = io.Closer(&ChunkStreamReader{})
|
||||
|
||||
// doNewChunkStreamReader builds a reader over chunks. If their manifests cannot
|
||||
// be resolved, the reader fails every read with that error and reports it as its
|
||||
// SourceError.
|
||||
func doNewChunkStreamReader(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
|
||||
|
||||
chunkViews := ViewFromChunks(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
|
||||
chunkViews, err := ViewFromChunks(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
|
||||
if err != nil {
|
||||
return &ChunkStreamReader{
|
||||
lookupFileId: lookupFileIdFn,
|
||||
sourceErr: err,
|
||||
}
|
||||
}
|
||||
|
||||
var totalSize int64
|
||||
for x := chunkViews.Front(); x != nil; x = x.Next {
|
||||
@@ -487,6 +496,10 @@ func insideChunk(offset int64, chunk *ChunkView) bool {
|
||||
}
|
||||
|
||||
func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
|
||||
// a reader without chunk views has a source error only if ViewFromChunks failed
|
||||
if c.head == nil && c.sourceErr != nil {
|
||||
return c.sourceErr
|
||||
}
|
||||
// stay in the same chunk
|
||||
if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
|
||||
return nil
|
||||
|
||||
+122
-22
@@ -5,9 +5,12 @@ package azure
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -26,6 +29,93 @@ func init() {
|
||||
seaweedkms.RegisterProvider("azure", NewAzureKMSProvider)
|
||||
}
|
||||
|
||||
// vaultHost returns the host of a configured Key Vault URL, or "" when the URL
|
||||
// carries none. Hosts are compared case-insensitively, without a trailing dot,
|
||||
// and without an explicit :443, which names the same vault as no port.
|
||||
func vaultHost(vaultURL string) string {
|
||||
parsed, err := url.Parse(vaultURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
host := strings.TrimSuffix(strings.ToLower(parsed.Host), ":443")
|
||||
return strings.TrimSuffix(host, ".")
|
||||
}
|
||||
|
||||
// splitKeyID turns a Key Vault key identifier into the (name, version) pair the
|
||||
// azkeys client expects. A plain key name, or anything that is not a URL, is
|
||||
// returned unchanged together with an empty version, which the client resolves
|
||||
// to the latest version.
|
||||
//
|
||||
// A URL naming a different vault than the one this provider is configured for
|
||||
// is rejected: the client addresses only its own vault, so dropping the host
|
||||
// would silently encrypt under this vault's same-named key. A URL for this
|
||||
// vault that does not name a key under /keys/ is rejected too: passing the URL
|
||||
// on as a key name would only surface as a confusing vault-side 404.
|
||||
func (p *AzureKMSProvider) splitKeyID(keyID string) (string, string, error) {
|
||||
parsed, err := url.Parse(keyID)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return keyID, "", nil
|
||||
}
|
||||
if host := vaultHost(p.vaultURL); host != "" && vaultHost(keyID) != host {
|
||||
return "", "", fmt.Errorf("key ID %q names vault %q, but this provider is configured for %q", keyID, parsed.Host, host)
|
||||
}
|
||||
parts := strings.Split(strings.Trim(parsed.Path, "/"), "/")
|
||||
if len(parts) < 2 || parts[0] != "keys" || parts[1] == "" {
|
||||
return "", "", fmt.Errorf("key ID %q is a Key Vault URL but does not name a key", keyID)
|
||||
}
|
||||
if len(parts) == 2 {
|
||||
return parts[1], "", nil
|
||||
}
|
||||
if len(parts) == 3 {
|
||||
return parts[1], parts[2], nil
|
||||
}
|
||||
return "", "", fmt.Errorf("key ID %q has too many path segments for a key URL", keyID)
|
||||
}
|
||||
|
||||
// encodeCiphertext stores the wrapped data key in the JSON envelope as base64.
|
||||
// The envelope is JSON, so raw binary would be replaced by U+FFFD on the way in.
|
||||
func encodeCiphertext(ciphertext []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(ciphertext)
|
||||
}
|
||||
|
||||
// contextDigest digests the encryption context for storage in the envelope.
|
||||
// RSA-OAEP takes no AAD, so the context is bound to the wrapped key by
|
||||
// recording its hash and checking it on decrypt instead. Unlike KMS-checked
|
||||
// AAD this is a client-side mismatch check only — it is not authenticated by
|
||||
// the vault, and an actor who can rewrite an object's envelope could swap in
|
||||
// a ciphertext and digest from another object.
|
||||
func contextDigest(context map[string]string) string {
|
||||
encoded, _ := json.Marshal(context) // map keys marshal in sorted order
|
||||
sum := sha256.Sum256(encoded)
|
||||
return base64.StdEncoding.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// checkContext verifies the request's encryption context matches the context
|
||||
// recorded when the key was wrapped.
|
||||
func checkContext(envelope *seaweedkms.CiphertextEnvelope, context map[string]string) error {
|
||||
recorded, _ := envelope.ProviderSpecific["encryption_context_sha256"].(string)
|
||||
if recorded == "" {
|
||||
if len(context) == 0 {
|
||||
return nil
|
||||
}
|
||||
} else if recorded == contextDigest(context) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("encryption context does not match the wrapped key")
|
||||
}
|
||||
|
||||
// decodeCiphertext reads back what encodeCiphertext wrote.
|
||||
func decodeCiphertext(ciphertext string) ([]byte, error) {
|
||||
decoded, err := base64.StdEncoding.Strict().DecodeString(ciphertext)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("wrapped data key is not valid base64: %w", err)
|
||||
}
|
||||
if len(decoded) == 0 {
|
||||
return nil, fmt.Errorf("wrapped data key is empty")
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
|
||||
// AzureKMSProvider implements the KMSProvider interface using Azure Key Vault
|
||||
type AzureKMSProvider struct {
|
||||
client *azkeys.Client
|
||||
@@ -155,19 +245,23 @@ func (p *AzureKMSProvider) GenerateDataKey(ctx context.Context, req *seaweedkms.
|
||||
Value: dataKey,
|
||||
}
|
||||
|
||||
// Add encryption context as Additional Authenticated Data (AAD) if provided
|
||||
// RSA-OAEP takes no additional authenticated data: Key Vault rejects the
|
||||
// request with BadParameter when AAD is set on an RSA-OAEP key. The
|
||||
// context is bound to the wrapped key via a digest in the envelope
|
||||
// instead, which Decrypt verifies.
|
||||
var providerSpecific map[string]interface{}
|
||||
if len(req.EncryptionContext) > 0 {
|
||||
// Marshal encryption context to JSON for deterministic AAD
|
||||
aadBytes, err := json.Marshal(req.EncryptionContext)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal encryption context: %w", err)
|
||||
providerSpecific = map[string]interface{}{
|
||||
"encryption_context_sha256": contextDigest(req.EncryptionContext),
|
||||
}
|
||||
encryptParams.AAD = aadBytes
|
||||
glog.V(4).Infof("Azure KMS: Using encryption context as AAD for key %s", req.KeyID)
|
||||
}
|
||||
|
||||
// Call Azure Key Vault to encrypt the data key
|
||||
encryptResult, err := p.client.Encrypt(ctx, req.KeyID, "", encryptParams, nil)
|
||||
keyName, keyVersion, err := p.splitKeyID(req.KeyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encryptResult, err := p.client.Encrypt(ctx, keyName, keyVersion, encryptParams, nil)
|
||||
if err != nil {
|
||||
return nil, p.convertAzureError(err, req.KeyID)
|
||||
}
|
||||
@@ -179,7 +273,7 @@ func (p *AzureKMSProvider) GenerateDataKey(ctx context.Context, req *seaweedkms.
|
||||
}
|
||||
|
||||
// Create standardized envelope format for consistent API behavior
|
||||
envelopeBlob, err := seaweedkms.CreateEnvelope("azure", actualKeyID, string(encryptResult.Result), nil)
|
||||
envelopeBlob, err := seaweedkms.CreateEnvelope("azure", actualKeyID, encodeCiphertext(encryptResult.Result), providerSpecific)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create ciphertext envelope: %w", err)
|
||||
}
|
||||
@@ -215,8 +309,11 @@ func (p *AzureKMSProvider) Decrypt(ctx context.Context, req *seaweedkms.DecryptR
|
||||
return nil, fmt.Errorf("envelope missing key ID")
|
||||
}
|
||||
|
||||
// Convert string back to bytes
|
||||
ciphertext := []byte(envelope.Ciphertext)
|
||||
// Convert the base64 envelope field back to the raw wrapped key
|
||||
ciphertext, err := decodeCiphertext(envelope.Ciphertext)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid Azure ciphertext envelope: %w", err)
|
||||
}
|
||||
|
||||
// Prepare decryption parameters
|
||||
decryptAlgorithm := azkeys.JSONWebKeyEncryptionAlgorithmRSAOAEP256
|
||||
@@ -225,20 +322,19 @@ func (p *AzureKMSProvider) Decrypt(ctx context.Context, req *seaweedkms.DecryptR
|
||||
Value: ciphertext,
|
||||
}
|
||||
|
||||
// Add encryption context as Additional Authenticated Data (AAD) if provided
|
||||
if len(req.EncryptionContext) > 0 {
|
||||
// Marshal encryption context to JSON for deterministic AAD (must match encryption)
|
||||
aadBytes, err := json.Marshal(req.EncryptionContext)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal encryption context: %w", err)
|
||||
}
|
||||
decryptParams.AAD = aadBytes
|
||||
glog.V(4).Infof("Azure KMS: Using encryption context as AAD for decryption of key %s", keyID)
|
||||
// RSA-OAEP takes no AAD, so the encryption context is bound via a digest
|
||||
// recorded in the envelope rather than authenticated by the vault.
|
||||
if err := checkContext(envelope, req.EncryptionContext); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Call Azure Key Vault to decrypt the data key
|
||||
glog.V(4).Infof("Azure KMS: Decrypting data key using key %s", keyID)
|
||||
decryptResult, err := p.client.Decrypt(ctx, keyID, "", decryptParams, nil)
|
||||
decryptName, decryptVersion, err := p.splitKeyID(keyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
decryptResult, err := p.client.Decrypt(ctx, decryptName, decryptVersion, decryptParams, nil)
|
||||
if err != nil {
|
||||
return nil, p.convertAzureError(err, keyID)
|
||||
}
|
||||
@@ -270,7 +366,11 @@ func (p *AzureKMSProvider) DescribeKey(ctx context.Context, req *seaweedkms.Desc
|
||||
|
||||
// Get key from Azure Key Vault
|
||||
glog.V(4).Infof("Azure KMS: Describing key %s", req.KeyID)
|
||||
result, err := p.client.GetKey(ctx, req.KeyID, "", nil)
|
||||
describeName, describeVersion, err := p.splitKeyID(req.KeyID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := p.client.GetKey(ctx, describeName, describeVersion, nil)
|
||||
if err != nil {
|
||||
return nil, p.convertAzureError(err, req.KeyID)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
//go:build azurekms
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys"
|
||||
|
||||
seaweedkms "github.com/seaweedfs/seaweedfs/weed/kms"
|
||||
)
|
||||
|
||||
func TestSplitKeyID(t *testing.T) {
|
||||
provider := &AzureKMSProvider{vaultURL: "https://myvault.vault.azure.net"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
keyID string
|
||||
wantName string
|
||||
wantVersion string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "plain key name",
|
||||
keyID: "my-key",
|
||||
wantName: "my-key",
|
||||
},
|
||||
{
|
||||
name: "key url without version",
|
||||
keyID: "https://myvault.vault.azure.net/keys/my-key",
|
||||
wantName: "my-key",
|
||||
},
|
||||
{
|
||||
name: "key url with version",
|
||||
keyID: "https://myvault.vault.azure.net/keys/my-key/abc123",
|
||||
wantName: "my-key",
|
||||
wantVersion: "abc123",
|
||||
},
|
||||
{
|
||||
name: "key url with trailing slash",
|
||||
keyID: "https://myvault.vault.azure.net/keys/my-key/",
|
||||
wantName: "my-key",
|
||||
},
|
||||
{
|
||||
name: "key url with a mixed case host",
|
||||
keyID: "https://MyVault.vault.azure.net/keys/my-key",
|
||||
wantName: "my-key",
|
||||
},
|
||||
{
|
||||
name: "key url with a non keys path",
|
||||
keyID: "https://myvault.vault.azure.net/secrets/my-secret",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "key url with extra path segments",
|
||||
keyID: "https://myvault.vault.azure.net/keys/my-key/abc123/extra",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "key url with an empty key name",
|
||||
keyID: "https://myvault.vault.azure.net/keys//abc123",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "name that only looks like a url",
|
||||
keyID: "my-key:abc123",
|
||||
wantName: "my-key:abc123",
|
||||
},
|
||||
{
|
||||
name: "key url with explicit default port",
|
||||
keyID: "https://myvault.vault.azure.net:443/keys/my-key",
|
||||
wantName: "my-key",
|
||||
},
|
||||
{
|
||||
name: "key url with non-default port",
|
||||
keyID: "https://myvault.vault.azure.net:8443/keys/my-key",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "key url from another vault",
|
||||
keyID: "https://othervault.vault.azure.net/keys/my-key",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
name, version, err := provider.splitKeyID(test.keyID)
|
||||
if test.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("splitKeyID(%q) succeeded, want error", test.keyID)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("splitKeyID(%q): %v", test.keyID, err)
|
||||
}
|
||||
if name != test.wantName {
|
||||
t.Fatalf("name = %q, want %q", name, test.wantName)
|
||||
}
|
||||
if version != test.wantVersion {
|
||||
t.Fatalf("version = %q, want %q", version, test.wantVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The client addresses only the vault it was built for, so a key URL naming a
|
||||
// different vault must not resolve to this vault's same-named key.
|
||||
func TestSplitKeyIDExplicitDefaultPort(t *testing.T) {
|
||||
provider := &AzureKMSProvider{vaultURL: "https://myvault.vault.azure.net:443"}
|
||||
name, _, err := provider.splitKeyID("https://myvault.vault.azure.net/keys/my-key")
|
||||
if err != nil || name != "my-key" {
|
||||
t.Fatalf("splitKeyID = %q, %v; want my-key, nil", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitKeyIDRejectsForeignVault(t *testing.T) {
|
||||
provider := &AzureKMSProvider{vaultURL: "https://myvault.vault.azure.net/"}
|
||||
|
||||
if _, _, err := provider.splitKeyID("https://evil.vault.azure.net/keys/my-key/abc123"); err == nil {
|
||||
t.Fatal("splitKeyID accepted a key URL from another vault")
|
||||
}
|
||||
}
|
||||
|
||||
// A trailing DNS dot names the same vault, so both sides of the comparison are
|
||||
// normalized the same way.
|
||||
func TestSplitKeyIDTrailingDot(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vaultURL string
|
||||
keyID string
|
||||
}{
|
||||
{
|
||||
name: "trailing dot on the key url",
|
||||
vaultURL: "https://myvault.vault.azure.net",
|
||||
keyID: "https://myvault.vault.azure.net./keys/my-key/abc123",
|
||||
},
|
||||
{
|
||||
name: "trailing dot on the configured vault",
|
||||
vaultURL: "https://myvault.vault.azure.net./",
|
||||
keyID: "https://myvault.vault.azure.net/keys/my-key/abc123",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
provider := &AzureKMSProvider{vaultURL: tt.vaultURL}
|
||||
name, version, err := provider.splitKeyID(tt.keyID)
|
||||
if err != nil || name != "my-key" || version != "abc123" {
|
||||
t.Fatalf("splitKeyID = %q, %q, %v; want my-key, abc123, nil", name, version, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The envelope is JSON, so a raw binary wrapped key would be replaced by
|
||||
// U+FFFD on the way in and could never be decrypted.
|
||||
func TestCiphertextEnvelopeRoundTrip(t *testing.T) {
|
||||
wrapped := make([]byte, 256)
|
||||
if _, err := rand.Read(wrapped); err != nil {
|
||||
t.Fatalf("generate wrapped key: %v", err)
|
||||
}
|
||||
|
||||
keyID := "https://myvault.vault.azure.net/keys/my-key/abc123"
|
||||
envelopeBlob, err := seaweedkms.CreateEnvelope("azure", keyID, encodeCiphertext(wrapped), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("create envelope: %v", err)
|
||||
}
|
||||
|
||||
envelope, err := seaweedkms.ParseEnvelope(envelopeBlob)
|
||||
if err != nil {
|
||||
t.Fatalf("parse envelope: %v", err)
|
||||
}
|
||||
if envelope.KeyID != keyID {
|
||||
t.Fatalf("key id = %q, want %q", envelope.KeyID, keyID)
|
||||
}
|
||||
|
||||
decoded, err := decodeCiphertext(envelope.Ciphertext)
|
||||
if err != nil {
|
||||
t.Fatalf("decode ciphertext: %v", err)
|
||||
}
|
||||
if !bytes.Equal(decoded, wrapped) {
|
||||
t.Fatal("decoded wrapped key differs from the encrypted one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeCiphertextRejectsInvalidInput(t *testing.T) {
|
||||
for _, ciphertext := range []string{"", "not base64!!", "a"} {
|
||||
if _, err := decodeCiphertext(ciphertext); err == nil {
|
||||
t.Fatalf("decodeCiphertext(%q) succeeded, want error", ciphertext)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RSA-OAEP cannot authenticate the encryption context, so the context is
|
||||
// bound to the wrapped key via a digest in the envelope.
|
||||
func TestEncryptionContextBinding(t *testing.T) {
|
||||
context := map[string]string{"aws:s3:bucket": "bucket", "aws:s3:object": "key"}
|
||||
envelopeBlob, err := seaweedkms.CreateEnvelope("azure", "my-key", "d3JhcHBlZA==", map[string]interface{}{
|
||||
"encryption_context_sha256": contextDigest(context),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create envelope: %v", err)
|
||||
}
|
||||
envelope, err := seaweedkms.ParseEnvelope(envelopeBlob)
|
||||
if err != nil {
|
||||
t.Fatalf("parse envelope: %v", err)
|
||||
}
|
||||
|
||||
if err := checkContext(envelope, context); err != nil {
|
||||
t.Fatalf("matching context rejected: %v", err)
|
||||
}
|
||||
if err := checkContext(envelope, map[string]string{"aws:s3:object": "other"}); err == nil {
|
||||
t.Fatal("mismatched context accepted")
|
||||
}
|
||||
if err := checkContext(envelope, nil); err == nil {
|
||||
t.Fatal("missing context accepted for a context-bound key")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeCredential struct{}
|
||||
|
||||
func (fakeCredential) GetToken(_ context.Context, _ policy.TokenRequestOptions) (azcore.AccessToken, error) {
|
||||
return azcore.AccessToken{Token: "token", ExpiresOn: time.Now().Add(time.Hour)}, nil
|
||||
}
|
||||
|
||||
type fakeTransport struct {
|
||||
do func(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
func (t fakeTransport) Do(req *http.Request) (*http.Response, error) {
|
||||
return t.do(req)
|
||||
}
|
||||
|
||||
func jsonResponse(status int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Header: http.Header{"Content-Type": []string{"application/json"}},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
}
|
||||
|
||||
// Drive GenerateDataKey and Decrypt through a fake Key Vault transport so the
|
||||
// whole path — key ID split, request body shape, envelope encoding — is
|
||||
// exercised, not just the helpers.
|
||||
func TestGenerateDataKeyDecryptRoundTrip(t *testing.T) {
|
||||
vaultURL := "https://myvault.vault.azure.net"
|
||||
var wrapped []byte
|
||||
var sawAAD bool
|
||||
|
||||
client, err := azkeys.NewClient(vaultURL, fakeCredential{}, &azkeys.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Transport: fakeTransport{do: func(req *http.Request) (*http.Response, error) {
|
||||
if req.Header.Get("Authorization") == "" {
|
||||
// Key Vault answers an unauthenticated request with the
|
||||
// challenge the client then satisfies.
|
||||
resp := jsonResponse(401, `{"error":{"code":"Unauthorized"}}`)
|
||||
resp.Header.Set("WWW-Authenticate", `Bearer authorization="https://login.windows.net/tenant", resource="https://vault.azure.net"`)
|
||||
return resp, nil
|
||||
}
|
||||
var body []byte
|
||||
if rc, err := req.GetBody(); err == nil {
|
||||
body, _ = io.ReadAll(rc)
|
||||
rc.Close()
|
||||
} else if req.Body != nil {
|
||||
body, _ = io.ReadAll(req.Body)
|
||||
}
|
||||
sawAAD = bytes.Contains(body, []byte(`"aad"`))
|
||||
var params struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
if err := json.Unmarshal(body, ¶ms); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := base64.RawURLEncoding.DecodeString(params.Value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case strings.HasSuffix(req.URL.Path, "/encrypt"):
|
||||
wrapped = value
|
||||
return jsonResponse(200, `{"kid":"`+vaultURL+`/keys/my-key/abc123","value":"`+base64.RawURLEncoding.EncodeToString(wrapped)+`"}`), nil
|
||||
case strings.HasSuffix(req.URL.Path, "/decrypt"):
|
||||
if !bytes.Equal(value, wrapped) {
|
||||
return jsonResponse(400, `{"error":{"code":"BadParameter"}}`), nil
|
||||
}
|
||||
return jsonResponse(200, `{"kid":"`+vaultURL+`/keys/my-key/abc123","value":"`+params.Value+`"}`), nil
|
||||
default:
|
||||
return jsonResponse(404, `{"error":{"code":"NotFound"}}`), nil
|
||||
}
|
||||
}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("new client: %v", err)
|
||||
}
|
||||
|
||||
provider := &AzureKMSProvider{client: client, vaultURL: vaultURL}
|
||||
contextMap := map[string]string{"aws:s3:bucket": "bucket"}
|
||||
|
||||
resp, err := provider.GenerateDataKey(context.Background(), &seaweedkms.GenerateDataKeyRequest{
|
||||
KeyID: vaultURL + "/keys/my-key",
|
||||
KeySpec: seaweedkms.KeySpecAES256,
|
||||
EncryptionContext: contextMap,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateDataKey: %v", err)
|
||||
}
|
||||
if sawAAD {
|
||||
t.Fatal("encrypt request carried AAD")
|
||||
}
|
||||
|
||||
decrypted, err := provider.Decrypt(context.Background(), &seaweedkms.DecryptRequest{
|
||||
CiphertextBlob: resp.CiphertextBlob,
|
||||
EncryptionContext: contextMap,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Decrypt: %v", err)
|
||||
}
|
||||
if !bytes.Equal(decrypted.Plaintext, resp.Plaintext) {
|
||||
t.Fatal("decrypted key differs from generated key")
|
||||
}
|
||||
|
||||
if _, err := provider.Decrypt(context.Background(), &seaweedkms.DecryptRequest{
|
||||
CiphertextBlob: resp.CiphertextBlob,
|
||||
EncryptionContext: map[string]string{"aws:s3:bucket": "other"},
|
||||
}); err == nil {
|
||||
t.Fatal("Decrypt succeeded with a different encryption context")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckContextEmpty(t *testing.T) {
|
||||
// Keys wrapped without a context carry no digest; decrypting them with
|
||||
// no context must succeed, and with a context must fail.
|
||||
blob, err := seaweedkms.CreateEnvelope("azure", "my-key", "d3JhcHBlZA==", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("create envelope: %v", err)
|
||||
}
|
||||
envelope, err := seaweedkms.ParseEnvelope(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("parse envelope: %v", err)
|
||||
}
|
||||
for _, context := range []map[string]string{nil, {}} {
|
||||
if err := checkContext(envelope, context); err != nil {
|
||||
t.Fatalf("empty context rejected for an unbound key: %v", err)
|
||||
}
|
||||
}
|
||||
if err := checkContext(envelope, map[string]string{"k": "v"}); err == nil {
|
||||
t.Fatal("context accepted for an unbound key")
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package mount
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/seaweedfs/go-fuse/v2/fuse"
|
||||
)
|
||||
|
||||
@@ -39,5 +41,5 @@ func (l fuseDirEntryList) TakesLookupRef() bool { return true }
|
||||
// ReadDirectoryInto runs a readdir against sink. ReadDir and ReadDirPlus are
|
||||
// this with the kernel reply buffer as the sink.
|
||||
func (wfs *WFS) ReadDirectoryInto(input *fuse.ReadIn, sink DirEntrySink, isPlusMode bool) fuse.Status {
|
||||
return wfs.doReadDirectory(input, sink, isPlusMode)
|
||||
return wfs.doReadDirectory(context.Background(), input, sink, isPlusMode)
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ func TestEnsureVisitedReplaysBufferedEventsAfterSnapshot(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
t.Fatalf("ensure visited: %v", err)
|
||||
}
|
||||
if applyErr != nil {
|
||||
@@ -504,7 +504,7 @@ func TestEnsureVisitedPreservesLocalOnlyEntry(t *testing.T) {
|
||||
}},
|
||||
}}
|
||||
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
t.Fatalf("ensure visited: %v", err)
|
||||
}
|
||||
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
|
||||
@@ -550,7 +550,7 @@ func TestEnsureVisitedDropsUnpinnedStaleEntry(t *testing.T) {
|
||||
}},
|
||||
}}
|
||||
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
t.Fatalf("ensure visited: %v", err)
|
||||
}
|
||||
if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/stale.txt")); err != filer_pb.ErrNotFound || entry != nil {
|
||||
@@ -597,7 +597,7 @@ func TestEnsureVisitedConfirmsTransientEmptyListing(t *testing.T) {
|
||||
},
|
||||
}}
|
||||
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 0); err != nil {
|
||||
t.Fatalf("ensure visited: %v", err)
|
||||
}
|
||||
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
|
||||
@@ -619,7 +619,7 @@ func TestEnsureVisitedCachesGenuinelyEmptyDirectory(t *testing.T) {
|
||||
}
|
||||
accessor := &buildFilerAccessor{client: client}
|
||||
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/empty"), 0); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/empty"), 0); err != nil {
|
||||
t.Fatalf("ensure visited: %v", err)
|
||||
}
|
||||
if !mc.IsDirectoryCached(util.FullPath("/empty")) {
|
||||
|
||||
@@ -26,7 +26,7 @@ func (e *DirectoryTooLargeError) Error() string {
|
||||
|
||||
// maxCacheableEntries is the directory size above which a build gives up, or 0
|
||||
// to cache everything.
|
||||
func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath, maxCacheableEntries int) error {
|
||||
func EnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerClient, dirPath util.FullPath, maxCacheableEntries int) error {
|
||||
// Collect all uncached paths from target directory up to root
|
||||
var uncachedPaths []util.FullPath
|
||||
currentPath := dirPath
|
||||
@@ -61,7 +61,7 @@ func EnsureVisited(mc *MetaCache, client filer_pb.FilerClient, dirPath util.Full
|
||||
|
||||
// Fetch all uncached directories in parallel with context for cancellation
|
||||
// If one fetch fails, cancel the others to avoid unnecessary work
|
||||
g, ctx := errgroup.WithContext(context.Background())
|
||||
g, ctx := errgroup.WithContext(ctx)
|
||||
for _, p := range uncachedPaths {
|
||||
path := p // capture for closure
|
||||
g.Go(func() error {
|
||||
@@ -91,8 +91,10 @@ const (
|
||||
)
|
||||
|
||||
func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerClient, path util.FullPath, maxCacheableEntries int) error {
|
||||
// Use singleflight to deduplicate concurrent requests for the same path
|
||||
_, err, _ := mc.visitGroup.Do(string(path), func() (interface{}, error) {
|
||||
// Use singleflight to deduplicate concurrent requests for the same path.
|
||||
// DoChan so a canceled request stops waiting even when another request
|
||||
// owns the in-flight build.
|
||||
resultCh := mc.visitGroup.DoChan(string(path), func() (interface{}, error) {
|
||||
// Check for cancellation before starting
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
@@ -222,7 +224,12 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
|
||||
cleanupDone = true // Prevent deferred cleanup after successful publish
|
||||
return nil, nil
|
||||
})
|
||||
return err
|
||||
select {
|
||||
case result := <-resultCh:
|
||||
return result.Err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func IsHiddenSystemEntry(dir, name string) bool {
|
||||
|
||||
@@ -37,7 +37,7 @@ func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
|
||||
|
||||
accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(10)}}
|
||||
|
||||
err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5)
|
||||
err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5)
|
||||
var tooLarge *DirectoryTooLargeError
|
||||
if !errors.As(err, &tooLarge) {
|
||||
t.Fatalf("EnsureVisited = %v, want DirectoryTooLargeError", err)
|
||||
@@ -61,7 +61,7 @@ func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
|
||||
}
|
||||
|
||||
// A second visit fails fast without streaming to the limit again.
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
|
||||
t.Fatalf("second EnsureVisited = %v, want DirectoryTooLargeError", err)
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,7 @@ func TestEnsureVisitedStepsOverOversizedAncestor(t *testing.T) {
|
||||
mc.markOversized(util.FullPath("/huge"))
|
||||
|
||||
accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(3)}}
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
|
||||
t.Fatalf("EnsureVisited under an oversized ancestor: %v", err)
|
||||
}
|
||||
if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
|
||||
@@ -92,7 +92,7 @@ func TestEnsureVisitedUnderTheLimitStillCaches(t *testing.T) {
|
||||
defer mc.Shutdown()
|
||||
|
||||
accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(5)}}
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/dir"), 5); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5); err != nil {
|
||||
t.Fatalf("EnsureVisited: %v", err)
|
||||
}
|
||||
if !mc.IsDirectoryCached(util.FullPath("/dir")) {
|
||||
@@ -123,7 +123,7 @@ func TestEnsureVisitedAncestorFoundOversizedMidVisit(t *testing.T) {
|
||||
"/huge/sub": listResponses(3),
|
||||
}}}
|
||||
|
||||
if err := EnsureVisited(mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
|
||||
if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
|
||||
t.Fatalf("EnsureVisited: %v", err)
|
||||
}
|
||||
if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
|
||||
|
||||
+37
-12
@@ -502,12 +502,30 @@ func (wfs *WFS) Init(server *fuse.Server) {
|
||||
wfs.fuseServer = server
|
||||
}
|
||||
|
||||
// ctxFromCancel bridges a FUSE request's interrupt channel into a context so
|
||||
// metadata lookups can abandon a stalled filer call when the kernel has given
|
||||
// up on the operation.
|
||||
func ctxFromCancel(cancel <-chan struct{}) (context.Context, context.CancelFunc) {
|
||||
ctx, cancelFunc := context.WithCancel(context.Background())
|
||||
if cancel == nil {
|
||||
return ctx, cancelFunc
|
||||
}
|
||||
go func() {
|
||||
select {
|
||||
case <-cancel:
|
||||
cancelFunc()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
return ctx, cancelFunc
|
||||
}
|
||||
|
||||
// maybeReadEntry resolves an inode to the entry metadata operations act on. An
|
||||
// open handle answers ahead of the path: unlink drops the name while the
|
||||
// descriptor stays valid, so an unlinked-but-open file returns its handle's
|
||||
// entry with an empty path instead of ENOENT. A removed directory has no
|
||||
// handle; its remembered entry answers the same way.
|
||||
func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) {
|
||||
func (wfs *WFS) maybeReadEntry(ctx context.Context, inode uint64) (path util.FullPath, fh *FileHandle, entry *filer_pb.Entry, status fuse.Status) {
|
||||
var found bool
|
||||
if fh, found = wfs.fhMap.FindFileHandle(inode); found {
|
||||
path, _ = wfs.inodeToPath.GetPath(inode)
|
||||
@@ -525,7 +543,7 @@ func (wfs *WFS) maybeReadEntry(inode uint64) (path util.FullPath, fh *FileHandle
|
||||
}
|
||||
return
|
||||
}
|
||||
entry, _, status = wfs.maybeLoadEntry(path)
|
||||
entry, _, status = wfs.maybeLoadEntry(ctx, path)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -554,7 +572,7 @@ func (wfs *WFS) isLocalOnlyEntry(entry *filer.Entry) bool {
|
||||
|
||||
// maybeLoadEntry returns the entry and the log position it reflects, or a zero
|
||||
// position when unknown.
|
||||
func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, entryVersion, fuse.Status) {
|
||||
func (wfs *WFS) maybeLoadEntry(ctx context.Context, fullpath util.FullPath) (*filer_pb.Entry, entryVersion, fuse.Status) {
|
||||
// glog.V(3).Infof("read entry cache miss %s", fullpath)
|
||||
_, name := fullpath.DirAndName()
|
||||
|
||||
@@ -573,7 +591,7 @@ func (wfs *WFS) maybeLoadEntry(fullpath util.FullPath) (*filer_pb.Entry, entryVe
|
||||
}, entryVersion{}, fuse.OK
|
||||
}
|
||||
|
||||
entry, version, status := wfs.lookupEntry(fullpath)
|
||||
entry, version, status := wfs.lookupEntry(ctx, fullpath)
|
||||
if status != fuse.OK {
|
||||
return nil, entryVersion{}, status
|
||||
}
|
||||
@@ -592,7 +610,7 @@ const expiredDirRebuildCooldown = 30 * time.Second
|
||||
// local entries do not leak back into lookup results.
|
||||
// It also returns the log position the entry reflects: the entry's stored
|
||||
// version, the lookup response's, or zero if unknown.
|
||||
func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion, fuse.Status) {
|
||||
func (wfs *WFS) lookupEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, entryVersion, fuse.Status) {
|
||||
dir, _ := fullpath.DirAndName()
|
||||
dirPath := util.FullPath(dir)
|
||||
|
||||
@@ -605,9 +623,11 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
|
||||
// flush of the target entry land first so the rebuilt cache does not
|
||||
// capture pre-flush metadata and bypass the wait below.
|
||||
if inode, found := wfs.inodeToPath.GetInode(fullpath); found {
|
||||
wfs.waitForPendingAsyncFlush(inode)
|
||||
if !wfs.waitForPendingAsyncFlush(ctx, inode) {
|
||||
return nil, entryVersion{}, fuse.EINTR
|
||||
}
|
||||
}
|
||||
if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
|
||||
if err := wfs.ensureDirectoryVisited(ctx, dirPath); err != nil {
|
||||
// Record the attempt so the cooldown suppresses repeated rebuilds
|
||||
// while the listing keeps failing; once it elapses a later lookup
|
||||
// retries. Oversized dirs are already marked read-through.
|
||||
@@ -619,7 +639,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
|
||||
}
|
||||
|
||||
if wfs.metaCache.IsDirectoryCached(dirPath) && wfs.metaCache.IsNameFresh(fullpath) {
|
||||
cachedEntry, cachedVersionTsNs, cacheErr := wfs.metaCache.FindEntry(context.Background(), fullpath)
|
||||
cachedEntry, cachedVersionTsNs, cacheErr := wfs.metaCache.FindEntry(ctx, fullpath)
|
||||
if cacheErr != nil && cacheErr != filer_pb.ErrNotFound {
|
||||
glog.Errorf("lookupEntry: cache lookup for %s failed: %v", fullpath, cacheErr)
|
||||
return nil, entryVersion{}, fuse.EIO
|
||||
@@ -650,7 +670,9 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
|
||||
// truncate's old size or a write's old chunks. The cache paths above are
|
||||
// already consistent and must not pay this wait.
|
||||
if inode, found := wfs.inodeToPath.GetInode(fullpath); found {
|
||||
wfs.waitForPendingAsyncFlush(inode)
|
||||
if !wfs.waitForPendingAsyncFlush(ctx, inode) {
|
||||
return nil, entryVersion{}, fuse.EINTR
|
||||
}
|
||||
}
|
||||
|
||||
// Directory not cached - fetch directly from filer without caching the entire directory.
|
||||
@@ -658,7 +680,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
|
||||
var entry *filer_pb.Entry
|
||||
var lookupVersion entryVersion
|
||||
lookupDir, lookupName := fullpath.DirAndName()
|
||||
err := wfs.withFilerClient(context.Background(), false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
|
||||
err := wfs.withFilerClient(ctx, false, wfs.filerRPCWait(), func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
|
||||
resp, lookupErr := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: lookupDir,
|
||||
Name: lookupName,
|
||||
@@ -671,6 +693,9 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, entryVersion{}, fuse.EINTR
|
||||
}
|
||||
if err == filer_pb.ErrNotFound {
|
||||
// The entry may exist in the local store from a deferred create
|
||||
// (deferFilerCreate=true) that hasn't been flushed yet. Only trust
|
||||
@@ -689,7 +714,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
|
||||
wfs.pendingAsyncFlushMu.Unlock()
|
||||
|
||||
if hasDirtyHandle || hasPendingFlush {
|
||||
if localEntry, localVersionTsNs, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil {
|
||||
if localEntry, localVersionTsNs, localErr := wfs.metaCache.FindEntry(ctx, fullpath); localErr == nil && localEntry != nil {
|
||||
glog.V(4).Infof("lookupEntry found deferred entry in local cache %s", fullpath)
|
||||
return localEntry, entryVersion{tsNs: localVersionTsNs}, fuse.OK
|
||||
}
|
||||
@@ -723,7 +748,7 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, entryVersion,
|
||||
// up without -v=4 — and include layer-by-layer state so the
|
||||
// next failed run pinpoints which layer dropped the entry.
|
||||
localPresent := false
|
||||
if localEntry, _, localErr := wfs.metaCache.FindEntry(context.Background(), fullpath); localErr == nil && localEntry != nil {
|
||||
if localEntry, _, localErr := wfs.metaCache.FindEntry(ctx, fullpath); localErr == nil && localEntry != nil {
|
||||
localPresent = true
|
||||
}
|
||||
glog.Warningf("lookupEntry: filer ErrNotFound for tracked path %s (inode=%d dirtyHandle=%v pendingFlush=%v localCache=%v dirCached=%v) — possible coherence bug",
|
||||
|
||||
@@ -82,7 +82,9 @@ func clearSupplementaryGroupCache() {
|
||||
* This method is not called under Linux kernel versions 2.4.x
|
||||
*/
|
||||
func (wfs *WFS) Access(cancel <-chan struct{}, input *fuse.AccessIn) (code fuse.Status) {
|
||||
_, _, entry, code := wfs.maybeReadEntry(input.NodeId)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
_, _, entry, code := wfs.maybeReadEntry(ctx, input.NodeId)
|
||||
if code != fuse.OK {
|
||||
return code
|
||||
}
|
||||
|
||||
@@ -26,7 +26,9 @@ func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse
|
||||
}
|
||||
|
||||
inode := input.NodeId
|
||||
path, fh, entry, status := wfs.maybeReadEntry(inode)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
path, fh, entry, status := wfs.maybeReadEntry(ctx, inode)
|
||||
if status != fuse.OK {
|
||||
return status
|
||||
}
|
||||
@@ -65,7 +67,9 @@ func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse
|
||||
return fuse.Status(syscall.ENOSPC)
|
||||
}
|
||||
|
||||
path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
path, fh, entry, status := wfs.maybeReadEntry(ctx, input.NodeId)
|
||||
if status != fuse.OK || entry == nil {
|
||||
return status
|
||||
}
|
||||
@@ -336,7 +340,7 @@ func (wfs *WFS) touchDirMtimeCtimeBest(dirPath util.FullPath) {
|
||||
// touchDirMtimeCtime updates a directory's mtime and ctime on the filer.
|
||||
// POSIX requires this when entries are created or removed in the directory.
|
||||
func (wfs *WFS) touchDirMtimeCtime(dirPath util.FullPath) {
|
||||
dirEntry, _, code := wfs.maybeLoadEntry(dirPath)
|
||||
dirEntry, _, code := wfs.maybeLoadEntry(context.Background(), dirPath)
|
||||
if code != fuse.OK || dirEntry == nil || dirEntry.Attributes == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -27,7 +27,9 @@ func (wfs *WFS) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
fullFilePath := dirPath.Child(name)
|
||||
|
||||
// Use shared lookup logic that checks cache first, then filer if needed
|
||||
localEntry, _, status := wfs.lookupEntry(fullFilePath)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
localEntry, _, status := wfs.lookupEntry(ctx, fullFilePath)
|
||||
if status != fuse.OK {
|
||||
return status
|
||||
}
|
||||
|
||||
@@ -158,7 +158,7 @@ func TestLookupRebuildsExpiredDirectoryCache(t *testing.T) {
|
||||
}
|
||||
startFakeFiler(t, wfs, fake)
|
||||
|
||||
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
|
||||
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, dir, 0); err != nil {
|
||||
t.Fatalf("EnsureVisited: %v", err)
|
||||
}
|
||||
if !wfs.inodeToPath.IsChildrenCached(dir) {
|
||||
@@ -174,7 +174,7 @@ func TestLookupRebuildsExpiredDirectoryCache(t *testing.T) {
|
||||
lookupCallsBefore := fake.lookupCalls.Load()
|
||||
|
||||
for _, name := range []string{"a", "b", "c"} {
|
||||
entry, _, status := wfs.lookupEntry(dir.Child(name))
|
||||
entry, _, status := wfs.lookupEntry(context.Background(), dir.Child(name))
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("lookupEntry %s: %v", name, status)
|
||||
}
|
||||
@@ -211,7 +211,7 @@ func TestLookupRebuildDeduplicatesConcurrentRebuilds(t *testing.T) {
|
||||
}
|
||||
startFakeFiler(t, wfs, fake)
|
||||
|
||||
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
|
||||
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, dir, 0); err != nil {
|
||||
t.Fatalf("EnsureVisited: %v", err)
|
||||
}
|
||||
expireDirCache(t, wfs.inodeToPath, dir)
|
||||
@@ -228,7 +228,7 @@ func TestLookupRebuildDeduplicatesConcurrentRebuilds(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func(i int, name string) {
|
||||
defer wg.Done()
|
||||
if _, _, status := wfs.lookupEntry(dir.Child(name)); status != fuse.OK {
|
||||
if _, _, status := wfs.lookupEntry(context.Background(), dir.Child(name)); status != fuse.OK {
|
||||
errs[i] = fmt.Errorf("lookupEntry %s: %v", name, status)
|
||||
}
|
||||
}(i, name)
|
||||
@@ -271,7 +271,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
|
||||
}
|
||||
startFakeFiler(t, wfs, fake)
|
||||
|
||||
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dir, 0); err != nil {
|
||||
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, dir, 0); err != nil {
|
||||
t.Fatalf("EnsureVisited: %v", err)
|
||||
}
|
||||
expireDirCache(t, wfs.inodeToPath, dir)
|
||||
@@ -280,7 +280,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
|
||||
fake.listErr = errors.New("persistent listing failure")
|
||||
listCallsBefore := fake.listCalls.Load()
|
||||
lookupCallsBefore := fake.lookupCalls.Load()
|
||||
if entry, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK || entry == nil || entry.Name() != "a" {
|
||||
if entry, _, status := wfs.lookupEntry(context.Background(), dir.Child("a")); status != fuse.OK || entry == nil || entry.Name() != "a" {
|
||||
t.Fatalf("lookupEntry a after failed rebuild: status=%v entry=%v", status, entry)
|
||||
}
|
||||
if got := fake.lookupCalls.Load() - lookupCallsBefore; got != 1 {
|
||||
@@ -293,7 +293,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
|
||||
// Immediate retry: cooldown suppresses the rebuild, falls through again.
|
||||
listCallsBefore = fake.listCalls.Load()
|
||||
lookupCallsBefore = fake.lookupCalls.Load()
|
||||
if _, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK {
|
||||
if _, _, status := wfs.lookupEntry(context.Background(), dir.Child("a")); status != fuse.OK {
|
||||
t.Fatalf("second lookupEntry a: %v", status)
|
||||
}
|
||||
if got := fake.listCalls.Load() - listCallsBefore; got != 0 {
|
||||
@@ -308,7 +308,7 @@ func TestLookupRebuildCooldownAfterFailure(t *testing.T) {
|
||||
advanceRebuildAttempt(t, wfs.inodeToPath, dir, expiredDirRebuildCooldown+time.Second)
|
||||
listCallsBefore = fake.listCalls.Load()
|
||||
lookupCallsBefore = fake.lookupCalls.Load()
|
||||
if _, _, status := wfs.lookupEntry(dir.Child("a")); status != fuse.OK {
|
||||
if _, _, status := wfs.lookupEntry(context.Background(), dir.Child("a")); status != fuse.OK {
|
||||
t.Fatalf("third lookupEntry a: %v", status)
|
||||
}
|
||||
if got := fake.listCalls.Load() - listCallsBefore; got != 1 {
|
||||
|
||||
@@ -156,13 +156,20 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string
|
||||
}
|
||||
entryFullPath := dirFullPath.Child(name)
|
||||
|
||||
targetEntry, _, targetCode := wfs.maybeLoadEntry(entryFullPath)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
targetEntry, _, targetCode := wfs.maybeLoadEntry(ctx, entryFullPath)
|
||||
if targetCode == fuse.EINTR {
|
||||
return targetCode
|
||||
}
|
||||
if targetCode != fuse.OK {
|
||||
targetEntry = nil
|
||||
}
|
||||
|
||||
// POSIX: enforce sticky bit on the parent directory.
|
||||
if dirEntry, _, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil {
|
||||
if dirEntry, _, dirCode := wfs.maybeLoadEntry(ctx, dirFullPath); dirCode != fuse.OK {
|
||||
return dirCode
|
||||
} else if dirEntry != nil && dirEntry.Attributes != nil {
|
||||
targetUid := uint32(0)
|
||||
if targetEntry != nil && targetEntry.Attributes != nil {
|
||||
targetUid = targetEntry.Attributes.Uid
|
||||
|
||||
@@ -176,14 +176,20 @@ func (wfs *WFS) FsyncDir(cancel <-chan struct{}, input *fuse.FsyncIn) (code fuse
|
||||
* '1'.
|
||||
*/
|
||||
func (wfs *WFS) ReadDir(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
|
||||
return wfs.doReadDirectory(input, fuseDirEntryList{out}, false)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
return wfs.doReadDirectory(ctx, input, fuseDirEntryList{out}, false)
|
||||
}
|
||||
|
||||
func (wfs *WFS) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out *fuse.DirEntryList) (code fuse.Status) {
|
||||
return wfs.doReadDirectory(input, fuseDirEntryList{out}, true)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
return wfs.doReadDirectory(ctx, input, fuseDirEntryList{out}, true)
|
||||
}
|
||||
|
||||
func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode bool) fuse.Status {
|
||||
func (wfs *WFS) doReadDirectory(ctx context.Context, input *fuse.ReadIn, out DirEntrySink, isPlusMode bool) fuse.Status {
|
||||
listingCtx := filer_pb.WithChunksOmitted(ctx)
|
||||
|
||||
// Get the directory handle and lock it for the duration of this operation.
|
||||
// This serializes concurrent readdir calls on the same handle, fixing the
|
||||
// race condition that caused hangs with NFS-Ganesha.
|
||||
@@ -282,7 +288,7 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
|
||||
dh.dropConsumed(input.Offset)
|
||||
|
||||
if wfs.inodeToPath.ShouldReadDirectoryDirect(dirPath) {
|
||||
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
|
||||
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
|
||||
}
|
||||
|
||||
// Read from cache first, then load next batch if needed
|
||||
@@ -292,32 +298,38 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
|
||||
if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
|
||||
skipCount := int64(input.Offset - dh.entryStreamOffset)
|
||||
|
||||
if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
|
||||
if err := wfs.ensureDirectoryVisited(ctx, dirPath); err != nil {
|
||||
var tooLarge *meta_cache.DirectoryTooLargeError
|
||||
if errors.As(err, &tooLarge) {
|
||||
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
|
||||
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fuse.EINTR
|
||||
}
|
||||
glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
|
||||
return fuse.EIO
|
||||
}
|
||||
|
||||
// Serve the maintained-but-unverified cache if the filer is unreachable.
|
||||
if err := meta_cache.EnsureListingFresh(context.Background(), wfs.metaCache, wfs, dirPath, ""); err != nil {
|
||||
if err := meta_cache.EnsureListingFresh(ctx, wfs.metaCache, wfs, dirPath, ""); err != nil {
|
||||
if errors.Is(err, meta_cache.ErrRefreshRangeTooLarge) {
|
||||
// re-tile with a full rebuild; serve this request direct
|
||||
wfs.purgeDirectoryCache(dirPath)
|
||||
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
|
||||
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
|
||||
}
|
||||
glog.V(1).Infof("refresh %s sections: %v", dirPath, err)
|
||||
}
|
||||
|
||||
// Load entries from beginning to fill cache up to the requested offset
|
||||
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(readdirContext, dirPath, "", false, skipCount+int64(batchSize), func(entry *filer.Entry) (bool, error) {
|
||||
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(listingCtx, dirPath, "", false, skipCount+int64(batchSize), func(entry *filer.Entry) (bool, error) {
|
||||
dh.entryStream = append(dh.entryStream, entry)
|
||||
return true, nil
|
||||
})
|
||||
dh.lastListedName = storeLastName
|
||||
if loadErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fuse.EINTR
|
||||
}
|
||||
glog.Errorf("list meta cache: %v", loadErr)
|
||||
return fuse.EIO
|
||||
}
|
||||
@@ -349,12 +361,15 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
|
||||
}
|
||||
|
||||
// Cache exhausted, load next batch
|
||||
if err := wfs.ensureDirectoryVisited(dirPath); err != nil {
|
||||
if err := wfs.ensureDirectoryVisited(ctx, dirPath); err != nil {
|
||||
var tooLarge *meta_cache.DirectoryTooLargeError
|
||||
if errors.As(err, &tooLarge) {
|
||||
// The direct path keeps the same pagination state on dh, so it
|
||||
// carries on from wherever the cached walk reached.
|
||||
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
|
||||
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fuse.EINTR
|
||||
}
|
||||
glog.Errorf("dir ReadDirAll %s: %v", dirPath, err)
|
||||
return fuse.EIO
|
||||
@@ -369,17 +384,17 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
|
||||
}
|
||||
|
||||
// Serve the maintained-but-unverified cache if the filer is unreachable.
|
||||
if err := meta_cache.EnsureListingFresh(context.Background(), wfs.metaCache, wfs, dirPath, lastEntryName); err != nil {
|
||||
if err := meta_cache.EnsureListingFresh(ctx, wfs.metaCache, wfs, dirPath, lastEntryName); err != nil {
|
||||
if errors.Is(err, meta_cache.ErrRefreshRangeTooLarge) {
|
||||
// re-tile with a full rebuild; serve this request direct
|
||||
wfs.purgeDirectoryCache(dirPath)
|
||||
return wfs.readDirectoryDirect(input, out, dh, dirPath, processEachEntryFn)
|
||||
return wfs.readDirectoryDirect(ctx, input, out, dh, dirPath, processEachEntryFn)
|
||||
}
|
||||
glog.V(1).Infof("refresh %s sections: %v", dirPath, err)
|
||||
}
|
||||
|
||||
bufferFull := false
|
||||
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(readdirContext, dirPath, lastEntryName, false, int64(batchSize), func(entry *filer.Entry) (bool, error) {
|
||||
storeLastName, loadErr := wfs.metaCache.ListDirectoryEntries(listingCtx, dirPath, lastEntryName, false, int64(batchSize), func(entry *filer.Entry) (bool, error) {
|
||||
currentIndex := int64(len(dh.entryStream))
|
||||
dh.entryStream = append(dh.entryStream, entry)
|
||||
if !processEachEntryFn(entry, currentIndex) {
|
||||
@@ -389,6 +404,9 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
|
||||
return true, nil
|
||||
})
|
||||
if loadErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fuse.EINTR
|
||||
}
|
||||
glog.Errorf("list meta cache: %v", loadErr)
|
||||
return fuse.EIO
|
||||
}
|
||||
@@ -408,8 +426,8 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out DirEntrySink, isPlusMode
|
||||
// ensureDirectoryVisited pulls the directory into the local cache, unless it is
|
||||
// too large to cache: then the directory is marked read-through, so later
|
||||
// listings go straight to the filer without re-asking.
|
||||
func (wfs *WFS) ensureDirectoryVisited(dirPath util.FullPath) error {
|
||||
err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath, wfs.option.CacheDirMaxEntries)
|
||||
func (wfs *WFS) ensureDirectoryVisited(ctx context.Context, dirPath util.FullPath) error {
|
||||
err := meta_cache.EnsureVisited(ctx, wfs.metaCache, wfs, dirPath, wfs.option.CacheDirMaxEntries)
|
||||
var tooLarge *meta_cache.DirectoryTooLargeError
|
||||
if errors.As(err, &tooLarge) {
|
||||
wfs.inodeToPath.MarkDirectoryReadThrough(dirPath, time.Now())
|
||||
@@ -417,14 +435,18 @@ func (wfs *WFS) ensureDirectoryVisited(dirPath util.FullPath) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out DirEntrySink, dh *DirectoryHandle, dirPath util.FullPath, processEachEntryFn func(entry *filer.Entry, index int64) bool) fuse.Status {
|
||||
func (wfs *WFS) readDirectoryDirect(ctx context.Context, input *fuse.ReadIn, out DirEntrySink, dh *DirectoryHandle, dirPath util.FullPath, processEachEntryFn func(entry *filer.Entry, index int64) bool) fuse.Status {
|
||||
listingCtx := filer_pb.WithChunksOmitted(ctx)
|
||||
var lastEntryName string
|
||||
|
||||
if input.Offset >= dh.entryStreamOffset {
|
||||
if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
|
||||
skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize
|
||||
entries, snapshotTs, err := loadDirectoryEntriesDirect(readdirContext, wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
|
||||
entries, snapshotTs, err := loadDirectoryEntriesDirect(listingCtx, wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fuse.EINTR
|
||||
}
|
||||
glog.Errorf("list filer directory: %v", err)
|
||||
return fuse.EIO
|
||||
}
|
||||
@@ -457,8 +479,11 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out DirEntrySink, dh *Di
|
||||
}
|
||||
}
|
||||
|
||||
entries, snapshotTs, err := loadDirectoryEntriesDirect(readdirContext, wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
|
||||
entries, snapshotTs, err := loadDirectoryEntriesDirect(listingCtx, wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return fuse.EINTR
|
||||
}
|
||||
glog.Errorf("list filer directory: %v", err)
|
||||
return fuse.EIO
|
||||
}
|
||||
|
||||
@@ -163,7 +163,7 @@ func walkOnce(tb testing.TB, wfs *WFS, dirInode uint64, sink *benchSink, forgets
|
||||
offset := uint64(0)
|
||||
for {
|
||||
sink.reset()
|
||||
status := wfs.doReadDirectory(&fuse.ReadIn{
|
||||
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
|
||||
InHeader: fuse.InHeader{NodeId: dirInode},
|
||||
Fh: uint64(dhid),
|
||||
Offset: offset,
|
||||
|
||||
@@ -111,7 +111,7 @@ func TestReadDirResumePastShrunkDirectory(t *testing.T) {
|
||||
defer wfs.ReleaseDirectoryHandle(dhid)
|
||||
|
||||
sink := &pagingSink{limit: 4096}
|
||||
status := wfs.doReadDirectory(&fuse.ReadIn{
|
||||
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
|
||||
InHeader: fuse.InHeader{NodeId: dirInode},
|
||||
Fh: uint64(dhid),
|
||||
Offset: 152, // past the end of a directory that now holds 100
|
||||
@@ -158,7 +158,7 @@ func TestReadDirWithExpiredEntries(t *testing.T) {
|
||||
var offset uint64
|
||||
for round := 0; round < 20; round++ {
|
||||
sink.round = 0
|
||||
status := wfs.doReadDirectory(&fuse.ReadIn{
|
||||
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
|
||||
InHeader: fuse.InHeader{NodeId: dirInode},
|
||||
Fh: uint64(dhid),
|
||||
Offset: offset,
|
||||
@@ -209,7 +209,7 @@ func TestReadDirTrimsConsumedEntries(t *testing.T) {
|
||||
for round := 0; round < 500; round++ {
|
||||
sink.round = 0
|
||||
before := len(sink.names)
|
||||
status := wfs.doReadDirectory(&fuse.ReadIn{
|
||||
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
|
||||
InHeader: fuse.InHeader{NodeId: dirInode},
|
||||
Fh: uint64(dhid),
|
||||
Offset: offset,
|
||||
@@ -301,7 +301,7 @@ func TestReadDirDirectTrimsConsumedEntries(t *testing.T) {
|
||||
for round := 0; round < 500; round++ {
|
||||
sink.round = 0
|
||||
before := len(sink.names)
|
||||
status := wfs.doReadDirectory(&fuse.ReadIn{
|
||||
status := wfs.doReadDirectory(context.Background(), &fuse.ReadIn{
|
||||
InHeader: fuse.InHeader{NodeId: dirInode},
|
||||
Fh: uint64(dhid),
|
||||
Offset: offset,
|
||||
|
||||
@@ -65,7 +65,9 @@ import (
|
||||
*/
|
||||
func (wfs *WFS) Open(cancel <-chan struct{}, in *fuse.OpenIn, out *fuse.OpenOut) (status fuse.Status) {
|
||||
var fileHandle *FileHandle
|
||||
fileHandle, status = wfs.AcquireHandle(in.NodeId, in.Flags, in.Uid, in.Gid)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
fileHandle, status = wfs.AcquireHandle(ctx, in.NodeId, in.Flags, in.Uid, in.Gid)
|
||||
if status == fuse.OK {
|
||||
out.Fh = uint64(fileHandle.fh)
|
||||
out.OpenFlags = 0
|
||||
|
||||
@@ -41,7 +41,9 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
|
||||
entryFullPath := dirFullPath.Child(name)
|
||||
var inode uint64
|
||||
|
||||
newEntry, _, code := wfs.maybeLoadEntry(entryFullPath)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
newEntry, _, code := wfs.maybeLoadEntry(ctx, entryFullPath)
|
||||
if code == fuse.OK {
|
||||
if newEntry == nil || newEntry.Attributes == nil {
|
||||
return fuse.EIO
|
||||
@@ -52,7 +54,7 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
|
||||
return fuse.Status(syscall.EEXIST)
|
||||
}
|
||||
inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, false, len(newEntry.HardLinkId) > 0, newEntry.Attributes.Inode, true)
|
||||
fileHandle, status := wfs.AcquireHandle(inode, in.Flags, in.Uid, in.Gid)
|
||||
fileHandle, status := wfs.AcquireHandle(ctx, inode, in.Flags, in.Uid, in.Gid)
|
||||
if status != fuse.OK {
|
||||
return status
|
||||
}
|
||||
@@ -86,14 +88,14 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
|
||||
}
|
||||
}
|
||||
|
||||
inode, newEntry, code = wfs.createRegularFile(dirFullPath, name, in.Mode, in.Uid, in.Gid, 0, !wfs.option.EagerFilerCreate, true)
|
||||
inode, newEntry, code = wfs.createRegularFile(ctx, dirFullPath, name, in.Mode, in.Uid, in.Gid, 0, !wfs.option.EagerFilerCreate, true)
|
||||
if code == fuse.Status(syscall.EEXIST) && in.Flags&syscall.O_EXCL == 0 {
|
||||
// Race: another process created the file between our check and create.
|
||||
// Reopen the winner's entry; AcquireHandle takes its own lock.
|
||||
if dlmLock != nil {
|
||||
dlmLock.Stop()
|
||||
}
|
||||
newEntry, _, code = wfs.maybeLoadEntry(entryFullPath)
|
||||
newEntry, _, code = wfs.maybeLoadEntry(ctx, entryFullPath)
|
||||
if code != fuse.OK {
|
||||
return code
|
||||
}
|
||||
@@ -101,7 +103,7 @@ func (wfs *WFS) Create(cancel <-chan struct{}, in *fuse.CreateIn, name string, o
|
||||
return fuse.EIO
|
||||
}
|
||||
inode = wfs.inodeToPath.Lookup(entryFullPath, newEntry.Attributes.Crtime, false, len(newEntry.HardLinkId) > 0, newEntry.Attributes.Inode, true)
|
||||
fileHandle, status := wfs.AcquireHandle(inode, in.Flags, in.Uid, in.Gid)
|
||||
fileHandle, status := wfs.AcquireHandle(ctx, inode, in.Flags, in.Uid, in.Gid)
|
||||
if status != fuse.OK {
|
||||
return status
|
||||
}
|
||||
@@ -177,7 +179,9 @@ func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out
|
||||
return
|
||||
}
|
||||
|
||||
inode, newEntry, code := wfs.createRegularFile(dirFullPath, name, in.Mode, in.Uid, in.Gid, in.Rdev, false, false)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
inode, newEntry, code := wfs.createRegularFile(ctx, dirFullPath, name, in.Mode, in.Uid, in.Gid, in.Rdev, false, false)
|
||||
if code != fuse.OK {
|
||||
return code
|
||||
}
|
||||
@@ -207,7 +211,9 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
}
|
||||
entryFullPath := dirFullPath.Child(name)
|
||||
|
||||
entry, _, code := wfs.maybeLoadEntry(entryFullPath)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
entry, _, code := wfs.maybeLoadEntry(ctx, entryFullPath)
|
||||
if code != fuse.OK {
|
||||
if code == fuse.ENOENT {
|
||||
return fuse.OK
|
||||
@@ -236,7 +242,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
// maybeLoadEntry path above. Do not fall back to the stale
|
||||
// pre-lock snapshot: proceeding with its HardLinkCounter would
|
||||
// reintroduce the stale-base update the lock is meant to prevent.
|
||||
fresh, _, freshCode := wfs.maybeLoadEntry(entryFullPath)
|
||||
fresh, _, freshCode := wfs.maybeLoadEntry(ctx, entryFullPath)
|
||||
if freshCode == fuse.ENOENT {
|
||||
return fuse.OK
|
||||
}
|
||||
@@ -247,7 +253,9 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
}
|
||||
|
||||
// POSIX: enforce sticky bit on the parent directory.
|
||||
if dirEntry, _, dirCode := wfs.maybeLoadEntry(dirFullPath); dirCode == fuse.OK && dirEntry != nil && dirEntry.Attributes != nil {
|
||||
if dirEntry, _, dirCode := wfs.maybeLoadEntry(ctx, dirFullPath); dirCode != fuse.OK {
|
||||
return dirCode
|
||||
} else if dirEntry != nil && dirEntry.Attributes != nil {
|
||||
targetUid := uint32(0)
|
||||
if entry != nil && entry.Attributes != nil {
|
||||
targetUid = entry.Attributes.Uid
|
||||
@@ -266,11 +274,11 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
// will remove it again.
|
||||
if inode, found := wfs.inodeToPath.GetInode(entryFullPath); found {
|
||||
wfs.markHandleDeleted(inode)
|
||||
wfs.waitForPendingAsyncFlush(inode)
|
||||
wfs.waitForPendingAsyncFlush(context.Background(), inode)
|
||||
} else if entry != nil && entry.Attributes != nil && entry.Attributes.Inode != 0 {
|
||||
inodeFromEntry := entry.Attributes.Inode
|
||||
wfs.markHandleDeleted(inodeFromEntry)
|
||||
wfs.waitForPendingAsyncFlush(inodeFromEntry)
|
||||
wfs.waitForPendingAsyncFlush(context.Background(), inodeFromEntry)
|
||||
}
|
||||
|
||||
// first, ensure the filer store can correctly delete
|
||||
@@ -337,7 +345,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
|
||||
}
|
||||
|
||||
func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode uint32, uid, gid, rdev uint32, deferFilerCreate bool, skipExistenceCheck bool) (inode uint64, newEntry *filer_pb.Entry, code fuse.Status) {
|
||||
func (wfs *WFS) createRegularFile(ctx context.Context, dirFullPath util.FullPath, name string, mode uint32, uid, gid, rdev uint32, deferFilerCreate bool, skipExistenceCheck bool) (inode uint64, newEntry *filer_pb.Entry, code fuse.Status) {
|
||||
if wfs.IsOverQuotaWithUncommitted() {
|
||||
return 0, nil, fuse.Status(syscall.ENOSPC)
|
||||
}
|
||||
@@ -346,7 +354,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u
|
||||
// SkipCheckParentDirectory, so this is the only parent check). With
|
||||
// default_permissions the kernel already verified write+search on it before
|
||||
// Create/Mknod, so skip only the mode-bit check and its group lookup.
|
||||
parentEntry, _, parentStatus := wfs.maybeLoadEntry(dirFullPath)
|
||||
parentEntry, _, parentStatus := wfs.maybeLoadEntry(ctx, dirFullPath)
|
||||
if parentStatus != fuse.OK {
|
||||
return 0, nil, parentStatus
|
||||
}
|
||||
@@ -367,7 +375,7 @@ func (wfs *WFS) createRegularFile(dirFullPath util.FullPath, name string, mode u
|
||||
|
||||
entryFullPath := dirFullPath.Child(name)
|
||||
if !skipExistenceCheck {
|
||||
if _, _, status := wfs.maybeLoadEntry(entryFullPath); status == fuse.OK {
|
||||
if _, _, status := wfs.maybeLoadEntry(ctx, entryFullPath); status == fuse.OK {
|
||||
return 0, nil, fuse.Status(syscall.EEXIST)
|
||||
} else if status != fuse.ENOENT {
|
||||
return 0, nil, status
|
||||
|
||||
@@ -528,7 +528,7 @@ func TestAcquireHandleHonorsDefaultPermissions(t *testing.T) {
|
||||
}
|
||||
inode := wfs.inodeToPath.Lookup(util.FullPath("/secret.txt"), entry.Attributes.Crtime, false, false, entry.Attributes.Inode, true)
|
||||
|
||||
fh, status := wfs.AcquireHandle(inode, syscall.O_RDONLY, 999, 999)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, syscall.O_RDONLY, 999, 999)
|
||||
if status != tc.want {
|
||||
t.Fatalf("AcquireHandle status = %v, want %v", status, tc.want)
|
||||
}
|
||||
@@ -549,7 +549,7 @@ func TestCreateRegularFileValidatesParentUnderDefaultPermissions(t *testing.T) {
|
||||
wfs, _ := newCreateTestWFS(t)
|
||||
wfs.option.DefaultPermissions = true
|
||||
|
||||
if _, _, code := wfs.createRegularFile(util.FullPath("/ghost"), "f.txt", 0o644, 99, 100, 0, false, false); code == fuse.OK {
|
||||
if _, _, code := wfs.createRegularFile(context.Background(), util.FullPath("/ghost"), "f.txt", 0o644, 99, 100, 0, false, false); code == fuse.OK {
|
||||
t.Fatal("createRegularFile returned OK for a missing parent directory")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,17 +47,8 @@ func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse
|
||||
fhActiveLock := fh.wfs.fhLockTable.AcquireLock("Read", fh.fh, util.SharedLock)
|
||||
defer fh.wfs.fhLockTable.ReleaseLock(fh.fh, fhActiveLock)
|
||||
|
||||
// Create a context that will be cancelled when the cancel channel receives a signal
|
||||
ctx, cancelFunc := context.WithCancel(context.Background())
|
||||
defer cancelFunc() // Ensure cleanup
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case <-cancel:
|
||||
cancelFunc()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
|
||||
offset := int64(in.Offset)
|
||||
totalRead, err := readDataByFileHandleWithContext(ctx, buff, fh, offset)
|
||||
|
||||
@@ -238,7 +238,10 @@ func (wfs *WFS) flushMetadataToFiler(ctx context.Context, fh *FileHandle, dir, n
|
||||
|
||||
manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.GetChunks())
|
||||
|
||||
chunks, _ := filer.CompactFileChunks(context.Background(), wfs.LookupFn(), nonManifestChunks)
|
||||
chunks, _, err := filer.CompactFileChunks(context.Background(), wfs.LookupFn(), nonManifestChunks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compact chunks of %s: %w", fileFullPath, err)
|
||||
}
|
||||
|
||||
if mergedChunks, mergeErr := wfs.maybeMergeChunks(fileFullPath, chunks, manifestChunks); mergeErr != nil {
|
||||
glog.V(0).Infof("maybeMergeChunks %s: %v", fileFullPath, mergeErr)
|
||||
|
||||
@@ -245,7 +245,10 @@ func TestCompactThenMergeCondition(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
compacted, _ := filer.CompactFileChunks(context.Background(), nil, tt.chunks)
|
||||
compacted, _, err := filer.CompactFileChunks(context.Background(), nil, tt.chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, merge := shouldMergeChunks(compacted, nil)
|
||||
if merge != tt.wantMerge {
|
||||
var total uint64
|
||||
@@ -294,7 +297,10 @@ func TestRandomWritesBloatDetection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
compacted, _ := filer.CompactFileChunks(context.Background(), nil, chunks)
|
||||
compacted, _, err := filer.CompactFileChunks(context.Background(), nil, chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
totalCompacted, reportedFileSize, merge := shouldMergeChunks(compacted, nil)
|
||||
|
||||
// Sanity: condition must be consistent with its own inputs.
|
||||
@@ -337,7 +343,10 @@ func TestFlushCycleManifestAccumulation(t *testing.T) {
|
||||
|
||||
// --- flush pipeline (mirrors flushMetadataToFiler) ---
|
||||
manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entryChunks)
|
||||
compacted, _ := filer.CompactFileChunks(context.Background(), nil, nonManifestChunks)
|
||||
compacted, _, err := filer.CompactFileChunks(context.Background(), nil, nonManifestChunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, _, merge := shouldMergeChunks(compacted, manifestChunks)
|
||||
if merge {
|
||||
@@ -406,9 +415,18 @@ func TestVisibleContentPreservedAfterCompact(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
origViews := filer.ViewFromChunks(context.Background(), nil, chunks, 0, math.MaxInt64)
|
||||
compacted, _ := filer.CompactFileChunks(context.Background(), nil, chunks)
|
||||
compViews := filer.ViewFromChunks(context.Background(), nil, compacted, 0, math.MaxInt64)
|
||||
origViews, err := filer.ViewFromChunks(context.Background(), nil, chunks, 0, math.MaxInt64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compacted, _, err := filer.CompactFileChunks(context.Background(), nil, chunks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compViews, err := filer.ViewFromChunks(context.Background(), nil, compacted, 0, math.MaxInt64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Collect all (offset, size) pairs from views.
|
||||
type viewKey struct {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package mount
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"syscall"
|
||||
|
||||
@@ -10,12 +11,14 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) {
|
||||
func (wfs *WFS) AcquireHandle(ctx context.Context, inode uint64, flags, uid, gid uint32) (fileHandle *FileHandle, status fuse.Status) {
|
||||
// If there is an in-flight async flush for this inode, wait for it to
|
||||
// complete before reopening. Otherwise the new handle would be built
|
||||
// from pre-close filer metadata and its next flush could overwrite the
|
||||
// data that was just written asynchronously.
|
||||
wfs.waitForPendingAsyncFlush(inode)
|
||||
if !wfs.waitForPendingAsyncFlush(ctx, inode) {
|
||||
return nil, fuse.EINTR
|
||||
}
|
||||
|
||||
path, pathStatus := wfs.inodeToPath.GetPath(inode)
|
||||
if pathStatus != fuse.OK {
|
||||
@@ -38,7 +41,7 @@ func (wfs *WFS) AcquireHandle(inode uint64, flags, uid, gid uint32) (fileHandle
|
||||
})
|
||||
entryVersionTsNs = entryVersion{tsNs: existingFh.entryVersionTsNs.Load(), signature: existingFh.entryVersionSignature.Load()}
|
||||
} else {
|
||||
entry, entryVersionTsNs, status = wfs.maybeLoadEntry(path)
|
||||
entry, entryVersionTsNs, status = wfs.maybeLoadEntry(ctx, path)
|
||||
if status != fuse.OK {
|
||||
return nil, status
|
||||
}
|
||||
@@ -125,14 +128,20 @@ func (wfs *WFS) ReleaseHandle(handleId FileHandleId) {
|
||||
// waitForPendingAsyncFlush blocks until any in-flight async flush for
|
||||
// the given inode completes. Called from AcquireHandle before building
|
||||
// new handle state, so the filer metadata reflects the flushed data.
|
||||
func (wfs *WFS) waitForPendingAsyncFlush(inode uint64) {
|
||||
func (wfs *WFS) waitForPendingAsyncFlush(ctx context.Context, inode uint64) bool {
|
||||
wfs.pendingAsyncFlushMu.Lock()
|
||||
done, found := wfs.pendingAsyncFlush[inode]
|
||||
wfs.pendingAsyncFlushMu.Unlock()
|
||||
|
||||
if found {
|
||||
glog.V(3).Infof("waitForPendingAsyncFlush: waiting for inode %d", inode)
|
||||
<-done
|
||||
if !found {
|
||||
return true
|
||||
}
|
||||
glog.V(3).Infof("waitForPendingAsyncFlush: waiting for inode %d", inode)
|
||||
select {
|
||||
case <-done:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -84,7 +84,6 @@ type fakeFilerServer struct {
|
||||
lookupCalls atomic.Int32
|
||||
lookupStarted chan struct{} // closed when the first lookup arrives
|
||||
lookupGate chan struct{} // first lookup waits here when non-nil
|
||||
createCalls atomic.Int32
|
||||
}
|
||||
|
||||
func (s *fakeFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
@@ -126,7 +125,6 @@ func (s *fakeFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream f
|
||||
}
|
||||
|
||||
func (s *fakeFilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
|
||||
s.createCalls.Add(1)
|
||||
return &filer_pb.CreateEntryResponse{
|
||||
MetadataEvent: &filer_pb.SubscribeMetadataResponse{
|
||||
Directory: req.Directory,
|
||||
@@ -501,7 +499,7 @@ func TestQueuedEventDoesNotRollBackHandleOpenedAfterEnqueue(t *testing.T) {
|
||||
// Open now: the lookup reaches the filer, which serves the newer
|
||||
// size-200 state versioned at log position 2000.
|
||||
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("AcquireHandle status = %v, want OK", status)
|
||||
}
|
||||
@@ -565,7 +563,7 @@ func TestAbortedBuildEventDoesNotRollBackLaterOpen(t *testing.T) {
|
||||
}
|
||||
|
||||
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("AcquireHandle status = %v, want OK", status)
|
||||
}
|
||||
@@ -628,7 +626,7 @@ func TestEventDuringOpenLookupIsFenced(t *testing.T) {
|
||||
}
|
||||
opened := make(chan openResult, 1)
|
||||
go func() {
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
opened <- openResult{fh, status}
|
||||
}()
|
||||
|
||||
@@ -757,7 +755,7 @@ func TestLocalAckDoesNotFenceUnrelatedDelayedEvents(t *testing.T) {
|
||||
// Open the file: the cached-store read must not claim position 2000 —
|
||||
// the local ack versioned only its own path.
|
||||
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("AcquireHandle status = %v, want OK", status)
|
||||
}
|
||||
@@ -796,14 +794,14 @@ func TestSlowerConcurrentOpenDoesNotOverwriteNewerHandle(t *testing.T) {
|
||||
}
|
||||
slow := make(chan openResult, 1)
|
||||
go func() {
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
slow <- openResult{fh, status}
|
||||
}()
|
||||
<-fake.lookupStarted
|
||||
|
||||
// The faster opener completes with newer state while the slow lookup is
|
||||
// still in flight.
|
||||
fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fastFh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("fast AcquireHandle status = %v, want OK", status)
|
||||
}
|
||||
@@ -927,12 +925,12 @@ func TestSlowerOpenRejectedWhenHandleDirtyOrLookupUnversioned(t *testing.T) {
|
||||
}
|
||||
slow := make(chan openResult, 1)
|
||||
go func() {
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
slow <- openResult{fh, status}
|
||||
}()
|
||||
<-fake.lookupStarted
|
||||
|
||||
fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fastFh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("fast AcquireHandle status = %v, want OK", status)
|
||||
}
|
||||
@@ -971,12 +969,12 @@ func TestUnversionedSlowerOpenDoesNotOverwrite(t *testing.T) {
|
||||
}
|
||||
slow := make(chan openResult, 1)
|
||||
go func() {
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
slow <- openResult{fh, status}
|
||||
}()
|
||||
<-fake.lookupStarted
|
||||
|
||||
fastFh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fastFh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("fast AcquireHandle status = %v, want OK", status)
|
||||
}
|
||||
@@ -1407,7 +1405,7 @@ func TestEmptyListingTrailerSnapshotSetsAbsenceFloor(t *testing.T) {
|
||||
startFakeFiler(t, wfs, &fakeFilerServer{listSnapshotTrailerTsNs: 4000})
|
||||
|
||||
wfs.inodeToPath.Lookup(util.FullPath("/dir"), time.Now().Unix(), true, false, 0, false)
|
||||
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, util.FullPath("/dir"), 0); err != nil {
|
||||
if err := meta_cache.EnsureVisited(context.Background(), wfs.metaCache, wfs, util.FullPath("/dir"), 0); err != nil {
|
||||
t.Fatalf("EnsureVisited: %v", err)
|
||||
}
|
||||
|
||||
@@ -1822,7 +1820,7 @@ func TestEventFromAnotherFilerIsNotFencedByForeignClock(t *testing.T) {
|
||||
startFakeFiler(t, wfs, &fakeFilerServer{lookupSize: 200, lookupLogTsNs: 5000, lookupSignature: 11})
|
||||
|
||||
inode := wfs.inodeToPath.Lookup(util.FullPath("/dir/file"), time.Now().Unix(), false, false, 0, false)
|
||||
fh, status := wfs.AcquireHandle(inode, 0, 0, 0)
|
||||
fh, status := wfs.AcquireHandle(context.Background(), inode, 0, 0, 0)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("AcquireHandle status = %v, want OK", status)
|
||||
}
|
||||
|
||||
@@ -57,7 +57,9 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
// probe can move the two apart even for an entry that does carry one.
|
||||
sourceInode := in.Oldnodeid
|
||||
|
||||
oldEntry, _, status := wfs.maybeLoadEntry(oldEntryPath)
|
||||
ctx, cancelFunc := ctxFromCancel(cancel)
|
||||
defer cancelFunc()
|
||||
oldEntry, _, status := wfs.maybeLoadEntry(ctx, oldEntryPath)
|
||||
if status != fuse.OK {
|
||||
return status
|
||||
}
|
||||
@@ -89,7 +91,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
// Do not fall back to the pre-lock snapshot: if every alias
|
||||
// was deleted while we waited, abort instead of deriving the
|
||||
// next counter from a stale entry.
|
||||
fresh, _, freshStatus := wfs.maybeLoadEntry(oldEntryPath)
|
||||
fresh, _, freshStatus := wfs.maybeLoadEntry(ctx, oldEntryPath)
|
||||
if freshStatus != fuse.OK {
|
||||
return freshStatus
|
||||
}
|
||||
@@ -137,21 +139,20 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
// apply changes to the filer, and also apply to local metaCache
|
||||
wfs.mapPbIdFromLocalToFiler(request.Entry)
|
||||
|
||||
ctx := context.Background()
|
||||
updateResp, err := wfs.streamUpdateEntry(ctx, updateOldEntryRequest)
|
||||
updateResp, err := wfs.streamUpdateEntry(context.Background(), updateOldEntryRequest)
|
||||
if err == nil {
|
||||
updateEvent := updateResp.GetMetadataEvent()
|
||||
if updateEvent == nil {
|
||||
updateEvent = metadataUpdateEvent(oldParentPath, updateOldEntryRequest.Entry)
|
||||
}
|
||||
if applyErr := wfs.applyLocalMetadataEvent(ctx, updateEvent); applyErr != nil {
|
||||
if applyErr := wfs.applyLocalMetadataEvent(context.Background(), updateEvent); applyErr != nil {
|
||||
glog.Warningf("link %s: best-effort metadata apply failed: %v", oldEntryPath, applyErr)
|
||||
wfs.inodeToPath.InvalidateChildrenCache(util.FullPath(oldParentPath))
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
var createResp *filer_pb.CreateEntryResponse
|
||||
createResp, err = wfs.streamCreateEntry(ctx, request)
|
||||
createResp, err = wfs.streamCreateEntry(context.Background(), request)
|
||||
if err != nil {
|
||||
// Rollback: restore original HardLinkId/Counter on the source entry
|
||||
oldEntry.HardLinkId = origHardLinkId
|
||||
@@ -161,7 +162,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
Entry: oldEntry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
}
|
||||
if _, rollbackErr := wfs.streamUpdateEntry(ctx, rollbackReq); rollbackErr != nil {
|
||||
if _, rollbackErr := wfs.streamUpdateEntry(context.Background(), rollbackReq); rollbackErr != nil {
|
||||
glog.Warningf("link rollback %s: %v", oldEntryPath, rollbackErr)
|
||||
}
|
||||
} else {
|
||||
@@ -169,7 +170,7 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
if createEvent == nil {
|
||||
createEvent = metadataCreateEvent(string(newParentPath), request.Entry)
|
||||
}
|
||||
if applyErr := wfs.applyLocalMetadataEvent(ctx, createEvent); applyErr != nil {
|
||||
if applyErr := wfs.applyLocalMetadataEvent(context.Background(), createEvent); applyErr != nil {
|
||||
glog.Warningf("link %s: best-effort metadata apply failed: %v", newParentPath.Child(name), applyErr)
|
||||
wfs.inodeToPath.InvalidateChildrenCache(newParentPath)
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ func TestLinkReportsKernelNodeIdWhenEntryHasNoPersistedInode(t *testing.T) {
|
||||
if _, status := wfs.inodeToPath.GetPath(sourceInode); status != fuse.OK {
|
||||
t.Fatalf("GetPath(%d) = %v, want OK", sourceInode, status)
|
||||
}
|
||||
linked, _, status := wfs.maybeLoadEntry(sourcePath)
|
||||
linked, _, status := wfs.maybeLoadEntry(context.Background(), sourcePath)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("reload source: %v", status)
|
||||
}
|
||||
@@ -229,7 +229,7 @@ func TestLinkSyncsHardLinkSiblingsOnThirdLink(t *testing.T) {
|
||||
// any sibling returns the attributes of the last write to any of them,
|
||||
// synced or not. Feeding the loop an authoritative counter that no other
|
||||
// write has stored is what makes its work visible.
|
||||
authoritative, _, status := wfs.maybeLoadEntry(sourcePath)
|
||||
authoritative, _, status := wfs.maybeLoadEntry(context.Background(), sourcePath)
|
||||
if status != fuse.OK {
|
||||
t.Fatalf("reload source: %v", status)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package mount
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -136,7 +137,10 @@ func (wfs *WFS) flushFileMetadata(fh *FileHandle) error {
|
||||
manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(snapshotChunks)
|
||||
|
||||
// Compact chunks to remove fully overlapped ones
|
||||
compactedChunks, _ := filer.CompactFileChunks(context.Background(), wfs.LookupFn(), nonManifestChunks)
|
||||
compactedChunks, _, err := filer.CompactFileChunks(context.Background(), wfs.LookupFn(), nonManifestChunks)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compact chunks of %s: %w", fileFullPath, err)
|
||||
}
|
||||
|
||||
// Try to create manifest chunks for large files
|
||||
// no chunk deleter here: a failed fold reports the blobs it saved
|
||||
|
||||
@@ -94,7 +94,7 @@ func (wfs *WFS) posixLockKeyForInode(inode uint64) (string, bool) {
|
||||
if status != fuse.OK {
|
||||
return "", false
|
||||
}
|
||||
if entry, _, st := wfs.maybeLoadEntry(path); st == fuse.OK && entry != nil && len(entry.HardLinkId) > 0 {
|
||||
if entry, _, st := wfs.maybeLoadEntry(context.Background(), path); st == fuse.OK && entry != nil && len(entry.HardLinkId) > 0 {
|
||||
return posixLockKeyPrefix + "hl:" + hex.EncodeToString(entry.HardLinkId), true
|
||||
}
|
||||
return posixLockKeyPrefix + string(path), true
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user