mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-17 12:05:52 +00:00
Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d9d2dec9ac | ||
|
|
16110fd387 | ||
|
|
e6d6438cad | ||
|
|
d9cd505a96 | ||
|
|
81e220e979 | ||
|
|
4e63394302 | ||
|
|
0a59ec11d6 | ||
|
|
1c003d0e52 | ||
|
|
68109213a4 | ||
|
|
bc6f528d97 | ||
|
|
a76ed8dd7a | ||
|
|
5882fb4af0 | ||
|
|
29f9d0b805 | ||
|
|
912b027e6f | ||
|
|
eabffd9294 | ||
|
|
590624908b | ||
|
|
d45d8750e8 | ||
|
|
937d9af4a5 | ||
|
|
d5cff0a38f |
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.38.3
|
||||
uses: github/codeql-action/init@v4.38.2
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4.38.3
|
||||
uses: github/codeql-action/autobuild@v4.38.2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -49,4 +49,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4.38.3
|
||||
uses: github/codeql-action/analyze@v4.38.2
|
||||
|
||||
@@ -405,7 +405,7 @@ jobs:
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4.38.3
|
||||
uses: github/codeql-action/upload-sarif@v4.38.2
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
|
||||
@@ -460,7 +460,7 @@ jobs:
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4.38.3
|
||||
uses: github/codeql-action/upload-sarif@v4.38.2
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
check-latest: true
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@v2.9.0
|
||||
uses: helm/chart-testing-action@v2.8.0
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
@@ -1869,7 +1869,7 @@ jobs:
|
||||
echo "IMAGE_TAG=$TAG" >> $GITHUB_ENV
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.15.1
|
||||
uses: helm/kind-action@v1.15.0
|
||||
|
||||
- name: Verify volume resize hook security contexts
|
||||
run: |
|
||||
|
||||
@@ -23,8 +23,8 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.18.0
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.17.0
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
github.com/go-zookeeper/zk v1.0.4 // indirect
|
||||
github.com/golang/protobuf v1.5.4
|
||||
@@ -91,7 +91,7 @@ require (
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.46.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
|
||||
golang.org/x/image v0.46.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
@@ -116,13 +116,13 @@ require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.2
|
||||
github.com/a-h/templ v0.3.1070
|
||||
github.com/a-h/templ v0.3.1020
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.2
|
||||
github.com/apache/iceberg-go v0.7.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.3
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
|
||||
@@ -157,7 +157,7 @@ require (
|
||||
github.com/twmb/avro v1.9.0
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1
|
||||
go.uber.org/atomic v1.12.0
|
||||
golang.org/x/sync v0.24.0
|
||||
@@ -172,7 +172,6 @@ require (
|
||||
cloud.google.com/go/pubsub/v2 v2.6.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v3 v3.4.1 // indirect
|
||||
@@ -294,10 +293,10 @@ require (
|
||||
cloud.google.com/go/iam v1.12.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.30.0 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
|
||||
@@ -314,7 +313,7 @@ require (
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.12.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.6 // indirect
|
||||
github.com/andybalholm/brotli v1.2.3 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
@@ -333,7 +332,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1
|
||||
github.com/aws/smithy-go v1.28.5
|
||||
github.com/aws/smithy-go v1.28.1
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
|
||||
@@ -34,8 +34,8 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2 h1:utpeoEeZjd+A8J41zvoLsOOrqXHhX1Kx/X/tCW9dEYQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs=
|
||||
@@ -48,12 +48,10 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2 h1:tQJdgQjbKKffEiMOKJmcQPkY01eFHRqWIZVefqP44nI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2/go.mod h1:kTkpJchXAdO7LGkKZP97a+Ou8TgaF4NT6MHwOIy5uSQ=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 h1:X/bTwe0qpvmvPPutc+LWo7T4cKMqDP9JHP4k+Co9ZB4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0/go.mod h1:+kX7ahyFelQ42F3RScM2CFgHJljp0phc/FieKLYVpFI=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
|
||||
@@ -120,8 +118,8 @@ github.com/ThreeDotsLabs/watermill v1.5.2 h1:0ES33Eq1jEsP/pWvtE4n8bE0bs+9Jq7boT7
|
||||
github.com/ThreeDotsLabs/watermill v1.5.2/go.mod h1:i9/968UriGphWfEbfMuYSD1qFbYRjb0mE0r+rV0FPp4=
|
||||
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb h1:wumPkzt4zaxO4rHPBrjDK8iZMR41C1qs7njNqlacwQg=
|
||||
github.com/TomiHiltunen/geohash-golang v0.0.0-20150112065804-b3e4e625abfb/go.mod h1:QiYsIBRQEO+Z4Rz7GoI+dsHVneZNONvhczuA+llOZNM=
|
||||
github.com/a-h/templ v0.3.1070 h1:3iCOXHXqFXa99OQzgkUOLJbNzv/QjFGLwKZkBAg+ADA=
|
||||
github.com/a-h/templ v0.3.1070/go.mod h1:iIqB3g2RHBe/WlnPwa+Bvhh1TKXHWLRnBD9NWHOLm78=
|
||||
github.com/a-h/templ v0.3.1020 h1:ypAT/L5ySWEnZ6Zft/5yfoWXYYkhFNvEFOeeqecg4tw=
|
||||
github.com/a-h/templ v0.3.1020/go.mod h1:A2DlK61v+K+NRoGnhmYbNYVmtYHcFO5/AisMvBdDxTM=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 h1:iLDOF0rdGTrol/q8OfPIIs5kLD8XvA2q75o6Uq/tgak=
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0/go.mod h1:DrEWcQJjz7t5iF2duaiyhg4jyoF0kxOD6LtECNGkZ/Q=
|
||||
github.com/aalpar/deheap v1.1.2 h1:MABHLcnjqsffb8GLkUFDigqpBBxOMz0DoKM9QfELeTw=
|
||||
@@ -144,8 +142,8 @@ github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktp
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/anchore/go-lzo v0.1.1 h1:IwL/fvkdtlIrYIXck6WxZ3nb8WjjHziYYmGxlooyOnM=
|
||||
github.com/anchore/go-lzo v0.1.1/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk=
|
||||
github.com/andybalholm/brotli v1.2.6 h1:ftYnfj6usCp+UGV5kSJ3+chpMQgU+gJf/AxsUQ52REI=
|
||||
github.com/andybalholm/brotli v1.2.6/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ=
|
||||
github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=
|
||||
github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
@@ -173,8 +171,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
|
||||
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
|
||||
@@ -215,8 +213,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/S
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM=
|
||||
github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
|
||||
github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
|
||||
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -464,8 +462,8 @@ github.com/fsnotify/fsevents v0.2.0 h1:BRlvlqjvNTfogHfeBOFvSC9N0Ddy+wzQCQukyoD7o
|
||||
github.com/fsnotify/fsevents v0.2.0/go.mod h1:B3eEk39i4hz8y1zaWS/wPrAP4O6wkIl7HQwKBr1qH/w=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw=
|
||||
github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
|
||||
@@ -529,8 +527,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
|
||||
github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.18.0 h1:DDM09PEhmNTf/vziolf2x6391y1hHxF5vQ4ytHl9xQc=
|
||||
github.com/go-redsync/redsync/v4 v4.18.0/go.mod h1:yuoqcQ55FS1VypSeVTMqFsJqTdWpEH5eA4MQGKJ9c4M=
|
||||
github.com/go-redsync/redsync/v4 v4.17.0 h1:FFJ+uxZs44y4Sq10//IFKic9T94AYl+u3Sog98AHzBo=
|
||||
github.com/go-redsync/redsync/v4 v4.17.0/go.mod h1:CKVA6qwT07S/916i+Yd9h1/8YFQhCCpPYTQhvvYytJo=
|
||||
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
|
||||
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
|
||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||
@@ -1065,10 +1063,10 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0=
|
||||
github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4=
|
||||
github.com/redis/rueidis v1.0.77 h1:ZR41bgJcm7oRFb3aSDPrRhC0eonDSrPzjvvZvHIlNjM=
|
||||
github.com/redis/rueidis v1.0.77/go.mod h1:L8mnCQJJaSNL6I4pIR6Rz732HTGS9vmuXm0yT9dRvjo=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.77 h1:S1xiLQsFv8XQ46e0LelC8ihGqbDzVRocXXZLzVAeh64=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.77/go.mod h1:aXCvRVUTvTHZY0owHuz9JjveO1hyuzyDWrpJh8/E3Ds=
|
||||
github.com/redis/rueidis v1.0.76 h1:RdDWuvlYBSp+bTrBvaXqJnNEL3VVzsnjo+0psPFgLc4=
|
||||
github.com/redis/rueidis v1.0.76/go.mod h1:UsfHPSbomB6QAVMk4iiFkzRy0nh9o7scDGa+SitvBY4=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.76 h1:7LikbiqCQqCsZXeZ+akgZMnjIV/J0VHih9PIX4gGZC4=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.76/go.mod h1:UatQQLVj4QMIsZtpvRWY28qm6r2d72idhcS+C/RM+Zg=
|
||||
github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
|
||||
github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
|
||||
github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrdo=
|
||||
@@ -1319,8 +1317,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 h1:7V
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3 h1:A48PcmIETWwbEMIGwndv52uME6J1eJ28MBgF7HVR74o=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3/go.mod h1:oRGF+67GWLbtNzqGygsPIUsjXmsRXD1KS5m3o5ppm2k=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1 h1:ZMByYe53zZ9R1o23fxJsbNhjzKFQkzpXm6uGj8gWHXQ=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1/go.mod h1:dJXJ1u00IqO8Vsph8fWmYj8b1E7jyphnvJPqA69emBY=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1 h1:T+fB2ZDHpYIGC7DWjK+rzZHLoexBF0zG/n3Q9DGNskY=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1/go.mod h1:dJXJ1u00IqO8Vsph8fWmYj8b1E7jyphnvJPqA69emBY=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.5 h1:qYIeU1N6a92FaDA0+/vR55l5IzRF50MeLZwUWwvy2xQ=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.5/go.mod h1:DlSltVQKEdXJotaAYwoHzMGDF9EJw5S3hg3LXe/5yYw=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.2 h1:CZFawwHjLcjsxcl/g0dGIXbI2r41okPGLCydF4qTHcc=
|
||||
@@ -1449,8 +1447,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g=
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
|
||||
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
|
||||
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
||||
+1059
-1080
File diff suppressed because it is too large
Load Diff
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 54 KiB |
Generated
+8
-15
@@ -234,9 +234,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.18.1"
|
||||
version = "1.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
|
||||
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"zeroize",
|
||||
@@ -244,15 +244,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.45.0"
|
||||
version = "0.39.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
|
||||
checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
"dunce",
|
||||
"fs_extra",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2678,12 +2677,6 @@ dependencies = [
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.18.1"
|
||||
@@ -3396,9 +3389,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
version = "0.23.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -3443,9 +3436,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
version = "0.103.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
|
||||
@@ -114,7 +114,7 @@ func TestS3Integration(t *testing.T) {
|
||||
// findAvailablePort finds an available port by binding to port 0
|
||||
|
||||
// startMiniCluster starts a weed mini instance directly without exec.
|
||||
// Extra flags can be appended via extraArgs.
|
||||
// Extra flags (e.g. "-s3.allowDeleteBucketNotEmpty=false") can be appended via extraArgs.
|
||||
func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) {
|
||||
// Allocate non-colliding ports (including gRPC offsets) for weed mini.
|
||||
ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
|
||||
@@ -872,7 +872,7 @@ func TestS3DeleteBucketNotEmpty(t *testing.T) {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
cluster, err := startMiniCluster(t, "-s3.allowDeleteBucketNotEmpty=false")
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
|
||||
@@ -228,8 +228,8 @@ test-with-server: start-server
|
||||
# Test with different configurations
|
||||
test-versioning-with-configs: check-deps
|
||||
@echo "Testing with different S3 configurations..."
|
||||
@echo "Testing with delete bucket not empty allowed..."
|
||||
@$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
|
||||
@echo "Testing with empty folder allowed..."
|
||||
@$(WEED_BINARY) mini -s3.port=$(S3_PORT) > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
|
||||
@sleep 5
|
||||
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
|
||||
@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
|
||||
|
||||
@@ -3,7 +3,6 @@ package sftp
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
@@ -603,32 +602,6 @@ func TestFileContent(t *testing.T) {
|
||||
sftpClient.Remove("/binary.bin")
|
||||
})
|
||||
|
||||
t.Run("WriteAtOffsetKeepsExistingBytes", func(t *testing.T) {
|
||||
file, err := sftpClient.Create("/resume.txt")
|
||||
require.NoError(t, err)
|
||||
_, err = file.Write([]byte("hello world"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, file.Close())
|
||||
|
||||
// Reopen without O_TRUNC and write past the start, as a resumed upload does
|
||||
file, err = sftpClient.OpenFile("/resume.txt", os.O_WRONLY)
|
||||
require.NoError(t, err)
|
||||
_, err = file.WriteAt([]byte("WORLD!"), 6)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, file.Close())
|
||||
|
||||
readFile, err := sftpClient.Open("/resume.txt")
|
||||
require.NoError(t, err)
|
||||
content, err := io.ReadAll(readFile)
|
||||
require.NoError(t, err)
|
||||
readFile.Close()
|
||||
|
||||
require.Equal(t, "hello WORLD!", string(content))
|
||||
|
||||
// Clean up
|
||||
sftpClient.Remove("/resume.txt")
|
||||
})
|
||||
|
||||
t.Run("EmptyFile", func(t *testing.T) {
|
||||
file, err := sftpClient.Create("/empty.txt")
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// JobReplicaHosts are the volume-replica hosts a job will read from.
|
||||
// Used so executor pick prefers a worker on one of those machines.
|
||||
// Sources with ShardIds are EC-cleanup rows, not encode replicas.
|
||||
func JobReplicaHosts(job *plugin_pb.JobSpec) []string {
|
||||
if job == nil {
|
||||
return nil
|
||||
}
|
||||
var addrs []string
|
||||
addrs = append(addrs, pluginworker.ReadStringListConfig(job.Parameters, "replica_servers")...)
|
||||
if src := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", "")); src != "" {
|
||||
addrs = append(addrs, src)
|
||||
}
|
||||
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
|
||||
params := &worker_pb.TaskParams{}
|
||||
if err := proto.Unmarshal(payload, params); err == nil {
|
||||
for _, s := range params.GetSources() {
|
||||
if s == nil || s.Node == "" || len(s.ShardIds) > 0 {
|
||||
continue
|
||||
}
|
||||
addrs = append(addrs, s.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
return util.HostKeysFromAddresses(addrs)
|
||||
}
|
||||
|
||||
// partitionByPreferredHost splits candidates into (host matches, others).
|
||||
// No preferred hosts, or no matches, returns (nil, all) so callers keep
|
||||
// the existing slots+round-robin path.
|
||||
func partitionByPreferredHost(candidates []*WorkerSession, preferredHosts []string) (pref, rest []*WorkerSession) {
|
||||
want := util.HostKeysFromAddresses(preferredHosts)
|
||||
if len(want) == 0 || len(candidates) == 0 {
|
||||
return nil, candidates
|
||||
}
|
||||
set := make(map[string]struct{}, len(want))
|
||||
for _, h := range want {
|
||||
set[h] = struct{}{}
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if c == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := set[util.HostKey(c.Address)]; ok {
|
||||
pref = append(pref, c)
|
||||
continue
|
||||
}
|
||||
rest = append(rest, c)
|
||||
}
|
||||
if len(pref) == 0 {
|
||||
return nil, candidates
|
||||
}
|
||||
return pref, rest
|
||||
}
|
||||
@@ -615,7 +615,7 @@ func (r *Plugin) ExecuteJob(
|
||||
releaseGate := r.acquireDetectionExecutionGate(jobType, true)
|
||||
defer releaseGate()
|
||||
|
||||
executor, err := r.registry.PickExecutor(jobType, JobReplicaHosts(job)...)
|
||||
executor, err := r.registry.PickExecutor(jobType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -887,37 +887,6 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
|
||||
})
|
||||
}
|
||||
|
||||
// bindQueuedJobWorker records the home worker while the job stays pending/queued.
|
||||
func (r *Plugin) bindQueuedJobWorker(job *plugin_pb.JobSpec, workerID string) {
|
||||
if job == nil || strings.TrimSpace(job.JobId) == "" || strings.TrimSpace(workerID) == "" {
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
r.jobsMu.Lock()
|
||||
tracked := r.jobs[job.JobId]
|
||||
if tracked == nil {
|
||||
r.jobsMu.Unlock()
|
||||
return
|
||||
}
|
||||
if isTerminalTrackedJobState(tracked.State) {
|
||||
r.jobsMu.Unlock()
|
||||
return
|
||||
}
|
||||
tracked.WorkerID = workerID
|
||||
tracked.Message = "queued on worker " + workerID
|
||||
tracked.UpdatedAt = timeToPtr(now)
|
||||
snap := cloneTrackedJob(*tracked)
|
||||
r.dirtyJobs = true
|
||||
r.jobsMu.Unlock()
|
||||
|
||||
r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
|
||||
detail.WorkerID = snap.WorkerID
|
||||
detail.Message = snap.Message
|
||||
detail.UpdatedAt = snap.UpdatedAt
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
|
||||
reason := "job canceled"
|
||||
if cause != nil {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -29,18 +28,12 @@ const (
|
||||
defaultScheduledMaxResults int32 = 1000
|
||||
defaultScheduledExecutionConcurrency = 1
|
||||
defaultScheduledPerWorkerConcurrency = 1
|
||||
// Hold a job on replica-host workers until their reservation load
|
||||
// reaches this; 0 disables the pin and spills immediately.
|
||||
defaultLocalQueuePin = 5
|
||||
// Wait this long after a home worker disappears before stealing
|
||||
// its queue; 0 steals immediately. Covers a reconnect blink.
|
||||
defaultOrphanRehomeGrace = 30 * time.Second
|
||||
maxScheduledExecutionConcurrency = 128
|
||||
defaultScheduledRetryBackoff = 5 * time.Second
|
||||
defaultClusterContextTimeout = 10 * time.Second
|
||||
defaultWaitingBacklogFloor = 8
|
||||
defaultWaitingBacklogMultiplier = 4
|
||||
maxEstimatedRuntimeCap = 8 * time.Hour
|
||||
maxScheduledExecutionConcurrency = 128
|
||||
defaultScheduledRetryBackoff = 5 * time.Second
|
||||
defaultClusterContextTimeout = 10 * time.Second
|
||||
defaultWaitingBacklogFloor = 8
|
||||
defaultWaitingBacklogMultiplier = 4
|
||||
maxEstimatedRuntimeCap = 8 * time.Hour
|
||||
// How long started jobs may drain past the run window close.
|
||||
scheduledExecutionDrainGrace = 30 * time.Minute
|
||||
)
|
||||
@@ -56,12 +49,6 @@ type schedulerPolicy struct {
|
||||
PerWorkerConcurrency int
|
||||
RetryLimit int
|
||||
ExecutorReserveBackoff time.Duration
|
||||
// LocalQueuePin is K: do not spill off replica-host workers while
|
||||
// min(load(pref)) < K. Zero is the old immediate-spill path.
|
||||
LocalQueuePin int
|
||||
// OrphanRehomeGrace is how long a missing home's queue is held
|
||||
// before steal-without-K. Zero steals as soon as the worker is gone.
|
||||
OrphanRehomeGrace time.Duration
|
||||
}
|
||||
|
||||
// laneSchedulerLoop is the main scheduling goroutine for a single lane.
|
||||
@@ -479,8 +466,6 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
|
||||
PerWorkerConcurrency: int(adminRuntime.PerWorkerExecutionConcurrency),
|
||||
RetryLimit: int(adminRuntime.RetryLimit),
|
||||
ExecutorReserveBackoff: 200 * time.Millisecond,
|
||||
LocalQueuePin: defaultLocalQueuePin,
|
||||
OrphanRehomeGrace: defaultOrphanRehomeGrace,
|
||||
}
|
||||
|
||||
if policy.DetectionInterval < r.schedulerTick {
|
||||
@@ -511,9 +496,6 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
|
||||
if policy.ExecutionTimeout < defaultScheduledExecutionTimeout {
|
||||
policy.ExecutionTimeout = defaultScheduledExecutionTimeout
|
||||
}
|
||||
if policy.LocalQueuePin < 0 {
|
||||
policy.LocalQueuePin = 0
|
||||
}
|
||||
|
||||
return policy, true, nil
|
||||
}
|
||||
@@ -831,22 +813,19 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
// whole dispatch lets manual shell operations interleave between jobs.
|
||||
jobsRequireLock := LaneRequiresLock(JobTypeLane(jobType))
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, len(proposals))
|
||||
jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
|
||||
for index, proposal := range proposals {
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
for _, job := range jobs {
|
||||
r.cancelQueuedJob(job, errSchedulerShutdown)
|
||||
}
|
||||
return 0, 0, len(jobs)
|
||||
default:
|
||||
}
|
||||
job := buildScheduledJobSpec(jobType, proposal, index)
|
||||
r.trackExecutionQueued(job)
|
||||
jobs = append(jobs, job)
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
close(jobQueue)
|
||||
return 0, 0, 0
|
||||
default:
|
||||
jobQueue <- job
|
||||
}
|
||||
}
|
||||
|
||||
queues := r.assignJobsToWorkerQueues(jobType, jobs, policy)
|
||||
close(jobQueue)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var statsMu sync.Mutex
|
||||
@@ -863,110 +842,130 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
|
||||
jobLoop:
|
||||
for job := range jobQueue {
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if queues.isEmpty() {
|
||||
return
|
||||
}
|
||||
|
||||
item, targetID := r.takeQueuedJob(queues, jobType, policy)
|
||||
if item == nil {
|
||||
if queues.isEmpty() {
|
||||
return
|
||||
}
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
queues.pushFront(item)
|
||||
return
|
||||
}
|
||||
|
||||
executor, release, reserveErr := r.reserveExecutorByID(jobType, policy, targetID)
|
||||
if reserveErr != nil {
|
||||
queues.pushFront(item)
|
||||
for {
|
||||
select {
|
||||
case <-r.shutdownCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var releaseJobLock func()
|
||||
if jobsRequireLock {
|
||||
var lockErr error
|
||||
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, item.job.JobId))
|
||||
if lockErr != nil {
|
||||
release()
|
||||
executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy)
|
||||
if reserveErr != nil {
|
||||
if ctx.Err() != nil {
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: item.job.JobId,
|
||||
JobType: item.job.JobType,
|
||||
JobType: jobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
|
||||
Message: fmt.Sprintf("scheduled execution reservation failed: %v", reserveErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
continue
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
|
||||
var releaseJobLock func()
|
||||
if jobsRequireLock {
|
||||
var lockErr error
|
||||
releaseJobLock, lockErr = r.acquireAdminLock(fmt.Sprintf("plugin scheduler %s job %s", jobType, job.JobId))
|
||||
if lockErr != nil {
|
||||
release()
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: job.JobId,
|
||||
JobType: job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution lock failed: %v", lockErr),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
// The lock may have been held by an operator for a
|
||||
// while; re-check the execution window.
|
||||
if ctx.Err() != nil {
|
||||
releaseJobLock()
|
||||
release()
|
||||
r.cancelQueuedJob(job, ctx.Err())
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
}
|
||||
|
||||
err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy)
|
||||
if releaseJobLock != nil {
|
||||
releaseJobLock()
|
||||
release()
|
||||
queues.pushFront(item)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
err := r.executeScheduledJobWithExecutor(ctx, executor, item.job, clusterContext, policy)
|
||||
if releaseJobLock != nil {
|
||||
releaseJobLock()
|
||||
}
|
||||
release()
|
||||
if errors.Is(err, errExecutorAtCapacity) {
|
||||
queues.pushFront(item)
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
r.cancelQueuedJob(item.job, err)
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
release()
|
||||
if errors.Is(err, errExecutorAtCapacity) {
|
||||
r.trackExecutionQueued(job)
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: item.job.JobId,
|
||||
JobType: item.job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution failed: %v", err),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||||
r.cancelQueuedJob(job, err)
|
||||
statsMu.Lock()
|
||||
canceledCount++
|
||||
statsMu.Unlock()
|
||||
continue jobLoop
|
||||
}
|
||||
statsMu.Lock()
|
||||
errorCount++
|
||||
statsMu.Unlock()
|
||||
r.appendActivity(JobActivity{
|
||||
JobID: job.JobId,
|
||||
JobType: job.JobType,
|
||||
Source: "admin_scheduler",
|
||||
Message: fmt.Sprintf("scheduled execution failed: %v", err),
|
||||
Stage: "failed",
|
||||
OccurredAt: timeToPtr(time.Now().UTC()),
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
statsMu.Lock()
|
||||
successCount++
|
||||
statsMu.Unlock()
|
||||
statsMu.Lock()
|
||||
successCount++
|
||||
statsMu.Unlock()
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -977,8 +976,8 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
if drainErr == nil {
|
||||
drainErr = errSchedulerShutdown
|
||||
}
|
||||
for _, item := range queues.drain() {
|
||||
r.cancelQueuedJob(item.job, drainErr)
|
||||
for job := range jobQueue {
|
||||
r.cancelQueuedJob(job, drainErr)
|
||||
canceledCount++
|
||||
}
|
||||
|
||||
@@ -989,7 +988,6 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
ctx context.Context,
|
||||
jobType string,
|
||||
policy schedulerPolicy,
|
||||
preferredHosts []string,
|
||||
) (*WorkerSession, func(), error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
@@ -1017,7 +1015,7 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
|
||||
}
|
||||
|
||||
executors, err := r.registry.ListExecutors(jobType, preferredHosts...)
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil {
|
||||
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
|
||||
if ctx.Err() != nil {
|
||||
@@ -1028,19 +1026,11 @@ func (r *Plugin) reserveScheduledExecutor(
|
||||
continue
|
||||
}
|
||||
|
||||
pref, rest := partitionByPreferredHost(executors, preferredHosts)
|
||||
lane := JobTypeLane(jobType)
|
||||
loadOf := func(e *WorkerSession) int { return r.executorReservationLoad(e, lane) }
|
||||
|
||||
if len(pref) > 0 {
|
||||
sortExecutorsByLoad(pref, loadOf)
|
||||
if executor, release, ok := r.tryReserveFirst(pref, jobType, policy); ok {
|
||||
return executor, release, nil
|
||||
for _, executor := range executors {
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
sortExecutorsByLoad(rest, loadOf)
|
||||
if executor, release, ok := r.tryReserveFirst(rest, jobType, policy); ok {
|
||||
return executor, release, nil
|
||||
}
|
||||
|
||||
@@ -1061,75 +1051,6 @@ func (r *Plugin) tryReserveExecutorCapacity(
|
||||
return r.tryReserveExecutorCapacityForLane(executor, jobType, policy, JobTypeLane(jobType))
|
||||
}
|
||||
|
||||
func (r *Plugin) tryReserveFirst(executors []*WorkerSession, jobType string, policy schedulerPolicy) (*WorkerSession, func(), bool) {
|
||||
for _, executor := range executors {
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if ok {
|
||||
return executor, release, true
|
||||
}
|
||||
}
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
func (r *Plugin) reserveExecutorByID(jobType string, policy schedulerPolicy, workerID string) (*WorkerSession, func(), error) {
|
||||
workerID = strings.TrimSpace(workerID)
|
||||
if workerID == "" {
|
||||
return nil, nil, fmt.Errorf("executor id is empty")
|
||||
}
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, executor := range executors {
|
||||
if executor == nil || executor.WorkerID != workerID {
|
||||
continue
|
||||
}
|
||||
release, ok := r.tryReserveExecutorCapacity(executor, jobType, policy)
|
||||
if !ok {
|
||||
return nil, nil, errExecutorAtCapacity
|
||||
}
|
||||
return executor, release, nil
|
||||
}
|
||||
return nil, nil, fmt.Errorf("executor %s is not available for %s", workerID, jobType)
|
||||
}
|
||||
|
||||
func (r *Plugin) executorReservationLoad(executor *WorkerSession, lane SchedulerLane) int {
|
||||
if executor == nil {
|
||||
return 0
|
||||
}
|
||||
load := 0
|
||||
if executor.Heartbeat != nil && executor.Heartbeat.ExecutionSlotsUsed > 0 {
|
||||
load = int(executor.Heartbeat.ExecutionSlotsUsed)
|
||||
}
|
||||
workerID := strings.TrimSpace(executor.WorkerID)
|
||||
if ls := r.lanes[lane]; ls != nil {
|
||||
ls.execMu.Lock()
|
||||
load += ls.execRes[workerID]
|
||||
ls.execMu.Unlock()
|
||||
return load
|
||||
}
|
||||
r.schedulerExecMu.Lock()
|
||||
load += r.schedulerExecReservations[workerID]
|
||||
r.schedulerExecMu.Unlock()
|
||||
return load
|
||||
}
|
||||
|
||||
func sortExecutorsByLoad(executors []*WorkerSession, load func(*WorkerSession) int) {
|
||||
sort.SliceStable(executors, func(i, j int) bool {
|
||||
if executors[i] == nil {
|
||||
return false
|
||||
}
|
||||
if executors[j] == nil {
|
||||
return true
|
||||
}
|
||||
li, lj := load(executors[i]), load(executors[j])
|
||||
if li != lj {
|
||||
return li < lj
|
||||
}
|
||||
return executors[i].WorkerID < executors[j].WorkerID
|
||||
})
|
||||
}
|
||||
|
||||
// tryReserveExecutorCapacityForLane reserves an execution slot on the
|
||||
// per-lane reservation pool so that lanes cannot starve each other.
|
||||
func (r *Plugin) tryReserveExecutorCapacityForLane(
|
||||
@@ -1656,8 +1577,6 @@ func (r *Plugin) DispatchProposals(
|
||||
ExecutionTimeout: defaultScheduledExecutionTimeout,
|
||||
RetryBackoff: defaultScheduledRetryBackoff,
|
||||
ExecutorReserveBackoff: 200 * time.Millisecond,
|
||||
LocalQueuePin: defaultLocalQueuePin,
|
||||
OrphanRehomeGrace: defaultOrphanRehomeGrace,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -61,12 +61,6 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
|
||||
if policy.JobTypeMaxRuntime != 30*time.Minute {
|
||||
t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
|
||||
}
|
||||
if policy.LocalQueuePin != defaultLocalQueuePin {
|
||||
t.Fatalf("unexpected local queue pin: got=%d", policy.LocalQueuePin)
|
||||
}
|
||||
if policy.OrphanRehomeGrace != defaultOrphanRehomeGrace {
|
||||
t.Fatalf("unexpected orphan rehome grace: got=%v", policy.OrphanRehomeGrace)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
|
||||
@@ -144,13 +138,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
|
||||
ExecutorReserveBackoff: time.Millisecond,
|
||||
}
|
||||
|
||||
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 2: %v", err)
|
||||
}
|
||||
@@ -272,7 +266,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
|
||||
|
||||
start := time.Now()
|
||||
pluginSvc.Shutdown()
|
||||
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy, nil)
|
||||
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
|
||||
if err == nil {
|
||||
t.Fatalf("expected reservation shutdown error")
|
||||
}
|
||||
@@ -303,7 +297,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
|
||||
ExecutorReserveBackoff: 5 * time.Millisecond,
|
||||
}
|
||||
|
||||
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve executor 1: %v", err)
|
||||
}
|
||||
@@ -314,7 +308,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
|
||||
}
|
||||
secondReserveCh := make(chan reserveResult, 1)
|
||||
go func() {
|
||||
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy, nil)
|
||||
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
|
||||
if release2 != nil {
|
||||
release2()
|
||||
}
|
||||
@@ -998,50 +992,3 @@ func TestLaneStatus_LifecycleNextDetectionSurvivesDefaultLanePrune(t *testing.T)
|
||||
status.NextDetectionAt, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveWaterfillsPreferredHosts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
ids := []string{"worker-a", "worker-b", "worker-c", "worker-d"}
|
||||
for _, id := range ids {
|
||||
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: id,
|
||||
Address: "10.0.0.1:9334",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 8,
|
||||
ExecutorReserveBackoff: time.Millisecond,
|
||||
LocalQueuePin: 5,
|
||||
}
|
||||
preferred := []string{"10.0.0.1:8080"}
|
||||
|
||||
seen := map[string]int{}
|
||||
var releases []func()
|
||||
defer func() {
|
||||
for _, release := range releases {
|
||||
release()
|
||||
}
|
||||
}()
|
||||
for i := 0; i < 4; i++ {
|
||||
exec, release, err := pluginSvc.reserveScheduledExecutor(context.Background(), "erasure_coding", policy, preferred)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve %d: %v", i, err)
|
||||
}
|
||||
releases = append(releases, release)
|
||||
seen[exec.WorkerID]++
|
||||
}
|
||||
if len(seen) != 4 {
|
||||
t.Fatalf("want one job per worker, got %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,24 +277,14 @@ func (r *Registry) PickDetector(jobType string) (*WorkerSession, error) {
|
||||
}
|
||||
|
||||
// PickExecutor picks one executor worker for a job type.
|
||||
// preferredHosts, when set, ranks workers whose Hello Address host matches a
|
||||
// replica/source host ahead of others (same-machine EC pull). Empty means the
|
||||
// previous slots+round-robin order.
|
||||
func (r *Registry) PickExecutor(jobType string, preferredHosts ...string) (*WorkerSession, error) {
|
||||
list, err := r.ListExecutors(jobType, preferredHosts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list[0], nil
|
||||
func (r *Registry) PickExecutor(jobType string) (*WorkerSession, error) {
|
||||
return r.pickByKind(jobType, false)
|
||||
}
|
||||
|
||||
// ListExecutors returns sorted executor candidates for one job type.
|
||||
// Ordering is by most available execution slots, then lexical worker ID.
|
||||
// The top tie group is rotated round-robin to prevent sticky assignment.
|
||||
// preferredHosts, when any worker matches, are listed first (then the rest);
|
||||
// rotation applies only inside the preferred group so a local worker is not
|
||||
// rotated away in favor of a remote one.
|
||||
func (r *Registry) ListExecutors(jobType string, preferredHosts ...string) ([]*WorkerSession, error) {
|
||||
func (r *Registry) ListExecutors(jobType string) ([]*WorkerSession, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -303,16 +293,8 @@ func (r *Registry) ListExecutors(jobType string, preferredHosts ...string) ([]*W
|
||||
return nil, fmt.Errorf("no executor worker available for job_type=%s", jobType)
|
||||
}
|
||||
|
||||
pref, rest := partitionByPreferredHost(candidates, preferredHosts)
|
||||
if len(pref) > 0 {
|
||||
sortByKind(pref, jobType, false)
|
||||
r.rotateTopCandidatesLocked(pref, jobType, false)
|
||||
sortByKind(rest, jobType, false)
|
||||
candidates = append(pref, rest...)
|
||||
} else {
|
||||
sortByKind(candidates, jobType, false)
|
||||
r.rotateTopCandidatesLocked(candidates, jobType, false)
|
||||
}
|
||||
sortByKind(candidates, jobType, false)
|
||||
r.rotateTopCandidatesLocked(candidates, jobType, false)
|
||||
|
||||
out := make([]*WorkerSession, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
|
||||
@@ -401,70 +401,3 @@ func TestRegistryHasCapableWorkerIgnoresNilCapability(t *testing.T) {
|
||||
t.Fatalf("expected HasCapableWorker to return false when capability has no detect or execute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestListExecutorsPrefersMatchingReplicaHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
r := NewRegistry()
|
||||
r.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: "worker-remote",
|
||||
Address: "10.87.0.4:33646",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
r.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: "worker-local",
|
||||
Address: "10.87.0.3:33646",
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 1},
|
||||
},
|
||||
})
|
||||
r.UpdateHeartbeat("worker-remote", &plugin_pb.WorkerHeartbeat{
|
||||
WorkerId: "worker-remote", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 8,
|
||||
})
|
||||
r.UpdateHeartbeat("worker-local", &plugin_pb.WorkerHeartbeat{
|
||||
WorkerId: "worker-local", ExecutionSlotsUsed: 0, ExecutionSlotsTotal: 1,
|
||||
})
|
||||
|
||||
// Without preference the emptier remote worker wins on slots.
|
||||
plain, err := r.ListExecutors("erasure_coding")
|
||||
if err != nil {
|
||||
t.Fatalf("ListExecutors: %v", err)
|
||||
}
|
||||
if plain[0].WorkerID != "worker-remote" {
|
||||
t.Fatalf("without preference want remote first, got %s", plain[0].WorkerID)
|
||||
}
|
||||
|
||||
// Replica on :8080 of the same host as worker-local.
|
||||
pref, err := r.ListExecutors("erasure_coding", "10.87.0.3:8080")
|
||||
if err != nil {
|
||||
t.Fatalf("ListExecutors preferred: %v", err)
|
||||
}
|
||||
if pref[0].WorkerID != "worker-local" {
|
||||
t.Fatalf("preferred host want worker-local first, got %s", pref[0].WorkerID)
|
||||
}
|
||||
if pref[1].WorkerID != "worker-remote" {
|
||||
t.Fatalf("remote should still be fallback, got %s", pref[1].WorkerID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobReplicaHostsFromSourcesAndReplicaServers(t *testing.T) {
|
||||
t.Parallel()
|
||||
job := &plugin_pb.JobSpec{
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "10.87.0.1:8080"},
|
||||
},
|
||||
"replica_servers": {
|
||||
Kind: &plugin_pb.ConfigValue_StringList{
|
||||
StringList: &plugin_pb.StringList{Values: []string{"10.87.0.1:8080", "10.87.0.2:8081"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
got := JobReplicaHosts(job)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,317 +0,0 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
// queuedJob is one scheduled spec parked on a home worker's admin-side queue.
|
||||
type queuedJob struct {
|
||||
job *plugin_pb.JobSpec
|
||||
home string
|
||||
}
|
||||
|
||||
// workerJobQueues holds per-worker FIFOs for one dispatch batch.
|
||||
// Home assignment is done once up front; run goroutines only pop/steal.
|
||||
// leftover is the homeless FIFO: replica hostnames with no currently
|
||||
// registered worker. Orphan grace does not apply here.
|
||||
type workerJobQueues struct {
|
||||
mu sync.Mutex
|
||||
q map[string][]*queuedJob
|
||||
leftover []*queuedJob
|
||||
remaining int
|
||||
missingSince map[string]time.Time
|
||||
homes map[string]struct{}
|
||||
}
|
||||
|
||||
func newWorkerJobQueues() *workerJobQueues {
|
||||
return &workerJobQueues{
|
||||
q: make(map[string][]*queuedJob),
|
||||
missingSince: make(map[string]time.Time),
|
||||
homes: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) hasHome(workerID string) bool {
|
||||
if qs == nil || workerID == "" {
|
||||
return false
|
||||
}
|
||||
_, ok := qs.homes[workerID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) push(home string, job *plugin_pb.JobSpec) {
|
||||
if qs == nil || job == nil {
|
||||
return
|
||||
}
|
||||
item := &queuedJob{job: job, home: home}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
qs.pushLocked(item, false)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) pushFront(item *queuedJob) {
|
||||
if qs == nil || item == nil {
|
||||
return
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
qs.pushLocked(item, true)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) pushLocked(item *queuedJob, front bool) {
|
||||
qs.remaining++
|
||||
if item.home == "" {
|
||||
if front {
|
||||
qs.leftover = append([]*queuedJob{item}, qs.leftover...)
|
||||
return
|
||||
}
|
||||
qs.leftover = append(qs.leftover, item)
|
||||
return
|
||||
}
|
||||
if front {
|
||||
qs.q[item.home] = append([]*queuedJob{item}, qs.q[item.home]...)
|
||||
return
|
||||
}
|
||||
qs.q[item.home] = append(qs.q[item.home], item)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) isEmpty() bool {
|
||||
if qs == nil {
|
||||
return true
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return qs.remaining == 0
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) queueLen(workerID string) int {
|
||||
if qs == nil {
|
||||
return 0
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return len(qs.q[workerID])
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) leftoverLen() int {
|
||||
if qs == nil {
|
||||
return 0
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
return len(qs.leftover)
|
||||
}
|
||||
|
||||
func (qs *workerJobQueues) drain() []*queuedJob {
|
||||
if qs == nil {
|
||||
return nil
|
||||
}
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
out := make([]*queuedJob, 0, qs.remaining)
|
||||
for id, qq := range qs.q {
|
||||
out = append(out, qq...)
|
||||
qs.q[id] = nil
|
||||
}
|
||||
out = append(out, qs.leftover...)
|
||||
qs.leftover = nil
|
||||
qs.remaining = 0
|
||||
return out
|
||||
}
|
||||
|
||||
func popFront(qq []*queuedJob) (*queuedJob, []*queuedJob) {
|
||||
item := qq[0]
|
||||
if len(qq) == 1 {
|
||||
return item, nil
|
||||
}
|
||||
return item, qq[1:]
|
||||
}
|
||||
|
||||
// assignJobsToWorkerQueues parks each job on the least-loaded replica-host
|
||||
// worker. A home exists only if a currently registered worker's -address
|
||||
// hostname matches a replica host. Otherwise the job is homeless: global
|
||||
// leftover FIFO, no fake home, no K, no orphan grace. Single-threaded.
|
||||
func (r *Plugin) assignJobsToWorkerQueues(
|
||||
jobType string,
|
||||
jobs []*plugin_pb.JobSpec,
|
||||
_ schedulerPolicy,
|
||||
) *workerJobQueues {
|
||||
qs := newWorkerJobQueues()
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
lane := JobTypeLane(jobType)
|
||||
queued := make(map[string]int, len(executors))
|
||||
|
||||
for _, job := range jobs {
|
||||
if job == nil {
|
||||
continue
|
||||
}
|
||||
if err != nil || len(executors) == 0 {
|
||||
qs.push("", job)
|
||||
continue
|
||||
}
|
||||
pref, _ := partitionByPreferredHost(executors, JobReplicaHosts(job))
|
||||
if len(pref) == 0 {
|
||||
// Never registered for that hostname: homeless, not an orphan.
|
||||
qs.push("", job)
|
||||
continue
|
||||
}
|
||||
for _, e := range pref {
|
||||
if e != nil && e.WorkerID != "" {
|
||||
qs.homes[e.WorkerID] = struct{}{}
|
||||
}
|
||||
}
|
||||
sortExecutorsByLoad(pref, func(e *WorkerSession) int {
|
||||
return r.executorReservationLoad(e, lane) + queued[e.WorkerID]
|
||||
})
|
||||
home := pref[0].WorkerID
|
||||
qs.push(home, job)
|
||||
queued[home]++
|
||||
r.bindQueuedJobWorker(job, home)
|
||||
}
|
||||
return qs
|
||||
}
|
||||
|
||||
// takeQueuedJob pops a job for a free worker.
|
||||
// Home worker (hostname matched a replica / has a home-queue identity):
|
||||
// 1. own home queue
|
||||
// 2. steal from another home if len>=K, or orphan past OrphanRehomeGrace
|
||||
// 3. homeless leftover FIFO
|
||||
// No-home worker (no matching replica, no home-queue identity):
|
||||
// 1. homeless leftover FIFO
|
||||
// 2. only if leftover empty: steal (K / orphan), same rules
|
||||
func (r *Plugin) takeQueuedJob(
|
||||
qs *workerJobQueues,
|
||||
jobType string,
|
||||
policy schedulerPolicy,
|
||||
) (item *queuedJob, targetID string) {
|
||||
if qs == nil {
|
||||
return nil, ""
|
||||
}
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil || len(executors) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
lane := JobTypeLane(jobType)
|
||||
pin := policy.LocalQueuePin
|
||||
defer func() {
|
||||
if item != nil && targetID != "" {
|
||||
r.bindQueuedJobWorker(item.job, targetID)
|
||||
}
|
||||
}()
|
||||
|
||||
qs.mu.Lock()
|
||||
defer qs.mu.Unlock()
|
||||
if qs.remaining == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
var bestHome *WorkerSession
|
||||
bestLoad := 0
|
||||
for _, exec := range executors {
|
||||
if len(qs.q[exec.WorkerID]) == 0 {
|
||||
continue
|
||||
}
|
||||
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
load := r.executorReservationLoad(exec, lane)
|
||||
if load >= limit {
|
||||
continue
|
||||
}
|
||||
if bestHome == nil || load < bestLoad || (load == bestLoad && exec.WorkerID < bestHome.WorkerID) {
|
||||
bestHome = exec
|
||||
bestLoad = load
|
||||
}
|
||||
}
|
||||
if bestHome != nil {
|
||||
id := bestHome.WorkerID
|
||||
item, qs.q[id] = popFront(qs.q[id])
|
||||
qs.remaining--
|
||||
return item, id
|
||||
}
|
||||
|
||||
var thief *WorkerSession
|
||||
for _, exec := range executors {
|
||||
limit := schedulerWorkerExecutionLimit(exec, jobType, policy)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
if r.executorReservationLoad(exec, lane) >= limit {
|
||||
continue
|
||||
}
|
||||
if thief == nil || exec.WorkerID < thief.WorkerID {
|
||||
thief = exec
|
||||
}
|
||||
}
|
||||
if thief == nil {
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
takeLeftover := func() (*queuedJob, string) {
|
||||
if len(qs.leftover) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
item, qs.leftover = popFront(qs.leftover)
|
||||
qs.remaining--
|
||||
return item, thief.WorkerID
|
||||
}
|
||||
if !qs.hasHome(thief.WorkerID) {
|
||||
if item, id := takeLeftover(); item != nil {
|
||||
return item, id
|
||||
}
|
||||
}
|
||||
|
||||
live := make(map[string]struct{}, len(executors))
|
||||
for _, exec := range executors {
|
||||
live[exec.WorkerID] = struct{}{}
|
||||
delete(qs.missingSince, exec.WorkerID)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
grace := policy.OrphanRehomeGrace
|
||||
victimID := ""
|
||||
victimLen := 0
|
||||
victimOrphan := false
|
||||
for id, qq := range qs.q {
|
||||
if len(qq) == 0 {
|
||||
delete(qs.missingSince, id)
|
||||
continue
|
||||
}
|
||||
_, isLive := live[id]
|
||||
orphan := !isLive
|
||||
if isLive {
|
||||
if pin > 0 && len(qq) < pin {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if _, ok := qs.missingSince[id]; !ok {
|
||||
qs.missingSince[id] = now
|
||||
}
|
||||
if grace > 0 && now.Sub(qs.missingSince[id]) < grace {
|
||||
continue
|
||||
}
|
||||
}
|
||||
better := victimID == ""
|
||||
if !better && orphan != victimOrphan {
|
||||
better = orphan
|
||||
}
|
||||
if !better && orphan == victimOrphan {
|
||||
better = len(qq) > victimLen || (len(qq) == victimLen && id < victimID)
|
||||
}
|
||||
if better {
|
||||
victimID = id
|
||||
victimLen = len(qq)
|
||||
victimOrphan = orphan
|
||||
}
|
||||
}
|
||||
if victimID != "" {
|
||||
item, qs.q[victimID] = popFront(qs.q[victimID])
|
||||
qs.remaining--
|
||||
return item, thief.WorkerID
|
||||
}
|
||||
return takeLeftover()
|
||||
}
|
||||
@@ -1,600 +0,0 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
func replicaJob(id, source string) *plugin_pb.JobSpec {
|
||||
return &plugin_pb.JobSpec{
|
||||
JobId: id,
|
||||
JobType: "erasure_coding",
|
||||
Summary: id,
|
||||
DedupeKey: id,
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func registerECWorkers(t *testing.T, pluginSvc *Plugin, workers map[string]string) {
|
||||
t.Helper()
|
||||
for id, addr := range workers {
|
||||
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: id,
|
||||
Address: addr,
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "erasure_coding", CanExecute: true, MaxExecutionConcurrency: 8},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignJobsWaterfillsPreferredWorkers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-a": "10.0.0.1:9334",
|
||||
"worker-b": "10.0.0.2:9334",
|
||||
"worker-c": "10.0.0.3:9334",
|
||||
"worker-d": "10.0.0.4:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("j1", "10.0.0.1:8080"),
|
||||
replicaJob("j2", "10.0.0.1:8080"),
|
||||
replicaJob("j3", "10.0.0.1:8080"),
|
||||
replicaJob("j4", "10.0.0.1:8080"),
|
||||
}
|
||||
// All four replica IPs listed so every worker is preferred.
|
||||
for _, job := range jobs {
|
||||
job.Parameters["replica_servers"] = &plugin_pb.ConfigValue{
|
||||
Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{
|
||||
Values: []string{"10.0.0.1:8080", "10.0.0.2:8080", "10.0.0.3:8080", "10.0.0.4:8080"},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, schedulerPolicy{LocalQueuePin: 5})
|
||||
if qs.queueLen("worker-a")+qs.queueLen("worker-b")+qs.queueLen("worker-c")+qs.queueLen("worker-d") != 4 {
|
||||
t.Fatalf("expected 4 assigned jobs")
|
||||
}
|
||||
for _, id := range []string{"worker-a", "worker-b", "worker-c", "worker-d"} {
|
||||
if qs.queueLen(id) != 1 {
|
||||
t.Fatalf("want 1 job on %s, got %d", id, qs.queueLen(id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignAndTakeDoesNotStarveOtherHomes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("local-1", "10.0.0.1:8080"),
|
||||
replicaJob("local-2", "10.0.0.1:8080"),
|
||||
replicaJob("remote-1", "10.0.0.2:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 1 {
|
||||
t.Fatalf("homes local=%d remote=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"))
|
||||
}
|
||||
|
||||
_, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || target2 != "worker-remote" {
|
||||
t.Fatalf("second take should start remote home, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOnlyWhenHomeQueueReachesK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 6)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 {
|
||||
t.Fatalf("all jobs should home on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
// 5 still queued on local (>=K) and local has no slot → steal to remote.
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("expected steal to remote, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 4 {
|
||||
t.Fatalf("after steal want 4 left on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
// 4 < K: do not steal again.
|
||||
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatalf("pin should hold remaining jobs, got target=%s", tgt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsImmediatelyWhenPinDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("a", "10.0.0.1:8080"),
|
||||
replicaJob("b", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 0}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("K=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOrphanedHomeWithoutK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("o1", "10.0.0.1:8080"),
|
||||
replicaJob("o2", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 {
|
||||
t.Fatalf("want 2 on local, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
|
||||
if item, tgt := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatalf("within grace should hold orphan queue, got target=%s", tgt)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 2 {
|
||||
t.Fatalf("held queue changed: %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
qs.mu.Lock()
|
||||
qs.missingSince["worker-local"] = time.Now().Add(-time.Minute)
|
||||
qs.mu.Unlock()
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("after grace should steal without K, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 1 {
|
||||
t.Fatalf("want 1 orphan left, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
stolen2, stealTarget2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen2 == nil || stealTarget2 != "worker-remote" {
|
||||
t.Fatalf("second orphan should also move, got job=%v target=%s", stolen2, stealTarget2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeKeepsOrphanWhenHomeReconnectsDuringGrace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{replicaJob("blink", "10.0.0.1:8080")}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
if item, _ := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy); item != nil {
|
||||
t.Fatal("expected hold during grace")
|
||||
}
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{"worker-local": "10.0.0.1:9334"})
|
||||
item, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item == nil || target != "worker-local" {
|
||||
t.Fatalf("reconnected home should run its queue, got job=%v target=%s", item, target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakeStealsOrphanImmediatelyWhenGraceDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{replicaJob("o1", "10.0.0.1:8080")}
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5, OrphanRehomeGrace: 0}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
pluginSvc.registry.Remove("worker-local")
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" {
|
||||
t.Fatalf("grace=0 should steal immediately, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindQueuedJobWorkerKeepsPendingState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
job := replicaJob("bind-1", "10.0.0.1:8080")
|
||||
pluginSvc.trackExecutionQueued(job)
|
||||
pluginSvc.bindQueuedJobWorker(job, "worker-local")
|
||||
|
||||
pluginSvc.jobsMu.Lock()
|
||||
tracked := pluginSvc.jobs[job.JobId]
|
||||
pluginSvc.jobsMu.Unlock()
|
||||
if tracked == nil {
|
||||
t.Fatal("missing tracked job")
|
||||
}
|
||||
if tracked.State != "job_state_pending" {
|
||||
t.Fatalf("state=%s", tracked.State)
|
||||
}
|
||||
if tracked.Stage != "queued" {
|
||||
t.Fatalf("stage=%s", tracked.Stage)
|
||||
}
|
||||
if tracked.WorkerID != "worker-local" {
|
||||
t.Fatalf("worker=%s", tracked.WorkerID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomelessAllRemoteWorkersUseFIFOWithoutPinOrGrace(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
// Workers never registered on the replica hostname — homeless, not orphan.
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-a": "10.0.0.5:9334",
|
||||
"worker-b": "10.0.0.6:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("h1", "10.0.0.1:8080"),
|
||||
replicaJob("h2", "10.0.0.1:8080"),
|
||||
replicaJob("h3", "10.0.0.1:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-a") != 0 || qs.queueLen("worker-b") != 0 {
|
||||
t.Fatalf("homeless jobs must not get a fake home, a=%d b=%d", qs.queueLen("worker-a"), qs.queueLen("worker-b"))
|
||||
}
|
||||
if qs.leftoverLen() != 3 {
|
||||
t.Fatalf("want 3 homeless FIFO, got %d", qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "h1" {
|
||||
t.Fatalf("FIFO first want h1, got %v", item1)
|
||||
}
|
||||
_, release1, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, target1)
|
||||
if err != nil {
|
||||
t.Fatalf("reserve 1: %v", err)
|
||||
}
|
||||
defer release1()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "h2" {
|
||||
t.Fatalf("FIFO second want h2 immediately, got %v", item2)
|
||||
}
|
||||
if target2 == "" || target2 == target1 {
|
||||
t.Fatalf("second take should hit the other idle worker, t1=%s t2=%s", target1, target2)
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("want 1 homeless left, got %d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMixedReplicaLocalJobsStayHomeBeforeHomeless(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("local-1", "10.0.0.1:8080"),
|
||||
replicaJob("local-2", "10.0.0.1:8080"),
|
||||
replicaJob("homeless-1", "10.0.0.9:8080"),
|
||||
replicaJob("homeless-2", "10.0.0.9:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 2 || qs.queueLen("worker-remote") != 0 || qs.leftoverLen() != 2 {
|
||||
t.Fatalf("homes local=%d remote=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-remote"), qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "local-1" || target1 != "worker-local" {
|
||||
t.Fatalf("first take want local-1 on home, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-remote" {
|
||||
t.Fatalf("remote should take homeless, not steal pinned home, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
if qs.queueLen("worker-local") != 1 {
|
||||
t.Fatalf("remaining replica-local job must stay home, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("one homeless should remain, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdleHomeWorkerTakesHomelessAfterEmptyHomeQueue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
})
|
||||
|
||||
jobs := []*plugin_pb.JobSpec{
|
||||
replicaJob("home-1", "10.0.0.1:8080"),
|
||||
replicaJob("homeless-1", "10.0.0.9:8080"),
|
||||
}
|
||||
policy := schedulerPolicy{
|
||||
PerWorkerConcurrency: 1,
|
||||
LocalQueuePin: 5,
|
||||
OrphanRehomeGrace: 30 * time.Second,
|
||||
}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 1 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || item1.job.JobId != "home-1" || target1 != "worker-local" {
|
||||
t.Fatalf("first take want home-1, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "homeless-1" || target2 != "worker-local" {
|
||||
t.Fatalf("idle home with empty queue may take homeless, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
if qs.leftoverLen() != 0 {
|
||||
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomelessWorkerTakesFIFOBeforeStealableHome(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-remote": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 7)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("home=%d leftover=%d", qs.queueLen("worker-local"), qs.leftoverLen())
|
||||
}
|
||||
if qs.hasHome("worker-remote") {
|
||||
t.Fatal("remote must have no home identity")
|
||||
}
|
||||
|
||||
_, target := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if target != "worker-local" {
|
||||
t.Fatalf("first take want local, got %s", target)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item, takeTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item == nil || takeTarget != "worker-remote" || item.job.JobId != "homeless-1" {
|
||||
t.Fatalf("remote must take homeless FIFO before K-steal, got job=%v target=%s", item, takeTarget)
|
||||
}
|
||||
if qs.leftoverLen() != 0 {
|
||||
t.Fatalf("homeless should be gone, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
if qs.queueLen("worker-local") != 5 {
|
||||
t.Fatalf("stealable home must stay until FIFO empty, got %d", qs.queueLen("worker-local"))
|
||||
}
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-remote" || stolen.job.JobId == "homeless-1" {
|
||||
t.Fatalf("after FIFO empty remote may steal >=K, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomeWorkerStealsBeforeHomelessAfterOwnQueueEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
registerECWorkers(t, pluginSvc, map[string]string{
|
||||
"worker-local": "10.0.0.1:9334",
|
||||
"worker-other": "10.0.0.2:9334",
|
||||
})
|
||||
|
||||
jobs := make([]*plugin_pb.JobSpec, 0, 8)
|
||||
for i := 0; i < 6; i++ {
|
||||
jobs = append(jobs, replicaJob(fmt.Sprintf("j%d", i), "10.0.0.1:8080"))
|
||||
}
|
||||
jobs = append(jobs, replicaJob("other-1", "10.0.0.2:8080"))
|
||||
jobs = append(jobs, replicaJob("homeless-1", "10.0.0.9:8080"))
|
||||
policy := schedulerPolicy{PerWorkerConcurrency: 1, LocalQueuePin: 5}
|
||||
qs := pluginSvc.assignJobsToWorkerQueues("erasure_coding", jobs, policy)
|
||||
if qs.queueLen("worker-local") != 6 || qs.queueLen("worker-other") != 1 || qs.leftoverLen() != 1 {
|
||||
t.Fatalf("local=%d other=%d leftover=%d", qs.queueLen("worker-local"), qs.queueLen("worker-other"), qs.leftoverLen())
|
||||
}
|
||||
if !qs.hasHome("worker-other") {
|
||||
t.Fatal("other must keep home identity")
|
||||
}
|
||||
|
||||
item1, target1 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item1 == nil || target1 != "worker-local" {
|
||||
t.Fatalf("first take want local home, got job=%v target=%s", item1, target1)
|
||||
}
|
||||
_, release, err := pluginSvc.reserveExecutorByID("erasure_coding", policy, "worker-local")
|
||||
if err != nil {
|
||||
t.Fatalf("reserve local: %v", err)
|
||||
}
|
||||
defer release()
|
||||
|
||||
item2, target2 := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if item2 == nil || item2.job.JobId != "other-1" || target2 != "worker-other" {
|
||||
t.Fatalf("other should drain own home first, got job=%v target=%s", item2, target2)
|
||||
}
|
||||
|
||||
stolen, stealTarget := pluginSvc.takeQueuedJob(qs, "erasure_coding", policy)
|
||||
if stolen == nil || stealTarget != "worker-other" || stolen.job.JobId == "homeless-1" {
|
||||
t.Fatalf("home worker with empty own queue still steals before homeless, got job=%v target=%s", stolen, stealTarget)
|
||||
}
|
||||
if qs.leftoverLen() != 1 {
|
||||
t.Fatalf("homeless should still wait, leftover=%d", qs.leftoverLen())
|
||||
}
|
||||
}
|
||||
@@ -154,7 +154,7 @@ func init() {
|
||||
filerS3Options.metricsHttpPort = cmdFiler.Flag.Int("s3.metricsPort", 0, "Prometheus metrics listen port")
|
||||
filerS3Options.metricsHttpIp = cmdFiler.Flag.String("s3.metricsIp", "", "metrics listen ip. If empty, default to same as -s3.ip.bind option.")
|
||||
cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
|
||||
filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
|
||||
filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
filerS3Options.autoCreateBucket = cmdFiler.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
|
||||
filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
|
||||
filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
|
||||
|
||||
@@ -336,16 +336,14 @@ func processUpdateEvent(
|
||||
}
|
||||
oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
|
||||
dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
|
||||
eventSource := renameSource{path: util.NewFullPath(resp.Directory, message.OldEntry.Name), loc: oldDest, stamp: message.OldEntry.GetRemoteEntry()}
|
||||
// sourceOf is where a remote-only entry's content is: the source its
|
||||
// pending-rename marker names, which may be an earlier rename's old key,
|
||||
// or else the event's old key
|
||||
sourceOf := func(entry *filer_pb.Entry, fallback renameSource) renameSource {
|
||||
sourceOf := func(entry *filer_pb.Entry, eventSource *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
|
||||
if src := string(entry.Extended[filer.ExtRemoteRenameSourceKey]); strings.HasPrefix(src, strings.TrimSuffix(mountedDir, "/")+"/") {
|
||||
// a pending entry carries the stamp of the source it reads
|
||||
return renameSource{path: util.FullPath(src), loc: toRemoteStorageLocation(util.FullPath(mountedDir), util.FullPath(src), remoteStorageMountLocation), stamp: entry.GetRemoteEntry()}
|
||||
return toRemoteStorageLocation(util.FullPath(mountedDir), util.FullPath(src), remoteStorageMountLocation)
|
||||
}
|
||||
return fallback
|
||||
return eventSource
|
||||
}
|
||||
if proto.Equal(oldDest, dest) && !shouldSendToRemote(message.NewEntry) {
|
||||
glog.V(2).Infof("skipping updating: %+v", resp)
|
||||
@@ -386,13 +384,13 @@ func processUpdateEvent(
|
||||
return err
|
||||
}
|
||||
if isRemoteOnly(current) {
|
||||
source := sourceOf(current, eventSource)
|
||||
source := sourceOf(current, oldDest)
|
||||
if err := completeRemoteOnlyRename(filerClient, client, message.NewParentPath, current, source, dest, storageClass); err != nil {
|
||||
return err
|
||||
}
|
||||
// a pending entry the filer copied on this rename leaves the
|
||||
// source its marker named
|
||||
if pending := sourceOf(message.OldEntry, renameSource{}); pending.loc != nil && !proto.Equal(pending.loc, source.loc) {
|
||||
if pending := sourceOf(message.OldEntry, nil); pending != nil && !proto.Equal(pending, source) {
|
||||
return releasePendingSource(filerSource, client, mountedDir, remoteStorageMountLocation, message.OldEntry)
|
||||
}
|
||||
return nil
|
||||
@@ -418,7 +416,7 @@ func processUpdateEvent(
|
||||
if errors.Is(writeErr, errSuperseded) {
|
||||
glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr)
|
||||
if !proto.Equal(oldDest, dest) {
|
||||
if err := uploadCurrentEntry(filerClient, filerSource, client, message.NewParentPath, message.NewEntry.Name, eventSource, dest, storageClass, sourceOf); err != nil {
|
||||
if err := uploadCurrentEntry(filerClient, filerSource, client, message.NewParentPath, message.NewEntry.Name, oldDest, dest, storageClass, sourceOf); err != nil {
|
||||
return err
|
||||
}
|
||||
return releaseRenamedSource(filerSource, client, mountedDir, remoteStorageMountLocation, message)
|
||||
@@ -483,14 +481,7 @@ func isRemoteOnly(entry *filer_pb.Entry) bool {
|
||||
// copied over the destination, the entry is stamped with the copy, and the
|
||||
// old key goes. With neither object present the content is lost: the event
|
||||
// fails and holds the offset for recovery.
|
||||
func completeRemoteOnlyRename(filerClient filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir string, current *filer_pb.Entry, source renameSource, dest *remote_pb.RemoteStorageLocation, storageClass string) error {
|
||||
oldDest := source.loc
|
||||
// the source's recorded stamp; a retry can find the entry already
|
||||
// stamped with the copy, whose ETag a multipart source does not share
|
||||
sourceStamp := source.stamp
|
||||
if sourceStamp == nil {
|
||||
sourceStamp = proto.Clone(current.RemoteEntry).(*filer_pb.RemoteEntry)
|
||||
}
|
||||
func completeRemoteOnlyRename(filerClient filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir string, current *filer_pb.Entry, oldDest, dest *remote_pb.RemoteStorageLocation, storageClass string) error {
|
||||
if existing, err := client.StatFile(dest); err == nil {
|
||||
if describes(current.RemoteEntry, existing) {
|
||||
glog.V(0).Infof("%s already holds the renamed content", remote_storage.FormatLocation(dest))
|
||||
@@ -500,7 +491,7 @@ func completeRemoteOnlyRename(filerClient filer_pb.FilerClient, client remote_st
|
||||
return err
|
||||
}
|
||||
}
|
||||
return deleteRenameSource(filerClient, client, dir, current.Name, source, sourceStamp)
|
||||
return deleteRenameSource(filerClient, client, dir, current.Name, oldDest)
|
||||
}
|
||||
glog.V(0).Infof("%s holds an object the entry does not describe (size %d, want %d); replacing it from %s", remote_storage.FormatLocation(dest), existing.RemoteSize, current.RemoteEntry.RemoteSize, remote_storage.FormatLocation(oldDest))
|
||||
} else if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
|
||||
@@ -526,42 +517,21 @@ func completeRemoteOnlyRename(filerClient filer_pb.FilerClient, client remote_st
|
||||
if err := updateLocalEntry(filerClient, dir, current, remoteEntry); err != nil {
|
||||
return err
|
||||
}
|
||||
return deleteRenameSource(filerClient, client, dir, current.Name, source, sourceStamp)
|
||||
}
|
||||
|
||||
// renameSource is where a renamed remote-only entry's content was read
|
||||
// from: the path, the object it maps to, and the stamp describing it.
|
||||
type renameSource struct {
|
||||
path util.FullPath
|
||||
loc *remote_pb.RemoteStorageLocation
|
||||
stamp *filer_pb.RemoteEntry
|
||||
return deleteRenameSource(filerClient, client, dir, current.Name, oldDest)
|
||||
}
|
||||
|
||||
// deleteRenameSource deletes a rename's old key once the renamed entry no
|
||||
// longer reads it: an entry still carrying the pending-source marker (its
|
||||
// stamp lost a race) keeps the source, and the event is retried. The key is
|
||||
// left alone when the object there is no longer the one stamp describes, as
|
||||
// when another rename copied onto it, visible yet or not.
|
||||
func deleteRenameSource(filerClient filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir, name string, source renameSource, stamp *filer_pb.RemoteEntry) error {
|
||||
// stamp lost a race) keeps the source, and the event is retried.
|
||||
func deleteRenameSource(filerClient filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir, name string, oldDest *remote_pb.RemoteStorageLocation) error {
|
||||
current, err := currentEntry(filerClient, dir, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current != nil && len(current.Extended[filer.ExtRemoteRenameSourceKey]) > 0 {
|
||||
return fmt.Errorf("%s still reads %s", util.NewFullPath(dir, name), remote_storage.FormatLocation(source.loc))
|
||||
return fmt.Errorf("%s still reads %s", util.NewFullPath(dir, name), remote_storage.FormatLocation(oldDest))
|
||||
}
|
||||
live, err := client.StatFile(source.loc)
|
||||
if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !describes(stamp, live) || stamp.GetRemoteETag() != "" && live.RemoteETag == "" {
|
||||
glog.V(0).Infof("keep %s: it is no longer the renamed content", remote_storage.FormatLocation(source.loc))
|
||||
return nil
|
||||
}
|
||||
return deleteRemoteFile(client, source.loc)
|
||||
return deleteRemoteFile(client, oldDest)
|
||||
}
|
||||
|
||||
// describes reports whether the object a stat returned is the one the entry's
|
||||
@@ -597,7 +567,7 @@ func openRemoteObject(client remote_storage.RemoteStorageClient, loc *remote_pb.
|
||||
// up once. A remote-only entry is finished the way completeRemoteOnlyRename
|
||||
// finishes a rename: its stamp can already describe the destination (a sync
|
||||
// plus remote.uncache in the meantime), and only then is it complete.
|
||||
func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir, name string, oldSource renameSource, dest *remote_pb.RemoteStorageLocation, storageClass string, sourceOf func(*filer_pb.Entry, renameSource) renameSource) error {
|
||||
func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.FilerClient, client remote_storage.RemoteStorageClient, dir, name string, oldDest, dest *remote_pb.RemoteStorageLocation, storageClass string, sourceOf func(*filer_pb.Entry, *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation) error {
|
||||
current, _, _, err := filer_pb.GetEntry(context.Background(), filerSource, util.NewFullPath(dir, name))
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil
|
||||
@@ -609,7 +579,7 @@ func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.F
|
||||
return nil
|
||||
}
|
||||
if isRemoteOnly(current) {
|
||||
return completeRemoteOnlyRename(filerClient, client, dir, current, sourceOf(current, oldSource), dest, storageClass)
|
||||
return completeRemoteOnlyRename(filerClient, client, dir, current, sourceOf(current, oldDest), dest, storageClass)
|
||||
}
|
||||
if shouldSendToRemote(current) {
|
||||
glog.V(0).Infof("leaving %s to the rewrite that superseded the rename", remote_storage.FormatLocation(dest))
|
||||
|
||||
@@ -691,10 +691,8 @@ type recordingRemote struct {
|
||||
// objects present on the remote, by path: StatFile and ReadFile answer
|
||||
// from it, everything else is ErrRemoteObjectNotFound.
|
||||
objects map[string][]byte
|
||||
// statETag is the ETag StatFile reports for a present object, unless
|
||||
// statETags names one for its path.
|
||||
statETag string
|
||||
statETags map[string]string
|
||||
// statETag is the ETag StatFile reports for a present object.
|
||||
statETag string
|
||||
}
|
||||
|
||||
func (r *recordingRemote) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (*filer_pb.RemoteEntry, error) {
|
||||
@@ -723,11 +721,7 @@ func (r *recordingRemote) StatFile(loc *remote_pb.RemoteStorageLocation) (*filer
|
||||
if !ok {
|
||||
return nil, remote_storage.ErrRemoteObjectNotFound
|
||||
}
|
||||
etag := r.statETag
|
||||
if e, ok := r.statETags[loc.Path]; ok {
|
||||
etag = e
|
||||
}
|
||||
return &filer_pb.RemoteEntry{StorageName: loc.Name, RemoteSize: int64(len(body)), RemoteETag: etag}, nil
|
||||
return &filer_pb.RemoteEntry{StorageName: loc.Name, RemoteSize: int64(len(body)), RemoteETag: r.statETag}, nil
|
||||
}
|
||||
|
||||
func (r *recordingRemote) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) ([]byte, error) {
|
||||
@@ -1393,7 +1387,7 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 1024})
|
||||
remote := &recordingRemote{}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", renameSource{loc: wantDelete}, wantWrite, "", eventSource)
|
||||
err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", wantDelete, wantWrite, "", eventSource)
|
||||
if err == nil || !strings.Contains(err.Error(), "neither") {
|
||||
t.Errorf("err = %v, want the lost-content failure", err)
|
||||
}
|
||||
@@ -1404,9 +1398,9 @@ func TestSupersededRenameUploadsCurrentEntry(t *testing.T) {
|
||||
|
||||
t.Run("uncached with the destination already synced: complete without a write", func(t *testing.T) {
|
||||
current := entryWith("b.txt", &filer_pb.RemoteEntry{StorageName: "gcs", RemoteMtime: 1786096669, RemoteSize: 7})
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/dir/a.txt": []byte("payload"), "/b/dir/b.txt": []byte("payload")}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/dir/b.txt": []byte("payload")}}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", renameSource{loc: wantDelete}, wantWrite, "", eventSource); err != nil {
|
||||
if err := uploadCurrentEntry(filerClient, filerClient, remote, "/buckets/b/dir", "b.txt", wantDelete, wantWrite, "", eventSource); err != nil {
|
||||
t.Fatalf("err = %v, want nil: the destination object matches the entry's stamp", err)
|
||||
}
|
||||
if len(remote.writes) != 0 {
|
||||
@@ -1439,7 +1433,7 @@ func TestCompleteRemoteOnlyRenameClearsThePendingSourceFirst(t *testing.T) {
|
||||
current := pending()
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
remote := &recordingRemote{objects: objects(), statETag: `"copy"`}
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b/dst", current, renameSource{path: "/buckets/b/src/a.jpg", loc: oldDest}, dest, ""); err != nil {
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b/dst", current, oldDest, dest, ""); err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if filerClient.updates != 1 {
|
||||
@@ -1454,7 +1448,7 @@ func TestCompleteRemoteOnlyRenameClearsThePendingSourceFirst(t *testing.T) {
|
||||
// the stamp did not land: the filer still has the marker
|
||||
filerClient := &stubFilerClient{entry: pending()}
|
||||
remote := &recordingRemote{objects: objects(), statETag: `"copy"`}
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b/dst", pending(), renameSource{path: "/buckets/b/src/a.jpg", loc: oldDest}, dest, ""); err == nil {
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b/dst", pending(), oldDest, dest, ""); err == nil {
|
||||
t.Fatal("err = nil, want the event retried")
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
@@ -1464,8 +1458,8 @@ func TestCompleteRemoteOnlyRenameClearsThePendingSourceFirst(t *testing.T) {
|
||||
}
|
||||
|
||||
// eventSource resolves a rename's source to the event's old key.
|
||||
func eventSource(_ *filer_pb.Entry, source renameSource) renameSource {
|
||||
return source
|
||||
func eventSource(_ *filer_pb.Entry, oldDest *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {
|
||||
return oldDest
|
||||
}
|
||||
|
||||
func TestRenameOfAPendingEntryFollowsItsSource(t *testing.T) {
|
||||
@@ -1511,7 +1505,7 @@ func TestRenameOfAPendingEntryFollowsItsSource(t *testing.T) {
|
||||
|
||||
t.Run("the second rename copies from the marker's source", func(t *testing.T) {
|
||||
current := pendingC()
|
||||
remote := &recordingRemote{objects: objects(), statETag: `"a"`}
|
||||
remote := &recordingRemote{objects: objects()}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, event("b.jpg", "c.jpg")); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -1717,37 +1711,3 @@ func TestMetadataOnlyUpdateRestampsSyncTs(t *testing.T) {
|
||||
t.Error("the restamped entry should read synced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteRemoteOnlyRenameKeepsAReusedOldKey(t *testing.T) {
|
||||
// A was renamed to B, then D to A before the sync got to the first
|
||||
// rename: the object at A is now D's, and the only copy of it
|
||||
oldDest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/a.jpg"}
|
||||
dest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/b.jpg"}
|
||||
current := &filer_pb.Entry{Name: "b.jpg", RemoteEntry: &filer_pb.RemoteEntry{RemoteETag: `"a"`, RemoteSize: 6}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-d"), "/b/b.jpg": []byte("data-a")}, statETag: `"a"`}
|
||||
remote.statETags = map[string]string{"/b/a.jpg": `"d"`}
|
||||
filerClient := &stubFilerClient{entry: current}
|
||||
if err := completeRemoteOnlyRename(filerClient, remote, "/buckets/b", current, renameSource{path: "/buckets/b/a.jpg", loc: oldDest}, dest, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.deletes) != 0 {
|
||||
t.Errorf("deletes = %+v, want the reused key kept", remote.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteRemoteOnlyRenameRetryUsesTheSourceStamp(t *testing.T) {
|
||||
// an earlier run copied a multipart source and stamped the entry with
|
||||
// the copy's ETag, then failed to delete the source
|
||||
oldDest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/a.jpg"}
|
||||
dest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/b/b.jpg"}
|
||||
current := &filer_pb.Entry{Name: "b.jpg", RemoteEntry: &filer_pb.RemoteEntry{RemoteETag: `"copy"`, RemoteSize: 6}}
|
||||
remote := &recordingRemote{objects: map[string][]byte{"/b/a.jpg": []byte("data-a"), "/b/b.jpg": []byte("data-a")},
|
||||
statETags: map[string]string{"/b/a.jpg": `"a-2"`, "/b/b.jpg": `"copy"`}}
|
||||
source := renameSource{path: "/buckets/b/a.jpg", loc: oldDest, stamp: &filer_pb.RemoteEntry{RemoteETag: `"a-2"`, RemoteSize: 6}}
|
||||
if err := completeRemoteOnlyRename(&stubFilerClient{entry: current}, remote, "/buckets/b", current, source, dest, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(remote.deletes) != 1 || remote.deletes[0].Path != "/b/a.jpg" {
|
||||
t.Errorf("deletes = %+v, want the unchanged source deleted on the retry", remote.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
+55
-158
@@ -12,7 +12,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -65,9 +64,8 @@ var (
|
||||
miniS3Options S3Options
|
||||
miniWebDavOptions WebDavOption
|
||||
miniAdminOptions AdminOptions
|
||||
// Flags explicitly provided on the command line or via WEED_* env vars,
|
||||
// captured before the saved options file is applied
|
||||
explicitFlags map[string]bool
|
||||
// Track which port flags were explicitly passed on CLI before config file is applied
|
||||
explicitPortFlags map[string]bool
|
||||
miniEnableWebDAV *bool
|
||||
miniEnableS3 *bool
|
||||
miniEnableAdminUI *bool
|
||||
@@ -378,19 +376,6 @@ The S3 endpoint is available at http://localhost:8333. For client
|
||||
configuration and IAM setup, see the project documentation or use the
|
||||
Admin UI (http://localhost:23646) to manage users and policies.
|
||||
|
||||
Notes:
|
||||
- Values mini computes itself (currently master.volumeSizeLimitMB) are
|
||||
pinned in <dir>/mini.options and applied again on the next start to
|
||||
keep them stable; applied values are printed at startup. Command-line
|
||||
flags are never saved and always win over saved values. Edit or
|
||||
delete mini.options to reset a computed value.
|
||||
- The official Docker image ships /etc/seaweedfs/filer.toml, which keeps
|
||||
the filer metadata in /data/filerldb2 regardless of -dir; mini warns
|
||||
at startup when a configured store dir is outside -dir. If you set
|
||||
-dir to another path, keep /data on a persistent volume as well.
|
||||
Otherwise the metadata is lost when the container is recreated, and
|
||||
buckets list no objects although the volume files are still in -dir.
|
||||
|
||||
`,
|
||||
}
|
||||
|
||||
@@ -407,7 +392,7 @@ var (
|
||||
miniMetricsHttpIp = cmdMini.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")
|
||||
miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")
|
||||
miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
|
||||
miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
miniS3AutoCreateBucket = cmdMini.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
|
||||
miniBucket = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")
|
||||
miniTableBucket = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables buckets to create on startup if they do not already exist, each name[:FORMAT] with FORMAT one of ICEBERG (default) or LANCE, e.g. warehouse,vectors:LANCE; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.")
|
||||
@@ -838,8 +823,8 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
|
||||
// If the port is not available, it finds the next available port and updates the pointer
|
||||
// The reservedPorts map contains ports that should not be allocated (for gRPC collision avoidance)
|
||||
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string) error {
|
||||
// Check if this port was explicitly specified by the user (from CLI or env, before the config file was applied)
|
||||
isExplicitPort := explicitFlags[flagName]
|
||||
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
|
||||
isExplicitPort := explicitPortFlags[flagName]
|
||||
|
||||
if *portPtr == 0 {
|
||||
return nil
|
||||
@@ -1087,16 +1072,10 @@ func initializeGrpcPortsOnIP(bindIp string) {
|
||||
}
|
||||
}
|
||||
|
||||
// miniOptionsFilePath returns the path of the mini.options file under the
|
||||
// first -dir folder.
|
||||
func miniOptionsFilePath(dataFolder string) string {
|
||||
return filepath.Join(util.ResolvePath(util.StringSplit(dataFolder, ",")[0]), "mini.options")
|
||||
}
|
||||
|
||||
// loadMiniConfigurationFile reads the mini.options file and returns parsed options
|
||||
// File format: one option per line, without leading dash (e.g., "ip=127.0.0.1")
|
||||
func loadMiniConfigurationFile(dataFolder string) (map[string]string, error) {
|
||||
configFile := miniOptionsFilePath(dataFolder)
|
||||
configFile := filepath.Join(util.ResolvePath(util.StringSplit(dataFolder, ",")[0]), "mini.options")
|
||||
|
||||
options := make(map[string]string)
|
||||
|
||||
@@ -1144,112 +1123,31 @@ func loadMiniConfigurationFile(dataFolder string) (map[string]string, error) {
|
||||
return options, nil
|
||||
}
|
||||
|
||||
// applyConfigFileOptions sets command-line flags from loaded configuration
|
||||
// file. Options for flags the user provided explicitly (command line or
|
||||
// WEED_* env var) are skipped, so passing a flag always wins over the saved
|
||||
// value — including passing it at its default value. Returns a map of the
|
||||
// applied keys to their display values (secrets redacted).
|
||||
func applyConfigFileOptions(options map[string]string) map[string]string {
|
||||
keys := make([]string, 0, len(options))
|
||||
for key := range options {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
applied := make(map[string]string)
|
||||
for _, key := range keys {
|
||||
value := options[key]
|
||||
// Redact secrets (admin.password, admin.readOnlyPassword, ...) in
|
||||
// logs and the startup message; only flag.Value.Set sees the value.
|
||||
display := value
|
||||
if isSensitiveOption(key) {
|
||||
display = "*****"
|
||||
}
|
||||
if explicitFlags[key] {
|
||||
glog.V(2).Infof("Skipping config file option %s=%s (explicitly specified)", key, display)
|
||||
// applyConfigFileOptions sets command-line flags from loaded configuration file
|
||||
func applyConfigFileOptions(options map[string]string) {
|
||||
for key, value := range options {
|
||||
// Skip port flags that were explicitly passed on CLI
|
||||
if explicitPortFlags[key] {
|
||||
glog.V(2).Infof("Skipping config file option %s=%s (explicitly specified on command line)", key, value)
|
||||
continue
|
||||
}
|
||||
// Set the flag value if it hasn't been explicitly set on command line
|
||||
flag := cmdMini.Flag.Lookup(key)
|
||||
if flag == nil {
|
||||
continue
|
||||
}
|
||||
if err := flag.Value.Set(value); err != nil {
|
||||
glog.Warningf("Failed to apply config file option: %s=%s: %v", key, display, err)
|
||||
continue
|
||||
}
|
||||
glog.V(2).Infof("Applied config file option: %s=%s", key, display)
|
||||
applied[key] = display
|
||||
}
|
||||
return applied
|
||||
}
|
||||
|
||||
// isSensitiveOption reports whether a saved option holds a secret that must
|
||||
// not appear in logs or the startup message.
|
||||
func isSensitiveOption(key string) bool {
|
||||
k := strings.ToLower(key)
|
||||
return strings.Contains(k, "password") || strings.Contains(k, "secret") || strings.Contains(k, "token")
|
||||
}
|
||||
|
||||
// warnIfFilerStoreOutsideDir warns when a filer.toml configures a local
|
||||
// filer store outside the -dir folders and the master meta folder. When a
|
||||
// filer.toml is found the filer skips defaultLevelDbDirectory, so metadata
|
||||
// can silently land on a path the volume data doesn't use — in a container
|
||||
// that path is lost on recreate unless it is persisted separately.
|
||||
func warnIfFilerStoreOutsideDir(dataFolders string, metaFolder string) {
|
||||
if !util.LoadConfiguration("filer", false) {
|
||||
return
|
||||
}
|
||||
v := util.GetViper()
|
||||
folders := append(util.StringSplit(dataFolders, ","), metaFolder)
|
||||
inside := func(storeDir string) bool {
|
||||
// The filer stores open the configured dir verbatim, so absolutize it
|
||||
// the same way (relative to the working directory) rather than
|
||||
// expanding "~", which the store never expands.
|
||||
resolved, err := filepath.Abs(storeDir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, folder := range folders {
|
||||
base, err := filepath.Abs(util.ResolvePath(folder))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Resolve symlinks only when both sides exist, so a not-yet-
|
||||
// created store dir under a symlinked -dir still compares
|
||||
// lexically against the same spelling.
|
||||
r, b := resolved, base
|
||||
if er, err := filepath.EvalSymlinks(resolved); err == nil {
|
||||
if eb, err := filepath.EvalSymlinks(base); err == nil {
|
||||
r, b = er, eb
|
||||
if flag != nil {
|
||||
// Only set if not already set (by command line)
|
||||
if flag.Value.String() == flag.DefValue {
|
||||
if err := flag.Value.Set(value); err != nil {
|
||||
glog.Warningf("Failed to apply config file option: %s=%s: %v", key, value, err)
|
||||
} else {
|
||||
glog.V(2).Infof("Applied config file option: %s=%s", key, value)
|
||||
}
|
||||
}
|
||||
if rel, err := filepath.Rel(b, r); err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
// Local filer stores keep metadata under a "dir" key; check both the
|
||||
// default store (leveldb2.enabled) and path-specific stores
|
||||
// (leveldb2.<id>.enabled, keyed under <section>.location).
|
||||
localStores := map[string]bool{"leveldb": true, "leveldb2": true, "leveldb3": true}
|
||||
for _, key := range v.AllKeys() {
|
||||
if !strings.HasSuffix(key, ".enabled") || !v.GetBool(key) {
|
||||
continue
|
||||
}
|
||||
section := strings.TrimSuffix(key, ".enabled")
|
||||
if !localStores[strings.Split(section, ".")[0]] {
|
||||
continue
|
||||
}
|
||||
if storeDir := v.GetString(section + ".dir"); storeDir != "" && !inside(storeDir) {
|
||||
glog.Warningf("filer store %s uses dir %s from filer.toml, outside -dir %s; persist that path or filer metadata will be lost when the container is recreated", section, storeDir, dataFolders)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// saveMiniConfiguration writes values mini computed itself to mini.options.
|
||||
// Command-line flags are never saved, so a dropped flag cannot resurrect on
|
||||
// the next start; the file exists to pin environment-derived values.
|
||||
// saveMiniConfiguration saves the current mini configuration to a file
|
||||
// The file format uses option=value format without leading dashes
|
||||
func saveMiniConfiguration(dataFolder string) error {
|
||||
configDir := util.ResolvePath(util.StringSplit(dataFolder, ",")[0])
|
||||
if err := os.MkdirAll(configDir, 0755); err != nil {
|
||||
@@ -1261,15 +1159,29 @@ func saveMiniConfiguration(dataFolder string) error {
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("#!/bin/bash\n")
|
||||
sb.WriteString("# Mini server auto-computed options\n")
|
||||
sb.WriteString("# Mini server configuration\n")
|
||||
sb.WriteString("# Format: option=value (no leading dash)\n")
|
||||
sb.WriteString("# This file is loaded on startup if it exists\n")
|
||||
sb.WriteString("# Command-line flags are not saved; only values mini computed itself.\n\n")
|
||||
sb.WriteString("# This file is loaded on startup if it exists\n\n")
|
||||
|
||||
// Pin the auto-calculated volume size so it stays stable across restarts
|
||||
// even as free disk space changes. Delete this line to force
|
||||
// recalculation on the next startup.
|
||||
// Collect all flags that were explicitly passed (except "dir")
|
||||
cmdMini.Flag.Visit(func(f *flag.Flag) {
|
||||
// Skip the "dir" option - it's environment-specific
|
||||
if f.Name == "dir" {
|
||||
return
|
||||
}
|
||||
value := f.Value.String()
|
||||
// Quote the value if it contains spaces
|
||||
if strings.Contains(value, " ") {
|
||||
sb.WriteString(fmt.Sprintf("%s=\"%s\"\n", f.Name, value))
|
||||
} else {
|
||||
sb.WriteString(fmt.Sprintf("%s=%s\n", f.Name, value))
|
||||
}
|
||||
})
|
||||
|
||||
// Add auto-calculated volume size if it was computed
|
||||
if !isFlagPassed("master.volumeSizeLimitMB") && miniMasterOptions.volumeSizeLimitMB != nil {
|
||||
sb.WriteString(fmt.Sprintf("\n# Auto-calculated volume size based on total disk capacity\n"))
|
||||
sb.WriteString(fmt.Sprintf("# Delete this line to force recalculation on next startup\n"))
|
||||
sb.WriteString(fmt.Sprintf("master.volumeSizeLimitMB=%d\n", *miniMasterOptions.volumeSizeLimitMB))
|
||||
}
|
||||
|
||||
@@ -1291,32 +1203,21 @@ func runMini(cmd *Command, args []string) bool {
|
||||
// can opt back into verbose output.
|
||||
quietMiniLogs()
|
||||
|
||||
// Capture which flags were explicitly provided on the command line or via
|
||||
// WEED_* env vars BEFORE the saved options file is applied. Saved options
|
||||
// only fill in flags the user did not provide this run, so passing a flag
|
||||
// at its default value also resets a saved value.
|
||||
explicitFlags = make(map[string]bool)
|
||||
cmdMini.Flag.Visit(func(f *flag.Flag) {
|
||||
explicitFlags[f.Name] = true
|
||||
})
|
||||
// Capture which port flags were explicitly passed on CLI BEFORE config file is applied
|
||||
// This is necessary to distinguish user-specified ports from defaults or config file options
|
||||
explicitPortFlags = make(map[string]bool)
|
||||
portFlagNames := []string{"master.port", "filer.port", "volume.port", "s3.port", "s3.port.iceberg", "webdav.port", "admin.port", "s3.iam.readOnly"}
|
||||
for _, flagName := range portFlagNames {
|
||||
explicitPortFlags[flagName] = isFlagPassed(flagName)
|
||||
}
|
||||
|
||||
// Load configuration from file if it exists. Capture the path before
|
||||
// applying: a saved "dir" option can change miniDataFolders.
|
||||
optionsFile := miniOptionsFilePath(*miniDataFolders)
|
||||
// Load configuration from file if it exists
|
||||
configOptions, err := loadMiniConfigurationFile(*miniDataFolders)
|
||||
if err != nil {
|
||||
glog.Warningf("Error loading configuration file: %v", err)
|
||||
}
|
||||
// Apply loaded options to flags (explicitly provided flags win)
|
||||
appliedOptions := applyConfigFileOptions(configOptions)
|
||||
if len(appliedOptions) > 0 {
|
||||
pairs := make([]string, 0, len(appliedOptions))
|
||||
for k, v := range appliedOptions {
|
||||
pairs = append(pairs, k+"="+v)
|
||||
}
|
||||
sort.Strings(pairs)
|
||||
fmt.Printf(" Applying %d saved option(s) from %s: %s\n", len(appliedOptions), optionsFile, strings.Join(pairs, " "))
|
||||
}
|
||||
// Apply loaded options to flags (CLI flags will override these)
|
||||
applyConfigFileOptions(configOptions)
|
||||
|
||||
if *miniOptions.debug {
|
||||
grace.StartDebugServer(*miniOptions.debugPort)
|
||||
@@ -1445,14 +1346,10 @@ func runMini(cmd *Command, args []string) bool {
|
||||
glog.Fatalf("Check Meta Folder (-dir=\"%s\") Writable: %s", *miniMasterOptions.metaFolder, err)
|
||||
}
|
||||
miniFilerOptions.defaultLevelDbDirectory = miniMasterOptions.metaFolder
|
||||
warnIfFilerStoreOutsideDir(*miniDataFolders, *miniMasterOptions.metaFolder)
|
||||
|
||||
// Calculate and set optimal volume size limit based on available disk space.
|
||||
// Skip when the user passed -master.volumeSizeLimitMB explicitly, and when
|
||||
// the value was loaded from mini.options — file-applied flags are not in
|
||||
// Visit, and the saved pin must survive restarts.
|
||||
_, sizeLimitFromFile := appliedOptions["master.volumeSizeLimitMB"]
|
||||
if !isFlagPassed("master.volumeSizeLimitMB") && !sizeLimitFromFile {
|
||||
// Calculate and set optimal volume size limit based on available disk space
|
||||
// Only auto-calculate if user didn't explicitly specify a value via -master.volumeSizeLimitMB
|
||||
if !isFlagPassed("master.volumeSizeLimitMB") {
|
||||
// User didn't override, use auto-calculated value
|
||||
// The -dir flag can accept comma-separated directories; use the first one for disk space calculation.
|
||||
// miniDataFolders was already tilde-resolved at the top of runMini.
|
||||
|
||||
+1
-1
@@ -114,7 +114,7 @@ func init() {
|
||||
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
cmdS3.Flag.Bool("allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
|
||||
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
|
||||
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
s3StandaloneOptions.autoCreateBucket = cmdS3.Flag.Bool("autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
|
||||
s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
|
||||
s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
|
||||
|
||||
@@ -182,7 +182,7 @@ func init() {
|
||||
s3Options.iamConfig = cmdServer.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3. Overrides -iam.config if both are provided.")
|
||||
s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
|
||||
cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
|
||||
s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", false, "allow recursive deleting all entries along with bucket")
|
||||
s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
s3Options.autoCreateBucket = cmdServer.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
|
||||
s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
|
||||
s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
|
||||
|
||||
@@ -82,7 +82,7 @@ func runPluginWorkerWithOptions(options pluginWorkerRunOptions) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir, options.Address)
|
||||
handlers, err := buildPluginWorkerHandlers(options.JobTypes, dialOption, options.MaxExecute, options.WorkingDir)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to build plugin worker handlers: %v", err)
|
||||
return false
|
||||
@@ -141,7 +141,7 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
|
||||
// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
|
||||
// (which may contain category names like "all", "default", "heavy" and/or
|
||||
// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
|
||||
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string, workerAddress ...string) ([]pluginworker.JobHandler, error) {
|
||||
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
|
||||
jobTypes = strings.TrimSpace(jobTypes)
|
||||
if jobTypes == "" {
|
||||
jobTypes = defaultPluginWorkerJobTypes
|
||||
@@ -152,16 +152,10 @@ func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxE
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addr := ""
|
||||
if len(workerAddress) > 0 {
|
||||
addr = strings.TrimSpace(workerAddress[0])
|
||||
}
|
||||
|
||||
opts := pluginworker.HandlerBuildOptions{
|
||||
GrpcDialOption: dialOption,
|
||||
MaxExecute: maxExecute,
|
||||
WorkingDir: workingDir,
|
||||
WorkerAddress: addr,
|
||||
}
|
||||
|
||||
handlers := make([]pluginworker.JobHandler, 0, len(factories))
|
||||
|
||||
@@ -1,359 +0,0 @@
|
||||
package ec
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// ecBalanceFakeServer answers the EC-shard RPC subset a balance run drives:
|
||||
// copy (rejected when full), mount, inventory, unmount, delete. The disk state
|
||||
// a real server tracks lives in shards; what the planner saw is rebuilt from it
|
||||
// through FetchTopology, so a landed move is visible to the next plan.
|
||||
type ecBalanceFakeServer struct {
|
||||
volume_server_pb.UnimplementedVolumeServerServer
|
||||
|
||||
mu sync.Mutex
|
||||
shards map[uint32]map[uint32]bool // vid -> shardId -> present
|
||||
full bool // VolumeEcShardsCopy answers "no space left"
|
||||
copyCalls int
|
||||
grpcAddr string
|
||||
stop func()
|
||||
}
|
||||
|
||||
func (f *ecBalanceFakeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.copyCalls++
|
||||
if f.full {
|
||||
return nil, fmt.Errorf("no space left")
|
||||
}
|
||||
return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *ecBalanceFakeServer) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest) (*volume_server_pb.VolumeEcShardsMountResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, sid := range req.ShardIds {
|
||||
f.shards[req.VolumeId][sid] = true
|
||||
}
|
||||
return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *ecBalanceFakeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
resp := &volume_server_pb.VolumeEcShardsInfoResponse{}
|
||||
for sid := range f.shards[req.VolumeId] {
|
||||
resp.EcShardInfos = append(resp.EcShardInfos, &volume_server_pb.EcShardInfo{
|
||||
VolumeId: req.VolumeId,
|
||||
ShardId: sid,
|
||||
Size: 1,
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (f *ecBalanceFakeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
|
||||
return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *ecBalanceFakeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, sid := range req.ShardIds {
|
||||
delete(f.shards[req.VolumeId], sid)
|
||||
}
|
||||
return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
|
||||
}
|
||||
|
||||
// nodeId returns the "host:httpPort.grpcPort" node id form the cluster uses, so
|
||||
// ToGrpcAddress resolves to this server's listener.
|
||||
func (f *ecBalanceFakeServer) nodeId() string {
|
||||
host, port, err := net.SplitHostPort(f.grpcAddr)
|
||||
if err != nil {
|
||||
return f.grpcAddr
|
||||
}
|
||||
return fmt.Sprintf("%s:1.%s", host, port)
|
||||
}
|
||||
|
||||
func (f *ecBalanceFakeServer) shardCount(vid uint32) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.shards[vid])
|
||||
}
|
||||
|
||||
func startEcBalanceFakeServer(t *testing.T, vid uint32, present ...uint32) *ecBalanceFakeServer {
|
||||
t.Helper()
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
|
||||
f := &ecBalanceFakeServer{
|
||||
shards: map[uint32]map[uint32]bool{vid: {}},
|
||||
grpcAddr: listener.Addr().String(),
|
||||
}
|
||||
for _, sid := range present {
|
||||
f.shards[vid][sid] = true
|
||||
}
|
||||
|
||||
server := grpc.NewServer()
|
||||
volume_server_pb.RegisterVolumeServerServer(server, f)
|
||||
go func() { _ = server.Serve(listener) }()
|
||||
f.stop = server.Stop
|
||||
t.Cleanup(server.Stop)
|
||||
return f
|
||||
}
|
||||
|
||||
// fetchEcBalanceTopology builds a FetchTopology that reports the fake servers'
|
||||
// live shard inventory.
|
||||
func fetchEcBalanceTopology(servers []*ecBalanceFakeServer) func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error) {
|
||||
return func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error) {
|
||||
var nodes []*master_pb.DataNodeInfo
|
||||
for _, f := range servers {
|
||||
di := &master_pb.DiskInfo{Type: string(types.HardDriveType), MaxVolumeCount: 100}
|
||||
f.mu.Lock()
|
||||
for v, set := range f.shards {
|
||||
if len(set) == 0 {
|
||||
continue
|
||||
}
|
||||
si := erasure_coding.NewShardsInfo()
|
||||
for sid := range set {
|
||||
si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(sid), 1))
|
||||
}
|
||||
di.EcShardInfos = append(di.EcShardInfos, &master_pb.VolumeEcShardInformationMessage{
|
||||
Id: v,
|
||||
Collection: "c1",
|
||||
EcIndexBits: si.Bitmap(),
|
||||
ShardSizes: si.SizesInt64(),
|
||||
DiskType: string(types.HardDriveType),
|
||||
})
|
||||
}
|
||||
f.mu.Unlock()
|
||||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||||
Id: f.nodeId(),
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{string(types.HardDriveType): di},
|
||||
})
|
||||
}
|
||||
return &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{
|
||||
Id: "r1",
|
||||
DataNodeInfos: nodes,
|
||||
}},
|
||||
}},
|
||||
}, 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcBalanceReplanAfterNoSpaceLeft reproduces seaweedfs/seaweedfs#11753:
|
||||
// the planner works from heartbeat free-slot counts that can disagree with the
|
||||
// volume server's own check, so a planned target can reject the copy with
|
||||
// "no space left" mid-run. The balance must re-plan without that node as a
|
||||
// target rather than fail the whole run.
|
||||
func TestEcBalanceReplanAfterNoSpaceLeft(t *testing.T) {
|
||||
const vid = uint32(1)
|
||||
var allShards []uint32
|
||||
for sid := 0; sid < erasure_coding.DataShardsCount+erasure_coding.ParityShardsCount; sid++ {
|
||||
allShards = append(allShards, uint32(sid))
|
||||
}
|
||||
|
||||
full := startEcBalanceFakeServer(t, vid)
|
||||
accept := startEcBalanceFakeServer(t, vid)
|
||||
source := startEcBalanceFakeServer(t, vid, allShards...)
|
||||
full.full = true
|
||||
|
||||
env := &Env{
|
||||
GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
FetchTopology: fetchEcBalanceTopology([]*ecBalanceFakeServer{source, full, accept}),
|
||||
}
|
||||
|
||||
err := EcBalance(env, []string{"c1"}, "", nil, types.HardDriveType, 3, 0, true, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("balance should re-plan around the full target, got: %v", err)
|
||||
}
|
||||
if full.copyCalls == 0 {
|
||||
t.Fatalf("expected the full server to have rejected at least one copy")
|
||||
}
|
||||
if got := full.shardCount(vid); got != 0 {
|
||||
t.Fatalf("full server should receive no shards, holds %d", got)
|
||||
}
|
||||
total := source.shardCount(vid) + full.shardCount(vid) + accept.shardCount(vid)
|
||||
if total != len(allShards) {
|
||||
t.Fatalf("shards lost or duplicated: total %d, want %d", total, len(allShards))
|
||||
}
|
||||
if source.shardCount(vid) == len(allShards) {
|
||||
t.Fatalf("no shards were moved off the source")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcBalanceReplanAfterNoSpaceLeftStaleSnapshot serves the pre-move
|
||||
// topology on every collection after the first, simulating a master heartbeat
|
||||
// that lags the landed moves. The re-plan must replay its completed moves onto
|
||||
// the stale snapshot instead of re-moving shards off the deleted source.
|
||||
func TestEcBalanceReplanAfterNoSpaceLeftStaleSnapshot(t *testing.T) {
|
||||
const vid = uint32(1)
|
||||
var allShards []uint32
|
||||
for sid := 0; sid < erasure_coding.DataShardsCount+erasure_coding.ParityShardsCount; sid++ {
|
||||
allShards = append(allShards, uint32(sid))
|
||||
}
|
||||
|
||||
full := startEcBalanceFakeServer(t, vid)
|
||||
accept := startEcBalanceFakeServer(t, vid)
|
||||
source := startEcBalanceFakeServer(t, vid, allShards...)
|
||||
full.full = true
|
||||
|
||||
live := fetchEcBalanceTopology([]*ecBalanceFakeServer{source, full, accept})
|
||||
var stale *master_pb.TopologyInfo
|
||||
calls := 0
|
||||
fetchTopology := func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error) {
|
||||
topo, size, err := live(delay)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
calls++
|
||||
if stale == nil {
|
||||
stale = topo
|
||||
}
|
||||
if calls > 1 {
|
||||
return stale, size, nil
|
||||
}
|
||||
return topo, size, nil
|
||||
}
|
||||
|
||||
env := &Env{
|
||||
GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
FetchTopology: fetchTopology,
|
||||
}
|
||||
|
||||
err := EcBalance(env, []string{"c1"}, "", nil, types.HardDriveType, 3, 0, true, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("balance should re-plan around the full target, got: %v", err)
|
||||
}
|
||||
if calls < 2 {
|
||||
t.Fatalf("expected a re-plan, topology collected %d time(s)", calls)
|
||||
}
|
||||
// Each shard lands on the accepting server exactly once; a stale re-plan
|
||||
// would copy already-moved shards again.
|
||||
if got, want := accept.copyCalls, accept.shardCount(vid); got != want {
|
||||
t.Fatalf("accepting server saw %d copies for %d held shards: completed moves were re-planned", got, want)
|
||||
}
|
||||
total := source.shardCount(vid) + full.shardCount(vid) + accept.shardCount(vid)
|
||||
if total != len(allShards) {
|
||||
t.Fatalf("shards lost or duplicated: total %d, want %d", total, len(allShards))
|
||||
}
|
||||
}
|
||||
|
||||
// TestEcBalanceNoReceivingTargetFails surfaces the refusal when every
|
||||
// destination that could take a shard has rejected a copy, rather than
|
||||
// reporting success for a plan that moved nothing.
|
||||
func TestEcBalanceNoReceivingTargetFails(t *testing.T) {
|
||||
const vid = uint32(1)
|
||||
var allShards []uint32
|
||||
for sid := 0; sid < erasure_coding.DataShardsCount+erasure_coding.ParityShardsCount; sid++ {
|
||||
allShards = append(allShards, uint32(sid))
|
||||
}
|
||||
|
||||
full := startEcBalanceFakeServer(t, vid)
|
||||
source := startEcBalanceFakeServer(t, vid, allShards...)
|
||||
full.full = true
|
||||
|
||||
env := &Env{
|
||||
GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
FetchTopology: fetchEcBalanceTopology([]*ecBalanceFakeServer{source, full}),
|
||||
}
|
||||
|
||||
err := EcBalance(env, []string{"c1"}, "", nil, types.HardDriveType, 3, 0, true, nil, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatalf("balance with no receiving target should fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no space left") {
|
||||
t.Fatalf("expected the refusal to surface, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyCompletedMovesReplaysDiskState checks that replaying a landed move
|
||||
// onto a stale snapshot carries the shard's disk attribution and the per-disk
|
||||
// slot accounting, so a re-plan does not treat the just-filled disk as empty.
|
||||
func TestApplyCompletedMovesReplaysDiskState(t *testing.T) {
|
||||
const vid = uint32(1)
|
||||
|
||||
source := newEcNode("dc1", "r1", "dn1", 99)
|
||||
source.Info.DiskInfos["hdd"] = &master_pb.DiskInfo{
|
||||
Type: "hdd",
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
|
||||
Id: vid,
|
||||
Collection: "c1",
|
||||
DiskId: 1,
|
||||
DiskType: "hdd",
|
||||
EcIndexBits: uint32(erasure_coding.ShardBits(0).Set(5)),
|
||||
}},
|
||||
}
|
||||
si := erasure_coding.NewShardsInfo()
|
||||
si.Set(erasure_coding.NewShardInfo(5, 1))
|
||||
source.Disks = map[uint32]*EcDisk{
|
||||
1: {DiskId: 1, DiskType: "hdd", FreeEcSlots: 99, EcShardCount: 1, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{1: si}},
|
||||
}
|
||||
|
||||
target := newEcNode("dc1", "r1", "dn2", 100)
|
||||
target.Disks = map[uint32]*EcDisk{
|
||||
1: {DiskId: 1, DiskType: "hdd", FreeEcSlots: 10, EcShardCount: 0, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
|
||||
2: {DiskId: 2, DiskType: "hdd", FreeEcSlots: 10, EcShardCount: 0, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
|
||||
}
|
||||
|
||||
nodes := []*EcNode{source, target}
|
||||
move := ecbalancer.Move{
|
||||
VolumeID: vid,
|
||||
ShardID: 5,
|
||||
Collection: "c1",
|
||||
SourceNode: "dn1",
|
||||
TargetNode: "dn2",
|
||||
TargetDisk: 2,
|
||||
Phase: "within_rack",
|
||||
}
|
||||
applyCompletedMoves(nodes, []ecbalancer.Move{move}, types.HardDriveType)
|
||||
|
||||
if ecNodeHasShard(source, needle.VolumeId(vid), 5) {
|
||||
t.Fatalf("source still holds shard 5")
|
||||
}
|
||||
if d := source.Disks[1]; d.EcShardCount != 0 || d.FreeEcSlots != 100 {
|
||||
t.Fatalf("source disk counts not restored: %+v", d)
|
||||
}
|
||||
if !ecNodeHasShard(target, needle.VolumeId(vid), 5) {
|
||||
t.Fatalf("target missing shard 5")
|
||||
}
|
||||
if d := target.Disks[2]; d.EcShardCount != 1 || d.FreeEcSlots != 9 {
|
||||
t.Fatalf("target disk 2 not charged: %+v", d)
|
||||
}
|
||||
if d := target.Disks[1]; d.EcShardCount != 0 {
|
||||
t.Fatalf("shard attributed to wrong disk: %+v", d)
|
||||
}
|
||||
if target.FreeEcSlot != 99 {
|
||||
t.Fatalf("target free slots = %d, want 99", target.FreeEcSlot)
|
||||
}
|
||||
|
||||
// Replaying again (or onto a fresher snapshot) must not double-count.
|
||||
applyCompletedMoves(nodes, []ecbalancer.Move{move}, types.HardDriveType)
|
||||
if d := target.Disks[2]; d.EcShardCount != 1 || d.FreeEcSlots != 9 {
|
||||
t.Fatalf("double-applied move: %+v", d)
|
||||
}
|
||||
if target.FreeEcSlot != 99 {
|
||||
t.Fatalf("target free slots = %d, want 99", target.FreeEcSlot)
|
||||
}
|
||||
}
|
||||
+49
-310
@@ -8,7 +8,6 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -763,26 +762,6 @@ type ecBalancer struct {
|
||||
// migratingVolumeIds are mid-encode volumes whose shards are ingested from
|
||||
// every disk-type bucket; see EcBalance.
|
||||
migratingVolumeIds map[uint32]bool
|
||||
// noReceiveTargets are node ids the planner may not pick as destinations;
|
||||
// noReceiveDisks narrows that to individual disks on a node: a server-side
|
||||
// auto-pick refusal means no disk qualifies, but an explicit-disk refusal
|
||||
// only convicts that disk. See EcBalance.
|
||||
noReceiveTargets map[string]bool
|
||||
noReceiveDisks map[string]map[uint32]bool
|
||||
// mu guards the execution journal written by concurrent move tasks.
|
||||
mu sync.Mutex
|
||||
// rejectedTargets and rejectedDisks collect the destinations that refused a
|
||||
// copy this attempt; completedMoves records applied moves so a retry can
|
||||
// replay them onto a stale-heartbeat snapshot; sawOtherErr marks any
|
||||
// failure that is not a "no space left" refusal and makes the run
|
||||
// non-retryable.
|
||||
rejectedTargets map[string]bool
|
||||
rejectedDisks map[string]map[uint32]bool
|
||||
completedMoves []ecbalancer.Move
|
||||
sawOtherErr bool
|
||||
// plannedMoves is the move count of the last plan, for the empty-replan
|
||||
// check in EcBalance.
|
||||
plannedMoves int
|
||||
}
|
||||
|
||||
// excludeNodes is a set of server addresses kept out of the balance as copy/move
|
||||
@@ -803,6 +782,32 @@ type ecBalancer struct {
|
||||
// so a plain ec.balance -diskType=X never drags deliberately tiered shards of
|
||||
// other types onto X disks.
|
||||
func EcBalance(env *Env, collections []string, dc string, ecReplicaPlacement *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, ioBytePerSecond int64, applyBalancing bool, excludeNodes map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId, migratingVolumeIds []needle.VolumeId) (err error) {
|
||||
// collect all ec nodes
|
||||
allEcNodes, totalFreeEcSlots, err := CollectEcNodesForDC(env, dc, diskType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Drop excluded nodes (and the slots they contribute) before planning so they
|
||||
// can be neither a target nor a source for any move this balance plans.
|
||||
if len(excludeNodes) > 0 {
|
||||
kept := allEcNodes[:0]
|
||||
var excludedFreeSlots int
|
||||
for _, en := range allEcNodes {
|
||||
if _, skip := excludeNodes[pb.NewServerAddressFromDataNode(en.Info)]; skip {
|
||||
excludedFreeSlots += en.FreeEcSlot
|
||||
glog.V(0).Infof("EC balance excluding node %s: skipped as unreachable by the encode orphan sweep", en.Info.Id)
|
||||
continue
|
||||
}
|
||||
kept = append(kept, en)
|
||||
}
|
||||
allEcNodes = kept
|
||||
totalFreeEcSlots -= excludedFreeSlots
|
||||
}
|
||||
|
||||
if totalFreeEcSlots < 1 {
|
||||
return fmt.Errorf("no free ec shard slots. only %d left", totalFreeEcSlots)
|
||||
}
|
||||
|
||||
var volumeIdFilter map[uint32]bool
|
||||
if len(volumeIds) > 0 {
|
||||
@@ -819,277 +824,22 @@ func EcBalance(env *Env, collections []string, dc string, ecReplicaPlacement *su
|
||||
}
|
||||
}
|
||||
|
||||
// The planner's free-slot numbers come from heartbeat snapshots that can lag
|
||||
// the volume server's own accounting, so a planned target can refuse a copy
|
||||
// with "no space left" while shards move concurrently (issue #11753). The
|
||||
// server's refusal is authoritative: re-collect the topology, mark the
|
||||
// refusing targets no-receive, and plan again. They stay usable as sources.
|
||||
noReceiveTargets := make(map[string]bool)
|
||||
noReceiveDisks := make(map[string]map[uint32]bool)
|
||||
var completed []ecbalancer.Move
|
||||
var lastErr error
|
||||
for {
|
||||
// collect all ec nodes
|
||||
allEcNodes, totalFreeEcSlots, err := CollectEcNodesForDC(env, dc, diskType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ecb := &ecBalancer{
|
||||
env: env,
|
||||
ecNodes: allEcNodes,
|
||||
replicaPlacement: ecReplicaPlacement,
|
||||
applyBalancing: applyBalancing,
|
||||
maxParallelization: maxParallelization,
|
||||
ioBytePerSecond: ioBytePerSecond,
|
||||
diskType: diskType,
|
||||
volumeIds: volumeIdFilter,
|
||||
migratingVolumeIds: migrating,
|
||||
}
|
||||
|
||||
// Drop excluded nodes (and the slots they contribute) before planning so they
|
||||
// can be neither a target nor a source for any move this balance plans.
|
||||
if len(excludeNodes) > 0 {
|
||||
kept := allEcNodes[:0]
|
||||
var excludedFreeSlots int
|
||||
for _, en := range allEcNodes {
|
||||
if _, skip := excludeNodes[pb.NewServerAddressFromDataNode(en.Info)]; skip {
|
||||
excludedFreeSlots += en.FreeEcSlot
|
||||
glog.V(0).Infof("EC balance excluding node %s: skipped as unreachable by the encode orphan sweep", en.Info.Id)
|
||||
continue
|
||||
}
|
||||
kept = append(kept, en)
|
||||
}
|
||||
allEcNodes = kept
|
||||
totalFreeEcSlots -= excludedFreeSlots
|
||||
}
|
||||
// A refused destination's slots cannot take a copy, so they do not
|
||||
// count toward the free-space precheck.
|
||||
for _, en := range allEcNodes {
|
||||
if noReceiveTargets[en.Info.Id] {
|
||||
totalFreeEcSlots -= en.FreeEcSlot
|
||||
continue
|
||||
}
|
||||
for diskId := range noReceiveDisks[en.Info.Id] {
|
||||
if d := en.Disks[diskId]; d != nil {
|
||||
totalFreeEcSlots -= d.FreeEcSlots
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if totalFreeEcSlots < 1 {
|
||||
if lastErr != nil {
|
||||
return fmt.Errorf("no receiving targets remain: %w", lastErr)
|
||||
}
|
||||
return fmt.Errorf("no free ec shard slots. only %d left", totalFreeEcSlots)
|
||||
}
|
||||
|
||||
// A fresh collection can still describe an applied move through a
|
||||
// heartbeat that predates its source delete; replay the journal.
|
||||
applyCompletedMoves(allEcNodes, completed, diskType)
|
||||
|
||||
ecb := &ecBalancer{
|
||||
env: env,
|
||||
ecNodes: allEcNodes,
|
||||
replicaPlacement: ecReplicaPlacement,
|
||||
applyBalancing: applyBalancing,
|
||||
maxParallelization: maxParallelization,
|
||||
ioBytePerSecond: ioBytePerSecond,
|
||||
diskType: diskType,
|
||||
volumeIds: volumeIdFilter,
|
||||
migratingVolumeIds: migrating,
|
||||
noReceiveTargets: noReceiveTargets,
|
||||
noReceiveDisks: noReceiveDisks,
|
||||
}
|
||||
|
||||
if len(collections) == 0 {
|
||||
glog.V(1).Infof("WARNING: No collections to balance EC volumes across.\n")
|
||||
}
|
||||
err = ecb.balance(collections)
|
||||
if !applyBalancing {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
if lastErr != nil && ecb.plannedMoves == 0 {
|
||||
return fmt.Errorf("no receiving targets remain: %w", lastErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Replan while the failed attempt surfaced newly-refusing targets; a node
|
||||
// once marked can no longer be picked, so the loop terminates. Any other
|
||||
// failure aborts instead of being hidden by the retry.
|
||||
newRejects := ecb.mergeRejectedTargets(noReceiveTargets, noReceiveDisks)
|
||||
if len(newRejects) == 0 || ecb.hasOtherError() {
|
||||
return err
|
||||
}
|
||||
lastErr = err
|
||||
completed = append(completed, ecb.completedMoves...)
|
||||
fmt.Printf("ec.balance: %v reported no space left; re-collecting topology and re-planning without them as targets\n", newRejects)
|
||||
}
|
||||
}
|
||||
|
||||
// applyCompletedMoves replays already-applied moves onto a freshly collected
|
||||
// node list: heartbeats can lag, so the snapshot may still show a moved shard
|
||||
// on its deleted source or lack it on its mounted target.
|
||||
func applyCompletedMoves(ecNodes []*EcNode, moves []ecbalancer.Move, diskType types.DiskType) {
|
||||
if len(moves) == 0 {
|
||||
return
|
||||
}
|
||||
byID := make(map[string]*EcNode, len(ecNodes))
|
||||
for _, en := range ecNodes {
|
||||
byID[en.Info.Id] = en
|
||||
}
|
||||
for _, m := range moves {
|
||||
vid := needle.VolumeId(m.VolumeID)
|
||||
shardId := erasure_coding.ShardId(m.ShardID)
|
||||
if src := byID[m.SourceNode]; src != nil {
|
||||
removeEcShard(src, vid, shardId)
|
||||
}
|
||||
if m.Phase == "dedup" {
|
||||
continue
|
||||
}
|
||||
if dst := byID[m.TargetNode]; dst != nil && !ecNodeHasShard(dst, vid, shardId) {
|
||||
addEcShard(dst, m.Collection, vid, shardId, m.TargetDisk, diskType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ecNodeHasShard(ecNode *EcNode, vid needle.VolumeId, shardId erasure_coding.ShardId) bool {
|
||||
for _, diskInfo := range ecNode.Info.DiskInfos {
|
||||
for _, eci := range diskInfo.EcShardInfos {
|
||||
if needle.VolumeId(eci.Id) == vid && erasure_coding.ShardBits(eci.EcIndexBits).Has(shardId) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// removeEcShard drops the shard from the node model, including the per-disk
|
||||
// bookkeeping of the disk that reported it.
|
||||
func removeEcShard(en *EcNode, vid needle.VolumeId, shardId erasure_coding.ShardId) {
|
||||
for _, diskInfo := range en.Info.DiskInfos {
|
||||
for _, eci := range diskInfo.EcShardInfos {
|
||||
if needle.VolumeId(eci.Id) != vid || !erasure_coding.ShardBits(eci.EcIndexBits).Has(shardId) {
|
||||
continue
|
||||
}
|
||||
if d := en.Disks[eci.DiskId]; d != nil {
|
||||
if si := d.EcShards[vid]; si != nil {
|
||||
si.Delete(shardId)
|
||||
}
|
||||
d.EcShardCount--
|
||||
d.FreeEcSlots++
|
||||
}
|
||||
}
|
||||
}
|
||||
en.DeleteEcVolumeShards(vid, []erasure_coding.ShardId{shardId})
|
||||
}
|
||||
|
||||
// addEcShard places the shard on the disk the move used when that disk is
|
||||
// known; otherwise (TargetDisk 0, server-picked) it falls back to the node's
|
||||
// disk-type bucket.
|
||||
func addEcShard(en *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, diskId uint32, diskType types.DiskType) {
|
||||
d := en.Disks[diskId]
|
||||
if diskId == 0 || d == nil {
|
||||
en.AddEcVolumeShards(vid, collection, []erasure_coding.ShardId{shardId}, diskType)
|
||||
return
|
||||
}
|
||||
diskInfo := en.Info.DiskInfos[string(diskType)]
|
||||
if diskInfo == nil {
|
||||
diskInfo = &master_pb.DiskInfo{Type: string(diskType)}
|
||||
en.Info.DiskInfos[string(diskType)] = diskInfo
|
||||
}
|
||||
var eci *master_pb.VolumeEcShardInformationMessage
|
||||
for _, e := range diskInfo.EcShardInfos {
|
||||
if needle.VolumeId(e.Id) == vid && e.DiskId == diskId {
|
||||
eci = e
|
||||
break
|
||||
}
|
||||
}
|
||||
if eci == nil {
|
||||
eci = &master_pb.VolumeEcShardInformationMessage{
|
||||
Id: uint32(vid),
|
||||
Collection: collection,
|
||||
DiskId: diskId,
|
||||
DiskType: string(diskType),
|
||||
}
|
||||
diskInfo.EcShardInfos = append(diskInfo.EcShardInfos, eci)
|
||||
}
|
||||
si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
|
||||
before := si.Count()
|
||||
si.Set(erasure_coding.NewShardInfo(shardId, 0))
|
||||
eci.EcIndexBits = si.Bitmap()
|
||||
eci.ShardSizes = si.SizesInt64()
|
||||
added := si.Count() - before
|
||||
|
||||
en.FreeEcSlot -= added
|
||||
d.EcShardCount += added
|
||||
d.FreeEcSlots -= added
|
||||
if d.EcShards == nil {
|
||||
d.EcShards = make(map[needle.VolumeId]*erasure_coding.ShardsInfo)
|
||||
}
|
||||
if d.EcShards[vid] == nil {
|
||||
d.EcShards[vid] = erasure_coding.NewShardsInfo()
|
||||
}
|
||||
d.EcShards[vid].Set(erasure_coding.NewShardInfo(shardId, 0))
|
||||
}
|
||||
|
||||
// mergeRejectedTargets folds the destinations this run saw refuse into the
|
||||
// no-receive sets and returns the newly added ids; an empty result means no
|
||||
// progress is possible.
|
||||
func (ecb *ecBalancer) mergeRejectedTargets(noReceiveNodes map[string]bool, noReceiveDisks map[string]map[uint32]bool) (added []string) {
|
||||
ecb.mu.Lock()
|
||||
defer ecb.mu.Unlock()
|
||||
for id := range ecb.rejectedTargets {
|
||||
if !noReceiveNodes[id] {
|
||||
noReceiveNodes[id] = true
|
||||
added = append(added, id)
|
||||
}
|
||||
}
|
||||
for id, disks := range ecb.rejectedDisks {
|
||||
if noReceiveNodes[id] {
|
||||
continue
|
||||
}
|
||||
if noReceiveDisks[id] == nil {
|
||||
noReceiveDisks[id] = make(map[uint32]bool)
|
||||
}
|
||||
for diskId := range disks {
|
||||
if !noReceiveDisks[id][diskId] {
|
||||
noReceiveDisks[id][diskId] = true
|
||||
added = append(added, fmt.Sprintf("%s:disk%d", id, diskId))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(added)
|
||||
return
|
||||
}
|
||||
|
||||
func (ecb *ecBalancer) hasOtherError() bool {
|
||||
ecb.mu.Lock()
|
||||
defer ecb.mu.Unlock()
|
||||
return ecb.sawOtherErr
|
||||
}
|
||||
|
||||
// recordMoveResult journals an apply-mode move outcome: a success is replayed
|
||||
// onto the next attempt's snapshot, a "no space left" marks the target
|
||||
// no-receive, and any other failure makes the run non-retryable.
|
||||
func (ecb *ecBalancer) recordMoveResult(m ecbalancer.Move, err error) {
|
||||
ecb.mu.Lock()
|
||||
defer ecb.mu.Unlock()
|
||||
switch {
|
||||
case err == nil:
|
||||
ecb.completedMoves = append(ecb.completedMoves, m)
|
||||
case m.TargetNode != "" && volume_move.IsNoSpaceLeftError(err):
|
||||
if m.TargetDisk > 0 {
|
||||
// The refusal convicts only the named disk; the node's other
|
||||
// disks may still have room.
|
||||
if ecb.rejectedDisks == nil {
|
||||
ecb.rejectedDisks = make(map[string]map[uint32]bool)
|
||||
}
|
||||
if ecb.rejectedDisks[m.TargetNode] == nil {
|
||||
ecb.rejectedDisks[m.TargetNode] = make(map[uint32]bool)
|
||||
}
|
||||
ecb.rejectedDisks[m.TargetNode][m.TargetDisk] = true
|
||||
} else {
|
||||
// The server auto-picked and found no usable disk: the whole
|
||||
// node is out.
|
||||
if ecb.rejectedTargets == nil {
|
||||
ecb.rejectedTargets = make(map[string]bool)
|
||||
}
|
||||
ecb.rejectedTargets[m.TargetNode] = true
|
||||
}
|
||||
default:
|
||||
ecb.sawOtherErr = true
|
||||
if len(collections) == 0 {
|
||||
glog.V(1).Infof("WARNING: No collections to balance EC volumes across.\n")
|
||||
}
|
||||
return ecb.balance(collections)
|
||||
}
|
||||
|
||||
// defaultECRatio resolves a collection's EC data/parity counts, defaulting to
|
||||
@@ -1102,7 +852,7 @@ func defaultECRatio(_ string) (int, int) {
|
||||
// balance plans EC shard moves with the shared planner and executes them. When
|
||||
// collections is empty all collections present are balanced.
|
||||
func (ecb *ecBalancer) balance(collections []string) error {
|
||||
topo, volumeRatio, selected := toBalancerTopology(ecb.ecNodes, collections, ecb.diskType, ecb.volumeIds, ecb.migratingVolumeIds, ecb.noReceiveTargets, ecb.noReceiveDisks)
|
||||
topo, volumeRatio, selected := toBalancerTopology(ecb.ecNodes, collections, ecb.diskType, ecb.volumeIds, ecb.migratingVolumeIds)
|
||||
if len(ecb.volumeIds) > 0 {
|
||||
requested := make([]uint32, 0, len(ecb.volumeIds))
|
||||
for vid := range ecb.volumeIds {
|
||||
@@ -1146,7 +896,6 @@ func (ecb *ecBalancer) balance(collections []string) error {
|
||||
}
|
||||
fmt.Printf("planned %d ec shard move(s) and %d ec shard deletion(s)\n", len(moves)-deletions, deletions)
|
||||
}
|
||||
ecb.plannedMoves = len(moves)
|
||||
return ecb.executeMoves(moves)
|
||||
}
|
||||
|
||||
@@ -1158,7 +907,7 @@ func (ecb *ecBalancer) balance(collections []string) error {
|
||||
// (0,0 when unreported, e.g. always in OSS), which Plan prefers over the
|
||||
// collection ratio for mixed-ratio clusters, and the set of volume ids that made
|
||||
// it into the topology.
|
||||
func toBalancerTopology(ecNodes []*EcNode, collections []string, diskType types.DiskType, volumeIds map[uint32]bool, migratingVolumeIds map[uint32]bool, noReceiveTargets map[string]bool, noReceiveDisks map[string]map[uint32]bool) (*ecbalancer.Topology, func(collection string, vid uint32) (int, int), map[uint32]bool) {
|
||||
func toBalancerTopology(ecNodes []*EcNode, collections []string, diskType types.DiskType, volumeIds map[uint32]bool, migratingVolumeIds map[uint32]bool) (*ecbalancer.Topology, func(collection string, vid uint32) (int, int), map[uint32]bool) {
|
||||
allowed := make(map[string]bool, len(collections))
|
||||
for _, c := range collections {
|
||||
allowed[c] = true
|
||||
@@ -1175,17 +924,11 @@ func toBalancerTopology(ecNodes []*EcNode, collections []string, diskType types.
|
||||
for _, en := range ecNodes {
|
||||
rackKey := string(en.DC) + ":" + string(en.Rack)
|
||||
node := topo.AddNode(en.Info.Id, string(en.DC), rackKey, en.FreeEcSlot)
|
||||
if noReceiveTargets[en.Info.Id] {
|
||||
node.SetNoReceive()
|
||||
}
|
||||
// Group by physical machine (host) so shards spread across machines, not just
|
||||
// nodes; the id stays the node identity used for moves.
|
||||
node.SetHost(pb.NewServerAddressFromDataNode(en.Info).ToHost())
|
||||
for diskId, d := range en.Disks {
|
||||
node.AddDisk(diskId, d.DiskType, d.FreeEcSlots, d.EcShardCount)
|
||||
if noReceiveDisks[en.Info.Id][diskId] {
|
||||
node.SetDiskNoReceive(diskId)
|
||||
}
|
||||
}
|
||||
for diskTypeKey, diskInfo := range en.Info.DiskInfos {
|
||||
if diskInfo == nil {
|
||||
@@ -1326,16 +1069,14 @@ func (ecb *ecBalancer) executeMove(byID map[string]*EcNode, m ecbalancer.Move) e
|
||||
// thing making this safe -- and the plan saying so is not evidence. A
|
||||
// topology entry can name a location holding nothing, and deleting on
|
||||
// that basis removes the last copy. Confirm the keep node has it.
|
||||
err := verifyEcShardOnKeepNode(grpcDialOption, m.Collection, vid, m.KeepNode, shardId)
|
||||
if err := verifyEcShardOnKeepNode(grpcDialOption, m.Collection, vid, m.KeepNode, shardId); err != nil {
|
||||
return err
|
||||
}
|
||||
addr := pb.NewServerAddressFromDataNode(src.Info)
|
||||
if err == nil {
|
||||
err = UnmountEcShards(grpcDialOption, vid, addr, shardIds)
|
||||
if err := UnmountEcShards(grpcDialOption, vid, addr, shardIds); err != nil {
|
||||
return err
|
||||
}
|
||||
if err == nil {
|
||||
err = SourceServerDeleteEcShards(grpcDialOption, m.Collection, vid, addr, shardIds)
|
||||
}
|
||||
ecb.recordMoveResult(m, err)
|
||||
return err
|
||||
return SourceServerDeleteEcShards(grpcDialOption, m.Collection, vid, addr, shardIds)
|
||||
}
|
||||
|
||||
dst := byID[m.TargetNode]
|
||||
@@ -1351,9 +1092,7 @@ func (ecb *ecBalancer) executeMove(byID map[string]*EcNode, m ecbalancer.Move) e
|
||||
// Dry-run: update the in-memory model only.
|
||||
return MoveMountedShardToEcNode(ecb.env, src, m.Collection, vid, shardId, dst, m.TargetDisk, false, ecb.diskType)
|
||||
}
|
||||
err := ecb.applyShardMoveRPC(src, dst, m.Collection, vid, shardId, m.TargetDisk)
|
||||
ecb.recordMoveResult(m, err)
|
||||
return err
|
||||
return ecb.applyShardMoveRPC(src, dst, m.Collection, vid, shardId, m.TargetDisk)
|
||||
}
|
||||
|
||||
// applyShardMoveRPC copies a shard to the destination disk, verifies the
|
||||
|
||||
@@ -390,14 +390,6 @@ func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpat
|
||||
return doDeleteFolderChildren()
|
||||
}
|
||||
|
||||
var likeEscaper = strings.NewReplacer("!", "!!", "%", "!%", "_", "!_")
|
||||
|
||||
// LikePattern escapes LIKE wildcards so the prefix matches literally;
|
||||
// '!' pairs with the ESCAPE '!' clause emitted by the SqlGenerators.
|
||||
func LikePattern(prefix string) string {
|
||||
return likeEscaper.Replace(prefix) + "%"
|
||||
}
|
||||
|
||||
func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
|
||||
|
||||
db, bucket, shortPath, err := store.getTxOrDB(ctx, dirPath, true)
|
||||
@@ -410,7 +402,7 @@ func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context,
|
||||
sqlText = store.GetSqlListInclusive(bucket)
|
||||
}
|
||||
|
||||
rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), LikePattern(prefix), limit+1)
|
||||
rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", limit+1)
|
||||
if err != nil {
|
||||
return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
|
||||
}
|
||||
|
||||
@@ -52,24 +52,6 @@ func TestGetTxOrDBRealBucket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLikePattern(t *testing.T) {
|
||||
cases := []struct {
|
||||
prefix, want string
|
||||
}{
|
||||
{"", "%"},
|
||||
{"plain", "plain%"},
|
||||
{"call_03_KNl", "call!_03!_KNl%"},
|
||||
{"100%", "100!%%"},
|
||||
{"a!b", "a!!b%"},
|
||||
{`a\b`, `a\b%`},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := LikePattern(c.prefix); got != c.want {
|
||||
t.Errorf("LikePattern(%q) = %q, want %q", c.prefix, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitPoolForKv(t *testing.T) {
|
||||
cases := []struct {
|
||||
maxOpen, mainOpen, kvOpen int
|
||||
|
||||
@@ -458,11 +458,11 @@ func retriedStreamFetchChunkData(ctx context.Context, writer io.Writer, urlStrin
|
||||
localProcessed += writtenCount
|
||||
totalWritten += writtenCount
|
||||
})
|
||||
if writeErr != nil {
|
||||
err = writeErr
|
||||
if !shouldRetry {
|
||||
break
|
||||
}
|
||||
if !shouldRetry {
|
||||
if writeErr != nil {
|
||||
err = writeErr
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -787,23 +787,3 @@ func TestFetchWholeChunkUsesProvidedFilerJwt(t *testing.T) {
|
||||
return ""
|
||||
}))
|
||||
}
|
||||
|
||||
// A completed fetch whose destination write failed is a failed copy, not a
|
||||
// short success: the caller must see the write error, or a temp disk filling
|
||||
// mid-copy would still upload the truncated file.
|
||||
func TestRetriedStreamFetchChunkDataReportsWriteError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("0123456789"))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
written, err := retriedStreamFetchChunkData(context.Background(), &failOnWrite{}, []string{srv.URL + "/5,abc"}, "", nil, false, true, 0, 0, nil)
|
||||
assert.Error(t, err)
|
||||
assert.Zero(t, written)
|
||||
}
|
||||
|
||||
type failOnWrite struct{}
|
||||
|
||||
func (w *failOnWrite) Write(p []byte) (int, error) {
|
||||
return 0, fmt.Errorf("no space left")
|
||||
}
|
||||
|
||||
+1
-1
@@ -890,7 +890,7 @@ func (f *Filer) DeleteEmptyParentDirectories(ctx context.Context, dirPath util.F
|
||||
glog.V(2).InfofCtx(ctx, "DeleteEmptyParentDirectories: deleting empty directory %s", dirPath)
|
||||
parentDir, _ := dirPath.DirAndName()
|
||||
if dirEntry, findErr := f.FindEntry(ctx, dirPath); findErr == nil {
|
||||
if delErr := f.doDeleteEntryMetaAndData(f.emptyFolderDeleteContext(ctx, dirPath), dirEntry, false, false, nil); delErr == nil {
|
||||
if delErr := f.doDeleteEntryMetaAndData(ctx, dirEntry, false, false, nil); delErr == nil {
|
||||
// Successfully deleted, continue checking upwards
|
||||
f.DeleteEmptyParentDirectories(ctx, util.FullPath(parentDir), stopAtPath)
|
||||
} else {
|
||||
|
||||
@@ -80,27 +80,6 @@ func keepsRemoteObject(ctx context.Context) bool {
|
||||
return ctx.Value(keepRemoteObjectKey{}) != nil
|
||||
}
|
||||
|
||||
// emptyFolderDeleteContext makes deleting a folder that is empty in the filer
|
||||
// metadata-only when it is under a remote mount. A mount is populated lazily,
|
||||
// so the folder can still hold remote objects never read, and removing its
|
||||
// remote directory deletes every object under that prefix.
|
||||
func (f *Filer) emptyFolderDeleteContext(ctx context.Context, dirPath util.FullPath) context.Context {
|
||||
if f.IsUnderRemoteMount(dirPath) {
|
||||
return WithKeepRemoteObject(ctx)
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// IsUnderRemoteMount reports whether p is a remote mount directory or lies under one.
|
||||
func (f *Filer) IsUnderRemoteMount(p util.FullPath) bool {
|
||||
if f.RemoteStorage == nil {
|
||||
return false
|
||||
}
|
||||
// Mount rules are keyed with a trailing slash, so this also matches the mount directory itself.
|
||||
_, remoteLoc := f.RemoteStorage.FindMountDirectory(util.FullPath(string(p) + "/"))
|
||||
return remoteLoc != nil
|
||||
}
|
||||
|
||||
func deleteEventEntry(ctx context.Context, entry *Entry) *Entry {
|
||||
if !keepsRemoteObject(ctx) {
|
||||
return entry
|
||||
@@ -129,9 +108,6 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
|
||||
if ifNotModifiedAfter > 0 && entry.Attr.Mtime.Unix() > ifNotModifiedAfter {
|
||||
return nil
|
||||
}
|
||||
if !isRecursive && entry.IsDirectory() {
|
||||
ctx = f.emptyFolderDeleteContext(ctx, p)
|
||||
}
|
||||
isDeleteCollection := f.IsBucket(entry)
|
||||
collectionName := ""
|
||||
if isDeleteCollection {
|
||||
@@ -194,7 +170,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
|
||||
func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) {
|
||||
|
||||
var dirTombstoneTs int64
|
||||
if isRecursive && !keepsRemoteObject(ctx) {
|
||||
if isRecursive {
|
||||
// Tombstone the directory before its children: when the store drops
|
||||
// the subtree without listing it, or a child error aborts the walk,
|
||||
// the ancestor tombstone still covers every descendant.
|
||||
@@ -225,9 +201,7 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry
|
||||
|
||||
for _, sub := range entries {
|
||||
lastFileName = sub.Name()
|
||||
if !keepsRemoteObject(ctx) {
|
||||
f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano())
|
||||
}
|
||||
f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano())
|
||||
if sub.IsDirectory() {
|
||||
subIsDeletingBucket := f.IsBucket(sub)
|
||||
err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, isFromOtherCluster, nil, onHardLinkIdsFn)
|
||||
@@ -290,10 +264,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
|
||||
}
|
||||
}
|
||||
|
||||
// A metadata-only delete owes the remote nothing, so lazy reads stay open.
|
||||
if !keepsRemoteObject(ctx) {
|
||||
f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano())
|
||||
}
|
||||
f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano())
|
||||
|
||||
if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
|
||||
return fmt.Errorf("filer store delete: %w", storeDeletionErr)
|
||||
|
||||
@@ -18,15 +18,11 @@ import (
|
||||
|
||||
type lazyFetchContextKey struct{}
|
||||
|
||||
// lazyParentsContextKey marks a FindEntry issued while materializing missing
|
||||
// parents: that fetch's own parent walk would only re-stat the same gaps.
|
||||
type lazyParentsContextKey struct{}
|
||||
|
||||
// maybeLazyFetchFromRemote is called by FindEntry when the store returns no
|
||||
// entry for p. If p is under a remote-storage mount, it stats the remote
|
||||
// object, builds a filer Entry from the result, and persists it via
|
||||
// CreateEntry together with its missing parent directories: remote.uncache
|
||||
// and cache eviction find entries by walking down from the mount directory.
|
||||
// CreateEntry with SkipCheckParentDirectory so phantom parent directories
|
||||
// under the mount are not required.
|
||||
//
|
||||
// On a CreateEntry failure after a successful StatFile the in-memory entry is
|
||||
// still returned (availability over consistency); the singleflight key is
|
||||
@@ -131,10 +127,7 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
|
||||
glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted during remote stat", p)
|
||||
return lazyFetchResult{nil}, nil
|
||||
}
|
||||
if ctx.Value(lazyParentsContextKey{}) == nil {
|
||||
f.fetchMissingRemoteParents(persistBaseCtx, mountDir, p)
|
||||
}
|
||||
saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, false, f.MaxFilenameLength)
|
||||
saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength)
|
||||
if saveErr != nil {
|
||||
glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)
|
||||
f.lazyFetchGroup.Forget(key)
|
||||
@@ -164,25 +157,6 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (
|
||||
return result.entry, nil
|
||||
}
|
||||
|
||||
// fetchMissingRemoteParents looks up p's missing parents under mountDir, top
|
||||
// down, through lazy fetch. A remote object stored at a parent's key is then
|
||||
// materialized first, so creating p promotes it to a prefix object instead of
|
||||
// hiding it behind a plain directory. Each missing parent costs one remote
|
||||
// stat, once per directory.
|
||||
func (f *Filer) fetchMissingRemoteParents(ctx context.Context, mountDir, p util.FullPath) {
|
||||
var missing []util.FullPath
|
||||
for dir, _ := p.DirAndName(); len(dir) > len(mountDir); dir, _ = util.FullPath(dir).DirAndName() {
|
||||
if _, err := f.Store.FindEntry(ctx, util.FullPath(dir)); !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
break
|
||||
}
|
||||
missing = append(missing, util.FullPath(dir))
|
||||
}
|
||||
parentCtx := context.WithValue(ctx, lazyParentsContextKey{}, true)
|
||||
for i := len(missing) - 1; i >= 0; i-- {
|
||||
f.FindEntry(parentCtx, missing[i])
|
||||
}
|
||||
}
|
||||
|
||||
// retractLazyRemoteEntry deletes the entry at entry.FullPath only when it is
|
||||
// still the entry a lazy remote read just materialized — a concurrent write
|
||||
// may have replaced it, and deleting by path alone would take that write down.
|
||||
|
||||
@@ -103,10 +103,6 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
|
||||
persistCtx := context.WithValue(context.WithoutCancel(ctx), lazyListContextKey{}, true)
|
||||
persistCtx = context.WithValue(persistCtx, lazyFetchContextKey{}, true)
|
||||
|
||||
// Entries listed under p must stay reachable by a walk from the mount
|
||||
// directory, the same reason the lazy fetch fills in missing parents.
|
||||
f.fetchMissingRemoteParents(context.WithoutCancel(ctx), mountDir, p)
|
||||
|
||||
listErr := client.ListDirectory(persistCtx, objectLoc, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
|
||||
childPath := p.Child(name)
|
||||
|
||||
@@ -219,7 +215,7 @@ func (f *Filer) updateDirectoryListingSyncedAt(ctx context.Context, p util.FullP
|
||||
dirEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
dirEntry.Extended[xattrRemoteListingSyncedAt] = []byte(fmt.Sprintf("%d", syncTime.Unix()))
|
||||
if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, false, f.MaxFilenameLength); saveErr != nil {
|
||||
if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {
|
||||
glog.Warningf("maybeLazyListFromRemote: create dir synced_at for %s: %v", p, saveErr)
|
||||
}
|
||||
return
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
@@ -36,7 +35,6 @@ type stubFilerStore struct {
|
||||
kv map[string][]byte
|
||||
insertErr error
|
||||
findErr error
|
||||
findErrByPath map[string]error
|
||||
kvPutErr error
|
||||
kvGetErr error
|
||||
kvDeleteErr error
|
||||
@@ -202,9 +200,6 @@ func (s *stubFilerStore) FindEntry(_ context.Context, p util.FullPath) (*Entry,
|
||||
if s.findErr != nil {
|
||||
return nil, s.findErr
|
||||
}
|
||||
if err := s.findErrByPath[string(p)]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e, ok := s.entries[string(p)]; ok {
|
||||
return e, nil
|
||||
}
|
||||
@@ -345,11 +340,6 @@ func putConfigEntry(store *stubFilerStore, path string, content []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// putDirEntry stores a plain directory entry, so a lazy fetch below it finds its parent in place.
|
||||
func putDirEntry(store *stubFilerStore, path string) {
|
||||
store.entries[path] = &Entry{FullPath: util.FullPath(path), Attr: Attr{Mode: os.ModeDir | 0755}}
|
||||
}
|
||||
|
||||
// --- tests ---
|
||||
|
||||
func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) {
|
||||
@@ -385,199 +375,6 @@ func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) {
|
||||
assert.Equal(t, []byte("zstd"), stored.Extended["Content-Encoding"])
|
||||
}
|
||||
|
||||
// pathStatRemoteClient stats only the objects it holds, keyed by remote path.
|
||||
type pathStatRemoteClient struct {
|
||||
stubRemoteClient
|
||||
objects map[string]*filer_pb.RemoteEntry
|
||||
statCalls []string
|
||||
}
|
||||
|
||||
func (c *pathStatRemoteClient) StatFile(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
|
||||
c.statCalls = append(c.statCalls, loc.Path)
|
||||
if remoteEntry, found := c.objects[loc.Path]; found {
|
||||
return remoteEntry, nil
|
||||
}
|
||||
return nil, remote_storage.ErrRemoteObjectNotFound
|
||||
}
|
||||
|
||||
func newPathStatFiler(t *testing.T, storageType string, client remote_storage.RemoteStorageClient) (*Filer, *stubFilerStore) {
|
||||
t.Helper()
|
||||
t.Cleanup(registerStubMaker(t, storageType, client))
|
||||
conf := &remote_pb.RemoteConf{Name: "mystore", Type: storageType}
|
||||
rs := NewFilerRemoteStorage()
|
||||
rs.storageNameToConf[conf.Name] = conf
|
||||
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
|
||||
Name: "mystore",
|
||||
Bucket: "mybucket",
|
||||
Path: "/",
|
||||
})
|
||||
store := newStubFilerStore()
|
||||
return newTestFiler(t, store, rs), store
|
||||
}
|
||||
|
||||
func TestMaybeLazyFetchFromRemote_PersistsParentDirectories(t *testing.T) {
|
||||
stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{
|
||||
"/a/b/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234},
|
||||
"/a/b/other.bin": {RemoteMtime: 1700000000, RemoteSize: 5678},
|
||||
}}
|
||||
f, store := newPathStatFiler(t, "stub_lazy_parents", stub)
|
||||
|
||||
const filePath = "/buckets/mybucket/a/b/file.bin"
|
||||
entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.Equal(t, []string{"/a/b/file.bin", "/a", "/a/b"}, stub.statCalls, "each missing parent is stat'ed once")
|
||||
|
||||
for _, dir := range []string{"/buckets/mybucket/a", "/buckets/mybucket/a/b"} {
|
||||
dirEntry := store.getEntry(dir)
|
||||
require.NotNil(t, dirEntry, "parent %s not persisted", dir)
|
||||
assert.True(t, dirEntry.IsDirectory(), "%s should be a directory", dir)
|
||||
assert.Nil(t, dirEntry.Remote, "%s holds no remote object", dir)
|
||||
}
|
||||
|
||||
// A sibling finds its parents in place and stats only itself.
|
||||
stub.statCalls = nil
|
||||
_, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/other.bin")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/a/b/other.bin"}, stub.statCalls)
|
||||
|
||||
// Once the content is cached, eviction walks down from the mount and must reach it.
|
||||
stored := store.getEntry(filePath)
|
||||
require.NotNil(t, stored)
|
||||
stored.Chunks = []*filer_pb.FileChunk{{FileId: "1,01637037d6", Size: 1234}}
|
||||
stored.Remote.LastLocalSyncTsNs = time.Now().UnixNano()
|
||||
require.NoError(t, store.UpdateEntry(context.Background(), stored))
|
||||
|
||||
evictable := f.ListEvictableRemoteEntries(context.Background(), []util.FullPath{"/buckets/mybucket"}, 0)
|
||||
require.Len(t, evictable, 1)
|
||||
assert.Equal(t, util.FullPath(filePath), evictable[0].FullPath)
|
||||
}
|
||||
|
||||
func TestMaybeLazyFetchFromRemote_ParentLookupStopsAtStoreError(t *testing.T) {
|
||||
stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{
|
||||
"/a/b/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234},
|
||||
}}
|
||||
f, store := newPathStatFiler(t, "stub_lazy_parent_store_error", stub)
|
||||
store.findErrByPath = map[string]error{"/buckets/mybucket/a/b": errors.New("store unavailable")}
|
||||
|
||||
_, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/file.bin")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/a/b/file.bin"}, stub.statCalls, "a parent the store cannot read is not treated as missing")
|
||||
}
|
||||
|
||||
func TestMaybeLazyFetchFromRemote_PromotesRemoteObjectAtParentKey(t *testing.T) {
|
||||
stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{
|
||||
"/reports": {RemoteMtime: 1700000000, RemoteSize: 10},
|
||||
"/reports/2026.csv": {RemoteMtime: 1700000000, RemoteSize: 20},
|
||||
}}
|
||||
f, store := newPathStatFiler(t, "stub_lazy_prefix_object", stub)
|
||||
|
||||
_, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/reports/2026.csv")
|
||||
require.NoError(t, err)
|
||||
|
||||
parent := store.getEntry("/buckets/mybucket/reports")
|
||||
require.NotNil(t, parent)
|
||||
assert.True(t, parent.IsDirectory())
|
||||
assert.Equal(t, []byte("true"), parent.Extended[s3_constants.SeaweedFSPrefixObject], "the remote object at the parent key must stay readable")
|
||||
require.NotNil(t, parent.Remote)
|
||||
assert.Equal(t, int64(10), parent.Remote.RemoteSize)
|
||||
}
|
||||
|
||||
func TestMaybeLazyFetchFromRemote_StatsEachParentOnce(t *testing.T) {
|
||||
stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{
|
||||
"/a/b/c/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234},
|
||||
}}
|
||||
f, _ := newPathStatFiler(t, "stub_lazy_parent_stats", stub)
|
||||
|
||||
_, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/c/file.bin")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/a/b/c/file.bin", "/a", "/a/b", "/a/b/c"}, stub.statCalls,
|
||||
"a parent still missing after its own stat must not be re-stat'ed by the next lookup")
|
||||
}
|
||||
|
||||
func TestMaybeLazyListFromRemote_PersistsParentDirectories(t *testing.T) {
|
||||
stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{}}
|
||||
stub.listDirFn = func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {
|
||||
return visitFn("/", "file.bin", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 1234})
|
||||
}
|
||||
f, store := newPathStatFiler(t, "stub_lazy_list_parents", stub)
|
||||
f.RemoteStorage.mapDirectoryToRemoteStorage("/buckets/listmount", &remote_pb.RemoteStorageLocation{
|
||||
Name: "mystore",
|
||||
Bucket: "listmount",
|
||||
Path: "/",
|
||||
ListingCacheTtlSeconds: 300,
|
||||
})
|
||||
|
||||
f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/listmount/a/b"))
|
||||
|
||||
assert.Equal(t, []string{"/a"}, stub.statCalls, "each missing parent is stat'ed once")
|
||||
for _, dir := range []string{"/buckets/listmount", "/buckets/listmount/a", "/buckets/listmount/a/b"} {
|
||||
dirEntry := store.getEntry(dir)
|
||||
require.NotNil(t, dirEntry, "%s not persisted", dir)
|
||||
assert.True(t, dirEntry.IsDirectory(), "%s should be a directory", dir)
|
||||
}
|
||||
|
||||
child := store.getEntry("/buckets/listmount/a/b/file.bin")
|
||||
require.NotNil(t, child)
|
||||
child.Chunks = []*filer_pb.FileChunk{{FileId: "1,01637037d6", Size: 1234}}
|
||||
child.Remote.LastLocalSyncTsNs = time.Now().UnixNano()
|
||||
require.NoError(t, store.UpdateEntry(context.Background(), child))
|
||||
|
||||
evictable := f.ListEvictableRemoteEntries(context.Background(), []util.FullPath{"/buckets/listmount"}, 0)
|
||||
require.Len(t, evictable, 1)
|
||||
assert.Equal(t, util.FullPath("/buckets/listmount/a/b/file.bin"), evictable[0].FullPath)
|
||||
}
|
||||
|
||||
func TestDeleteEntryMetaAndData_EmptyFolderUnderMountKeepsRemotePrefix(t *testing.T) {
|
||||
stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{
|
||||
"/dir/unread.bin": {RemoteMtime: 1700000000, RemoteSize: 1},
|
||||
}}
|
||||
f, store := newPathStatFiler(t, "stub_lazy_empty_folder", stub)
|
||||
dirPath := util.FullPath("/buckets/mybucket/dir")
|
||||
putDirEntry(store, string(dirPath))
|
||||
|
||||
require.NoError(t, f.DeleteEntryMetaAndData(context.Background(), dirPath, false, false, false, false, nil, 0))
|
||||
|
||||
assert.Empty(t, stub.removeCalls, "an empty local folder must not remove its remote prefix")
|
||||
assert.Nil(t, store.getEntry(string(dirPath)))
|
||||
entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/unread.bin")
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, entry, "a metadata-only delete must not block lazy reads under the folder")
|
||||
}
|
||||
|
||||
func TestDeleteEmptyParentDirectories_UnderMountKeepsRemotePrefix(t *testing.T) {
|
||||
stub := &pathStatRemoteClient{}
|
||||
f, store := newPathStatFiler(t, "stub_lazy_expired_parents", stub)
|
||||
f.DirBucketsPath = "/buckets"
|
||||
putDirEntry(store, "/buckets/mybucket/a")
|
||||
putDirEntry(store, "/buckets/mybucket/a/b")
|
||||
|
||||
f.DeleteEmptyParentDirectories(context.Background(), "/buckets/mybucket/a/b", "/buckets")
|
||||
|
||||
assert.Empty(t, stub.removeCalls, "an empty local folder must not remove its remote prefix")
|
||||
assert.Nil(t, store.getEntry("/buckets/mybucket/a/b"))
|
||||
assert.Nil(t, store.getEntry("/buckets/mybucket/a"))
|
||||
}
|
||||
|
||||
func TestIsUnderRemoteMount(t *testing.T) {
|
||||
rs := NewFilerRemoteStorage()
|
||||
rs.mapDirectoryToRemoteStorage("/buckets/mirror", &remote_pb.RemoteStorageLocation{
|
||||
Name: "mystore",
|
||||
Bucket: "mirror",
|
||||
Path: "/",
|
||||
})
|
||||
f := newTestFiler(t, newStubFilerStore(), rs)
|
||||
|
||||
for path, want := range map[util.FullPath]bool{
|
||||
"/buckets/mirror": true,
|
||||
"/buckets/mirror/a/b": true,
|
||||
"/buckets/mirror2/a": false,
|
||||
"/buckets/local/a": false,
|
||||
} {
|
||||
assert.Equal(t, want, f.IsUnderRemoteMount(path), path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeLazyFetchFromRemote_NotUnderMount(t *testing.T) {
|
||||
rs := NewFilerRemoteStorage()
|
||||
store := newStubFilerStore()
|
||||
|
||||
@@ -407,7 +407,7 @@ func (f *Filer) onRemoteDeletionEvents(event *filer_pb.SubscribeMetadataResponse
|
||||
if message == nil {
|
||||
return
|
||||
}
|
||||
if message.OldEntry != nil && !IsMetadataOnlyDelete(message.OldEntry) {
|
||||
if message.OldEntry != nil {
|
||||
sourcePath := filer_pb.MetadataEventSourceFullPath(event)
|
||||
if message.NewEntry == nil || sourcePath != filer_pb.MetadataEventTargetFullPath(event) {
|
||||
if f.RemoteStorage != nil && f.remoteTombstones != nil {
|
||||
|
||||
@@ -187,7 +187,6 @@ func TestMaybeLazyFetchFromRemote_SyncOffsetReleasesTombstone(t *testing.T) {
|
||||
},
|
||||
}
|
||||
f, store := newMountedTestFiler(t, storageType, stub, 0)
|
||||
putDirEntry(store, "/buckets/mybucket/dir")
|
||||
|
||||
filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
|
||||
deleteTsNs := time.Now().UnixNano()
|
||||
@@ -223,7 +222,6 @@ func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) {
|
||||
},
|
||||
}
|
||||
f, store := newMountedTestFiler(t, storageType, stub, 0)
|
||||
putDirEntry(store, "/buckets/mybucket/dir")
|
||||
|
||||
filePath := util.FullPath("/buckets/mybucket/dir/a.txt")
|
||||
store.entries[string(filePath)] = &Entry{
|
||||
@@ -259,35 +257,6 @@ func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) {
|
||||
assert.Equal(t, 1, stub.statCalls)
|
||||
}
|
||||
|
||||
func TestOnMetadataChangeEvent_MetadataOnlyDeleteSkipsTombstone(t *testing.T) {
|
||||
const storageType = "stub_tomb_metadata_only"
|
||||
stub := &countingRemoteClient{
|
||||
stubRemoteClient: stubRemoteClient{
|
||||
statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},
|
||||
},
|
||||
}
|
||||
f, store := newMountedTestFiler(t, storageType, stub, 0)
|
||||
putDirEntry(store, "/buckets/mybucket/dir")
|
||||
|
||||
f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/buckets/mybucket",
|
||||
TsNs: time.Now().UnixNano(),
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{
|
||||
Name: "dir",
|
||||
IsDirectory: true,
|
||||
Extended: map[string][]byte{ExtKeepRemoteObjectKey: []byte("true")},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// nothing is owed to the remote, so objects under the folder stay readable
|
||||
entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/a.txt")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.Equal(t, 1, stub.statCalls)
|
||||
}
|
||||
|
||||
func TestOnMetadataChangeEvent_PeerDeleteTombstones(t *testing.T) {
|
||||
const storageType = "stub_tomb_peer"
|
||||
stub := &countingRemoteClient{
|
||||
|
||||
@@ -64,12 +64,12 @@ func (gen *SqlGenMysql) nameExpr() string {
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
|
||||
name := gen.nameExpr()
|
||||
return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
|
||||
return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
|
||||
name := gen.nameExpr()
|
||||
return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s >= ? AND `directory` = ? AND %s LIKE ? ESCAPE '!' ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
|
||||
return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s >= ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
|
||||
}
|
||||
|
||||
func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
|
||||
|
||||
@@ -65,15 +65,6 @@ func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSqlEscapesLikeWildcards(t *testing.T) {
|
||||
gen := &SqlGenMysql{}
|
||||
for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
|
||||
if !strings.Contains(got, "LIKE ? ESCAPE '!'") {
|
||||
t.Fatalf("list query must declare the escape char matching LikePattern, got: %s", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBinaryCollation(t *testing.T) {
|
||||
binary := []string{"", "binary", "utf8mb4_bin", "utf8mb3_bin", "latin1_bin", "UTF8MB4_BIN"}
|
||||
for _, c := range binary {
|
||||
|
||||
@@ -77,12 +77,12 @@ func (gen *SqlGenPostgres) nameExpr() string {
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
|
||||
name := gen.nameExpr()
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
|
||||
name := gen.nameExpr()
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ESCAPE '!' ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
|
||||
return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
|
||||
}
|
||||
|
||||
func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
|
||||
|
||||
@@ -62,15 +62,6 @@ func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSqlEscapesLikeWildcards(t *testing.T) {
|
||||
gen := &SqlGenPostgres{}
|
||||
for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
|
||||
if !strings.Contains(got, "like $4 ESCAPE '!'") {
|
||||
t.Fatalf("list query must declare the escape char matching LikePattern, got: %s", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsByteOrderedCollation(t *testing.T) {
|
||||
byteOrdered := []string{"C", "POSIX", "c", "C.UTF-8", "C.utf8"}
|
||||
for _, c := range byteOrdered {
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
//go:build sqlite
|
||||
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer/mysql"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func TestListDirectoryPrefixedEntriesEscapesLikeWildcards(t *testing.T) {
|
||||
db, err := sql.Open("sqlite", "file::memory:?cache=shared")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE filemeta (dirhash BIGINT, name VARCHAR(1000), directory TEXT, meta BLOB, PRIMARY KEY (dirhash, name))`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
store := &SqliteStore{}
|
||||
store.SqlGenerator = &mysql.SqlGenMysql{}
|
||||
store.DB = db
|
||||
|
||||
ctx := context.Background()
|
||||
names := []string{
|
||||
"call_03_KNlHPt6grkZMe4WZVwQi9710", // target: literal _ prefix
|
||||
"callX03XKNlZZZ", // would match if _ were a wildcard
|
||||
"call_99_other",
|
||||
"100%_done", // % and _ in the name
|
||||
"100aXdone", // would match prefix "100%_" if unescaped
|
||||
"bang!file",
|
||||
}
|
||||
for _, n := range names {
|
||||
entry := &filer.Entry{FullPath: util.FullPath("/dir/" + n)}
|
||||
if err := store.InsertEntry(ctx, entry); err != nil {
|
||||
t.Fatalf("insert %s: %v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
list := func(prefix string) []string {
|
||||
var got []string
|
||||
_, err := store.ListDirectoryPrefixedEntries(ctx, "/dir", "", false, 100, prefix, func(e *filer.Entry) (bool, error) {
|
||||
got = append(got, e.Name())
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list %q: %v", prefix, err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
if got := list("call_03_"); len(got) != 1 || got[0] != "call_03_KNlHPt6grkZMe4WZVwQi9710" {
|
||||
t.Fatalf("prefix with _: got %v", got)
|
||||
}
|
||||
if got := list("call_"); len(got) != 2 {
|
||||
t.Fatalf("prefix call_: got %v", got)
|
||||
}
|
||||
if got := list("100%_"); len(got) != 1 || got[0] != "100%_done" {
|
||||
t.Fatalf("prefix with %%: got %v", got)
|
||||
}
|
||||
if got := list("bang!"); len(got) != 1 || got[0] != "bang!file" {
|
||||
t.Fatalf("prefix with !: got %v", got)
|
||||
}
|
||||
if got := list(""); len(got) != len(names) {
|
||||
t.Fatalf("empty prefix: got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ const (
|
||||
|
||||
SELECT name, meta
|
||||
FROM ` + asql.DEFAULT_TABLE + `
|
||||
WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix ESCAPE '!'
|
||||
WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix
|
||||
ORDER BY name ASC LIMIT $limit;`
|
||||
|
||||
listInclusiveDirectoryQuery = `
|
||||
@@ -68,6 +68,6 @@ const (
|
||||
|
||||
SELECT name, meta
|
||||
FROM ` + asql.DEFAULT_TABLE + `
|
||||
WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix ESCAPE '!'
|
||||
WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix
|
||||
ORDER BY name ASC LIMIT $limit;`
|
||||
)
|
||||
|
||||
@@ -288,7 +288,7 @@ func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath
|
||||
table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
|
||||
table.ValueParam("$directory", types.UTF8Value(*shortDir)),
|
||||
table.ValueParam("$start_name", types.UTF8Value(startFileName)),
|
||||
table.ValueParam("$prefix", types.UTF8Value(abstract_sql.LikePattern(prefix))),
|
||||
table.ValueParam("$prefix", types.UTF8Value(prefix+"%")),
|
||||
table.ValueParam("$limit", types.Uint64Value(uint64(chunkLimit))),
|
||||
)
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
@@ -203,14 +202,6 @@ func (m *Mover) DeleteEcShards(ctx context.Context, volumeId needle.VolumeId, co
|
||||
})
|
||||
}
|
||||
|
||||
// IsNoSpaceLeftError matches the volume server's refusal when no local disk
|
||||
// can take the copied shard ("no space left" from FindEcShardTargetLocation,
|
||||
// an ENOSPC writing to a caller-pinned disk, and the Rust server's same
|
||||
// reply), however the move chain wraps it.
|
||||
func IsNoSpaceLeftError(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), "no space left")
|
||||
}
|
||||
|
||||
// RemoveEcShards unmounts then deletes shards in place — the dedup path for a
|
||||
// shard that already has a copy elsewhere.
|
||||
func (m *Mover) RemoveEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
|
||||
|
||||
@@ -305,11 +305,6 @@ message VolumeGrowRequest {
|
||||
string data_node = 7;
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
string disk_type = 9;
|
||||
// If > 0, grow only when the average fill ratio of the layout's writable
|
||||
// volumes has reached this threshold (e.g. 0.6). Lets callers pre-grow on
|
||||
// fill level with one call, instead of listing volumes to compute the
|
||||
// ratio themselves. Ignored when there are no writable volumes.
|
||||
float grow_avg_fill_threshold = 10;
|
||||
}
|
||||
|
||||
message AssignResponse {
|
||||
@@ -616,9 +611,4 @@ message RaftLeadershipTransferResponse {
|
||||
}
|
||||
|
||||
message VolumeGrowResponse {
|
||||
// Average fill ratio of the layout's writable volumes at request time
|
||||
// (0 when there are none). Reported whether or not growth was triggered.
|
||||
float avg_fill = 1;
|
||||
// Number of writable volumes at request time.
|
||||
uint32 writable_count = 2;
|
||||
}
|
||||
|
||||
@@ -1739,13 +1739,8 @@ type VolumeGrowRequest struct {
|
||||
DataNode string `protobuf:"bytes,7,opt,name=data_node,json=dataNode,proto3" json:"data_node,omitempty"`
|
||||
MemoryMapMaxSizeMb uint32 `protobuf:"varint,8,opt,name=memory_map_max_size_mb,json=memoryMapMaxSizeMb,proto3" json:"memory_map_max_size_mb,omitempty"`
|
||||
DiskType string `protobuf:"bytes,9,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"`
|
||||
// If > 0, grow only when the average fill ratio of the layout's writable
|
||||
// volumes has reached this threshold (e.g. 0.6). Lets callers pre-grow on
|
||||
// fill level with one call, instead of listing volumes to compute the
|
||||
// ratio themselves. Ignored when there are no writable volumes.
|
||||
GrowAvgFillThreshold float32 `protobuf:"fixed32,10,opt,name=grow_avg_fill_threshold,json=growAvgFillThreshold,proto3" json:"grow_avg_fill_threshold,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *VolumeGrowRequest) Reset() {
|
||||
@@ -1841,13 +1836,6 @@ func (x *VolumeGrowRequest) GetDiskType() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *VolumeGrowRequest) GetGrowAvgFillThreshold() float32 {
|
||||
if x != nil {
|
||||
return x.GrowAvgFillThreshold
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type AssignResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Fid string `protobuf:"bytes,1,opt,name=fid,proto3" json:"fid,omitempty"`
|
||||
@@ -4671,12 +4659,7 @@ func (x *RaftLeadershipTransferResponse) GetNewLeader() string {
|
||||
}
|
||||
|
||||
type VolumeGrowResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// Average fill ratio of the layout's writable volumes at request time
|
||||
// (0 when there are none). Reported whether or not growth was triggered.
|
||||
AvgFill float32 `protobuf:"fixed32,1,opt,name=avg_fill,json=avgFill,proto3" json:"avg_fill,omitempty"`
|
||||
// Number of writable volumes at request time.
|
||||
WritableCount uint32 `protobuf:"varint,2,opt,name=writable_count,json=writableCount,proto3" json:"writable_count,omitempty"`
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -4711,20 +4694,6 @@ func (*VolumeGrowResponse) Descriptor() ([]byte, []int) {
|
||||
return file_master_proto_rawDescGZIP(), []int{68}
|
||||
}
|
||||
|
||||
func (x *VolumeGrowResponse) GetAvgFill() float32 {
|
||||
if x != nil {
|
||||
return x.AvgFill
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *VolumeGrowResponse) GetWritableCount() uint32 {
|
||||
if x != nil {
|
||||
return x.WritableCount
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type SuperBlockExtra_ErasureCoding struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Data uint32 `protobuf:"varint,1,opt,name=data,proto3" json:"data,omitempty"`
|
||||
@@ -5260,7 +5229,7 @@ const file_master_proto_rawDesc = "" +
|
||||
"\x15writable_volume_count\x18\t \x01(\rR\x13writableVolumeCount\x12\x1b\n" +
|
||||
"\tdisk_type\x18\n" +
|
||||
" \x01(\tR\bdiskType\x12,\n" +
|
||||
"\x12expected_data_size\x18\v \x01(\x04R\x10expectedDataSize\"\xf5\x02\n" +
|
||||
"\x12expected_data_size\x18\v \x01(\x04R\x10expectedDataSize\"\xbe\x02\n" +
|
||||
"\x11VolumeGrowRequest\x122\n" +
|
||||
"\x15writable_volume_count\x18\x01 \x01(\rR\x13writableVolumeCount\x12 \n" +
|
||||
"\vreplication\x18\x02 \x01(\tR\vreplication\x12\x1e\n" +
|
||||
@@ -5273,9 +5242,7 @@ const file_master_proto_rawDesc = "" +
|
||||
"\x04rack\x18\x06 \x01(\tR\x04rack\x12\x1b\n" +
|
||||
"\tdata_node\x18\a \x01(\tR\bdataNode\x122\n" +
|
||||
"\x16memory_map_max_size_mb\x18\b \x01(\rR\x12memoryMapMaxSizeMb\x12\x1b\n" +
|
||||
"\tdisk_type\x18\t \x01(\tR\bdiskType\x125\n" +
|
||||
"\x17grow_avg_fill_threshold\x18\n" +
|
||||
" \x01(\x02R\x14growAvgFillThreshold\"\xc4\x01\n" +
|
||||
"\tdisk_type\x18\t \x01(\tR\bdiskType\"\xc4\x01\n" +
|
||||
"\x0eAssignResponse\x12\x10\n" +
|
||||
"\x03fid\x18\x01 \x01(\tR\x03fid\x12\x14\n" +
|
||||
"\x05count\x18\x04 \x01(\x04R\x05count\x12\x14\n" +
|
||||
@@ -5510,10 +5477,8 @@ const file_master_proto_rawDesc = "" +
|
||||
"\x1eRaftLeadershipTransferResponse\x12'\n" +
|
||||
"\x0fprevious_leader\x18\x01 \x01(\tR\x0epreviousLeader\x12\x1d\n" +
|
||||
"\n" +
|
||||
"new_leader\x18\x02 \x01(\tR\tnewLeader\"V\n" +
|
||||
"\x12VolumeGrowResponse\x12\x19\n" +
|
||||
"\bavg_fill\x18\x01 \x01(\x02R\aavgFill\x12%\n" +
|
||||
"\x0ewritable_count\x18\x02 \x01(\rR\rwritableCount2\xf1\x12\n" +
|
||||
"new_leader\x18\x02 \x01(\tR\tnewLeader\"\x14\n" +
|
||||
"\x12VolumeGrowResponse2\xf1\x12\n" +
|
||||
"\aSeaweed\x12I\n" +
|
||||
"\rSendHeartbeat\x12\x14.master_pb.Heartbeat\x1a\x1c.master_pb.HeartbeatResponse\"\x00(\x010\x01\x12X\n" +
|
||||
"\rKeepConnected\x12\x1f.master_pb.KeepConnectedRequest\x1a .master_pb.KeepConnectedResponse\"\x00(\x010\x01\x12Q\n" +
|
||||
|
||||
@@ -37,9 +37,6 @@ type HandlerBuildOptions struct {
|
||||
GrpcDialOption grpc.DialOption
|
||||
MaxExecute int
|
||||
WorkingDir string
|
||||
// WorkerAddress is the advertised weed worker -address (host:port).
|
||||
// It is not a volume-server URL; handlers match replicas by hostname.
|
||||
WorkerAddress string
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -176,8 +176,6 @@ func TestBuildBucketMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetBucketMetadata(t *testing.T) {
|
||||
original := loadBucketMetadataFromFiler
|
||||
t.Cleanup(func() { loadBucketMetadataFromFiler = original })
|
||||
loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
|
||||
time.Sleep(time.Second)
|
||||
loadFilerBucketLock.Lock()
|
||||
|
||||
@@ -152,7 +152,7 @@ func TestBucketHasUserObjects_EmptyDirectories(t *testing.T) {
|
||||
wantHasUser: true,
|
||||
},
|
||||
{
|
||||
name: "empty *.versions folder and .uploads do not count",
|
||||
name: "bucket with only reserved folders (.uploads, *.versions)",
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/buckets/b": {
|
||||
{Name: s3_constants.MultipartUploadsFolder, IsDirectory: true},
|
||||
@@ -162,7 +162,7 @@ func TestBucketHasUserObjects_EmptyDirectories(t *testing.T) {
|
||||
wantHasUser: false,
|
||||
},
|
||||
{
|
||||
name: "nested versions directory with stored versions counts",
|
||||
name: "nested versions-named directories are internal",
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/buckets/b": {
|
||||
{Name: "logs", IsDirectory: true},
|
||||
@@ -174,19 +174,6 @@ func TestBucketHasUserObjects_EmptyDirectories(t *testing.T) {
|
||||
{Name: "v1", IsDirectory: false},
|
||||
},
|
||||
},
|
||||
wantHasUser: true,
|
||||
},
|
||||
{
|
||||
name: "nested versions directory left empty does not count",
|
||||
entriesByDir: map[string][]*filer_pb.Entry{
|
||||
"/buckets/b": {
|
||||
{Name: "logs", IsDirectory: true},
|
||||
},
|
||||
"/buckets/b/logs": {
|
||||
{Name: "foo" + s3_constants.VersionsFolder, IsDirectory: true},
|
||||
},
|
||||
"/buckets/b/logs/foo" + s3_constants.VersionsFolder: {},
|
||||
},
|
||||
wantHasUser: false,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -521,9 +521,8 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
|
||||
// bucketHasUserObjects checks whether a bucket contains any user objects.
|
||||
// Empty directories left behind by deleted objects and the .uploads staging
|
||||
// folder do not count as user objects; stored versions inside a <key>.versions
|
||||
// directory do.
|
||||
// Empty directories left behind by deleted objects and internal folders
|
||||
// (.uploads, *.versions) do not count as user objects.
|
||||
func (s3a *S3ApiServer) bucketHasUserObjects(bucket string) (bool, error) {
|
||||
return s3a.dirHasUserObjects(s3a.option.BucketsPath + "/" + bucket)
|
||||
}
|
||||
@@ -557,10 +556,9 @@ func (s3a *S3ApiServer) dirHasUserObjects(root string) (bool, error) {
|
||||
if entry.IsDirectoryKeyObject() {
|
||||
return true, nil
|
||||
}
|
||||
// Multipart staging folders are invisible to object listing at
|
||||
// every level, so they never count. A <key>.versions directory
|
||||
// is descended into since it may still hold stored versions.
|
||||
if entry.Name == s3_constants.MultipartUploadsFolder {
|
||||
// Internal folders are skipped by object listing at every level,
|
||||
// so a directory with a reserved name never counts.
|
||||
if isReservedDirectoryName(entry.Name) {
|
||||
continue
|
||||
}
|
||||
dirs = append(dirs, dir+"/"+entry.Name)
|
||||
|
||||
@@ -413,18 +413,3 @@ func TestTaggingCannotSetThePendingSourceMarker(t *testing.T) {
|
||||
assert.Equal(t, "2026", string(x.Extended["Seaweed-Album"]))
|
||||
assert.Empty(t, x.Extended[filer.ExtRemoteRenameSourceKey], "only the filer sets the pending source")
|
||||
}
|
||||
|
||||
func TestDeletingTagsKeepsThePendingSourceMarker(t *testing.T) {
|
||||
server, store := newRenameRemoteTestServer(t, &renameRemoteClient{})
|
||||
entry := remoteOnlyRenameEntry("/buckets/b/x.jpg", 101)
|
||||
entry.Extended = map[string][]byte{"Seaweed-Album": []byte("2026"), filer.ExtRemoteRenameSourceKey: []byte("/buckets/b/src/x.jpg")}
|
||||
store.entries["/buckets/b/x.jpg"] = entry
|
||||
|
||||
// no tag names: delete every tag
|
||||
server.DeleteTaggingHandler(httptest.NewRecorder(), httptest.NewRequest(http.MethodDelete, "/buckets/b/x.jpg?tagging", nil))
|
||||
|
||||
x, err := store.FindEntry(context.Background(), "/buckets/b/x.jpg")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, x.Extended["Seaweed-Album"])
|
||||
assert.Equal(t, "/buckets/b/src/x.jpg", string(x.Extended[filer.ExtRemoteRenameSourceKey]), "the marker is not a tag")
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Reque
|
||||
// delete all tags or specific tags
|
||||
hasDeletion := false
|
||||
for header, _ := range existingEntry.Extended {
|
||||
if strings.HasPrefix(header, needle.PairNamePrefix) && header != filer.ExtRemoteRenameSourceKey {
|
||||
if strings.HasPrefix(header, needle.PairNamePrefix) {
|
||||
if len(deletions) == 0 {
|
||||
delete(existingEntry.Extended, header)
|
||||
hasDeletion = true
|
||||
|
||||
@@ -58,10 +58,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
|
||||
writeJsonError(w, r, util.HttpStatusCancelled, err)
|
||||
case strings.HasSuffix(errStr, "is a file") || strings.HasSuffix(errStr, "already exists"):
|
||||
writeJsonError(w, r, http.StatusConflict, err)
|
||||
case errStr == constants.ErrMsgBadDigest,
|
||||
errStr == constants.ErrMsgAppendSaveInsideChunked,
|
||||
errStr == constants.ErrMsgAppendInlineMode,
|
||||
errStr == constants.ErrMsgOffsetWriteInline:
|
||||
case errStr == constants.ErrMsgBadDigest:
|
||||
writeJsonError(w, r, http.StatusBadRequest, err)
|
||||
default:
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
@@ -102,17 +99,8 @@ func (fs *FilerServer) doPostAutoChunk(ctx context.Context, w http.ResponseWrite
|
||||
if so.SaveInside {
|
||||
buf := bufPool.Get().(*bytes.Buffer)
|
||||
buf.Reset()
|
||||
if _, err := buf.ReadFrom(part1); err != nil {
|
||||
bufPool.Put(buf)
|
||||
return nil, nil, fmt.Errorf("read input: %w", err)
|
||||
}
|
||||
md5bytes = util.Md5(buf.Bytes())
|
||||
headerMd5 := r.Header.Get("Content-Md5")
|
||||
if headerMd5 != "" && !(util.Base64Encode(md5bytes) == headerMd5 || fmt.Sprintf("%x", md5bytes) == headerMd5) {
|
||||
bufPool.Put(buf)
|
||||
return nil, nil, errors.New(constants.ErrMsgBadDigest)
|
||||
}
|
||||
filerResult, replyerr, _ = fs.saveMetaData(ctx, r, fileName, contentType, so, chunkSize, md5bytes, nil, int64(buf.Len()), buf.Bytes())
|
||||
buf.ReadFrom(part1)
|
||||
filerResult, replyerr, _ = fs.saveMetaData(ctx, r, fileName, contentType, so, chunkSize, nil, nil, 0, buf.Bytes())
|
||||
bufPool.Put(buf)
|
||||
return
|
||||
}
|
||||
@@ -305,11 +293,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
||||
if entry != nil {
|
||||
inlineCandidate := len(entry.Content) > 0 ||
|
||||
(isAppend && entry.FileSize == 0 && entry.Remote == nil && len(entry.HardLinkId) == 0 && len(entry.GetChunks()) == 0)
|
||||
if isAppend && content != nil && !inlineCandidate {
|
||||
if so.SaveInside {
|
||||
replyerr = errors.New(constants.ErrMsgAppendSaveInsideChunked)
|
||||
return
|
||||
}
|
||||
if isAppend && !so.SaveInside && content != nil && !inlineCandidate {
|
||||
replyerr = fmt.Errorf("inline file changed while preparing append; retry the request")
|
||||
return
|
||||
}
|
||||
@@ -317,11 +301,11 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
||||
if inlineCandidate {
|
||||
if isOffsetWrite {
|
||||
// TODO: support inline offset writes separately from append semantics.
|
||||
replyerr = errors.New(constants.ErrMsgOffsetWriteInline)
|
||||
replyerr = fmt.Errorf("offset write to inline small file is not supported yet")
|
||||
return
|
||||
}
|
||||
if !isAppend || so.SaveInside || entry.Remote != nil || len(entry.HardLinkId) != 0 || len(entry.GetChunks()) != 0 {
|
||||
replyerr = errors.New(constants.ErrMsgAppendInlineMode)
|
||||
replyerr = fmt.Errorf("append to inline content with this storage mode is not supported")
|
||||
return
|
||||
}
|
||||
if entry.FileSize > uint64(len(entry.Content)) {
|
||||
|
||||
@@ -270,164 +270,6 @@ func TestFilerInlineAppendMultipartPostStaysInline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilerSaveInsideMultipartPostRecordsSize(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
content := []byte(`{"identities":[]}`)
|
||||
store := newRenameTestStore()
|
||||
f := newRenameTestFiler(t, store)
|
||||
fs := &FilerServer{filer: f, option: &FilerOption{MaxMB: 1}}
|
||||
|
||||
var body bytes.Buffer
|
||||
multipartWriter := multipart.NewWriter(&body)
|
||||
part, err := multipartWriter.CreateFormFile("file", "identity.json")
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file part: %v", err)
|
||||
}
|
||||
if _, err = part.Write(content); err != nil {
|
||||
t.Fatalf("write multipart file part: %v", err)
|
||||
}
|
||||
if err = multipartWriter.Close(); err != nil {
|
||||
t.Fatalf("close multipart body: %v", err)
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPost, "/", &body)
|
||||
r.Header.Set("Content-Type", multipartWriter.FormDataContentType())
|
||||
result, requestMD5, err := fs.doPostAutoChunk(ctx, httptest.NewRecorder(), r, 4, int64(body.Len()), &operation.StorageOption{SaveInside: true})
|
||||
if err != nil {
|
||||
t.Fatalf("POST saveInside: %v", err)
|
||||
}
|
||||
if result == nil || result.Size != int64(len(content)) {
|
||||
t.Fatalf("result = %#v, want size %d", result, len(content))
|
||||
}
|
||||
wantMD5 := md5.Sum(content)
|
||||
if !bytes.Equal(requestMD5, wantMD5[:]) {
|
||||
t.Fatalf("request MD5 = %x, want %x", requestMD5, wantMD5)
|
||||
}
|
||||
saved, err := f.FindEntry(ctx, "/identity.json")
|
||||
if err != nil {
|
||||
t.Fatalf("find saved entry: %v", err)
|
||||
}
|
||||
if !bytes.Equal(saved.Content, content) || saved.FileSize != uint64(len(content)) {
|
||||
t.Fatalf("saved entry: content=%q size=%d, want %q size %d", saved.Content, saved.FileSize, content, len(content))
|
||||
}
|
||||
if !bytes.Equal(saved.Md5, wantMD5[:]) {
|
||||
t.Fatalf("saved MD5 = %x, want %x", saved.Md5, wantMD5)
|
||||
}
|
||||
}
|
||||
|
||||
func saveInsideMultipartRequest(t *testing.T, target, fileName string, content []byte) *http.Request {
|
||||
t.Helper()
|
||||
var body bytes.Buffer
|
||||
multipartWriter := multipart.NewWriter(&body)
|
||||
part, err := multipartWriter.CreateFormFile("file", fileName)
|
||||
if err != nil {
|
||||
t.Fatalf("create multipart file part: %v", err)
|
||||
}
|
||||
if _, err = part.Write(content); err != nil {
|
||||
t.Fatalf("write multipart file part: %v", err)
|
||||
}
|
||||
if err = multipartWriter.Close(); err != nil {
|
||||
t.Fatalf("close multipart body: %v", err)
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPost, target, &body)
|
||||
r.Header.Set("Content-Type", multipartWriter.FormDataContentType())
|
||||
return r
|
||||
}
|
||||
|
||||
func TestFilerSaveInsideMultipartPostRejectsBadDigest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
content := []byte(`{"identities":[]}`)
|
||||
store := newRenameTestStore()
|
||||
f := newRenameTestFiler(t, store)
|
||||
fs := &FilerServer{filer: f, option: &FilerOption{MaxMB: 1}}
|
||||
|
||||
r := saveInsideMultipartRequest(t, "/", "identity.json", content)
|
||||
r.Header.Set("Content-Md5", util.Base64Encode([]byte("not the digest")))
|
||||
if _, _, err := fs.doPostAutoChunk(ctx, httptest.NewRecorder(), r, 4, r.ContentLength, &operation.StorageOption{SaveInside: true}); err == nil || !strings.Contains(err.Error(), "Content-Md5") {
|
||||
t.Fatalf("POST saveInside error = %v, want bad Content-MD5 error", err)
|
||||
}
|
||||
if _, err := f.FindEntry(ctx, "/identity.json"); !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
t.Fatalf("bad digest still saved the entry: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilerSaveInsideMultipartAppendRejected(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const filePath = "/append-target.txt"
|
||||
appendContent := []byte("new")
|
||||
|
||||
t.Run("chunk backed target", func(t *testing.T) {
|
||||
oldChunk := &filer_pb.FileChunk{FileId: "1,00000001", Offset: 0, Size: 4}
|
||||
store := newRenameTestStore()
|
||||
f := newRenameTestFiler(t, store)
|
||||
if err := store.InsertEntry(ctx, &filer.Entry{
|
||||
FullPath: util.FullPath(filePath),
|
||||
Attr: filer.Attr{
|
||||
Mtime: time.Unix(1, 0),
|
||||
Crtime: time.Unix(1, 0),
|
||||
FileSize: 4,
|
||||
},
|
||||
Chunks: []*filer_pb.FileChunk{oldChunk},
|
||||
}); err != nil {
|
||||
t.Fatalf("insert chunk-backed entry: %v", err)
|
||||
}
|
||||
|
||||
fs := &FilerServer{filer: f, option: &FilerOption{MaxMB: 1}}
|
||||
r := saveInsideMultipartRequest(t, filePath+"?op=append", "append-target.txt", appendContent)
|
||||
if _, _, err := fs.doPostAutoChunk(ctx, httptest.NewRecorder(), r, 4, r.ContentLength, &operation.StorageOption{SaveInside: true}); err == nil {
|
||||
t.Fatal("saveInside append to chunk-backed entry succeeded")
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
r = saveInsideMultipartRequest(t, filePath+"?op=append", "append-target.txt", appendContent)
|
||||
fs.autoChunk(ctx, recorder, r, r.ContentLength, &operation.StorageOption{SaveInside: true})
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("rejected saveInside append status = %d, want %d", recorder.Code, http.StatusBadRequest)
|
||||
}
|
||||
unchanged, err := f.FindEntry(ctx, util.FullPath(filePath))
|
||||
if err != nil {
|
||||
t.Fatalf("find entry after rejected append: %v", err)
|
||||
}
|
||||
if unchanged.FileSize != 4 || len(unchanged.Chunks) != 1 || len(unchanged.Content) != 0 {
|
||||
t.Fatalf("rejected append changed entry: size=%d chunks=%d content=%q", unchanged.FileSize, len(unchanged.Chunks), unchanged.Content)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("inline target", func(t *testing.T) {
|
||||
oldContent := []byte("old-")
|
||||
store := newRenameTestStore()
|
||||
f := newRenameTestFiler(t, store)
|
||||
if err := store.InsertEntry(ctx, &filer.Entry{
|
||||
FullPath: util.FullPath(filePath),
|
||||
Attr: filer.Attr{
|
||||
Mtime: time.Unix(1, 0),
|
||||
Crtime: time.Unix(1, 0),
|
||||
FileSize: uint64(len(oldContent)),
|
||||
},
|
||||
Content: append([]byte(nil), oldContent...),
|
||||
}); err != nil {
|
||||
t.Fatalf("insert inline entry: %v", err)
|
||||
}
|
||||
|
||||
fs := &FilerServer{filer: f, option: &FilerOption{MaxMB: 1}}
|
||||
r := saveInsideMultipartRequest(t, filePath+"?op=append", "append-target.txt", appendContent)
|
||||
if _, _, err := fs.doPostAutoChunk(ctx, httptest.NewRecorder(), r, 4, r.ContentLength, &operation.StorageOption{SaveInside: true}); err == nil {
|
||||
t.Fatal("saveInside append to inline entry succeeded")
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
r = saveInsideMultipartRequest(t, filePath+"?op=append", "append-target.txt", appendContent)
|
||||
fs.autoChunk(ctx, recorder, r, r.ContentLength, &operation.StorageOption{SaveInside: true})
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("rejected saveInside append status = %d, want %d", recorder.Code, http.StatusBadRequest)
|
||||
}
|
||||
unchanged, err := f.FindEntry(ctx, util.FullPath(filePath))
|
||||
if err != nil {
|
||||
t.Fatalf("find entry after rejected append: %v", err)
|
||||
}
|
||||
if !bytes.Equal(unchanged.Content, oldContent) || unchanged.FileSize != uint64(len(oldContent)) {
|
||||
t.Fatalf("rejected append changed entry: content=%q size=%d", unchanged.Content, unchanged.FileSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFilerInlineOffsetWriteRemainsUnsupported(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const filePath = "/offset.txt"
|
||||
|
||||
@@ -509,27 +509,6 @@ func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGro
|
||||
Force: true,
|
||||
Reason: "grpc volume grow",
|
||||
}
|
||||
|
||||
// Report the layout's fill observed at request time whether or not
|
||||
// growth was triggered, so callers can tell an empty layout from a
|
||||
// skipped one. A missing collection/layout reports zeros.
|
||||
var avgFill float64
|
||||
var writableCount int
|
||||
if vl := ms.Topo.FindVolumeLayout(req.Collection, replicaPlacement, ttl, types.ToDiskType(req.DiskType)); vl != nil {
|
||||
avgFill, writableCount = vl.WritableAvgFill(&volumeGrowOption)
|
||||
}
|
||||
|
||||
// Conditional grow: with a threshold set, grow only when the layout's
|
||||
// writable volumes have reached this average fill. A layout with no
|
||||
// writable volumes (none yet, or all just filled) skips the check — that
|
||||
// is the case growth exists to fix.
|
||||
if req.GrowAvgFillThreshold > 0 && writableCount > 0 && avgFill < float64(req.GrowAvgFillThreshold) {
|
||||
return &master_pb.VolumeGrowResponse{
|
||||
AvgFill: float32(avgFill),
|
||||
WritableCount: uint32(writableCount),
|
||||
}, nil
|
||||
}
|
||||
|
||||
replicaCount := int64(req.WritableVolumeCount * uint32(replicaPlacement.GetCopyCount()))
|
||||
|
||||
if ms.Topo.AvailableSpaceFor(&volumeGrowOption) < replicaCount {
|
||||
@@ -538,8 +517,5 @@ func (ms *MasterServer) VolumeGrow(ctx context.Context, req *master_pb.VolumeGro
|
||||
|
||||
ms.DoAutomaticVolumeGrow(&volumeGrowRequest)
|
||||
|
||||
return &master_pb.VolumeGrowResponse{
|
||||
AvgFill: float32(avgFill),
|
||||
WritableCount: uint32(writableCount),
|
||||
}, nil
|
||||
return &master_pb.VolumeGrowResponse{}, nil
|
||||
}
|
||||
|
||||
@@ -394,7 +394,7 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
|
||||
}
|
||||
location = vs.store.FindEcShardTargetLocation(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.DataShardsCount, shardIds...)
|
||||
if location == nil {
|
||||
return nil, status.Error(codes.ResourceExhausted, "no space left")
|
||||
return nil, fmt.Errorf("no space left")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -570,14 +570,6 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) {
|
||||
if fileSize == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
if len(f.entry.Content) > 0 {
|
||||
if f.off >= int64(len(f.entry.Content)) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
readSize = copy(p, f.entry.Content[f.off:])
|
||||
f.off += int64(readSize)
|
||||
return readSize, nil
|
||||
}
|
||||
if f.visibleIntervals == nil {
|
||||
f.visibleIntervals, err = filer.NonOverlappingVisibleIntervals(f.ctx, f.fs.filerClient.GetLookupFileIdFunction(), f.entry.GetChunks(), 0, fileSize)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,7 +3,6 @@ package weed_server
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
@@ -105,28 +104,3 @@ func TestWebDavFile_Read_ManifestResolveFailure(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, 0, n)
|
||||
}
|
||||
|
||||
// Small files stored inline in the entry have no chunks to read from.
|
||||
func TestWebDavFile_Read_InlineContent(t *testing.T) {
|
||||
content := []byte("hello inline")
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "small.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: uint64(len(content))},
|
||||
Content: content,
|
||||
}
|
||||
filerAddr := startFakeWebDavFiler(t, &noVolumeFiler{})
|
||||
fc := wdclient.NewFilerClient([]pb.ServerAddress{filerAddr}, grpc.WithTransportCredentials(insecure.NewCredentials()), "")
|
||||
t.Cleanup(fc.Close)
|
||||
f := &WebDavFile{fs: &WebDavFileSystem{filerClient: fc}, name: "/small.txt", entry: entry, ctx: context.Background()}
|
||||
|
||||
got, err := io.ReadAll(f)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, content, got)
|
||||
|
||||
_, err = f.Seek(6, io.SeekStart)
|
||||
assert.NoError(t, err)
|
||||
buf := make([]byte, 3)
|
||||
n, err := f.Read(buf)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "inl", string(buf[:n]))
|
||||
}
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
package sftpd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/sftpd/user"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
type copyTestFiler struct {
|
||||
filer_pb.UnimplementedSeaweedFilerServer
|
||||
entries map[string]*filer_pb.Entry
|
||||
volumeUrl string
|
||||
}
|
||||
|
||||
func (f *copyTestFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
if entry, ok := f.entries[req.Directory+"/"+req.Name]; ok {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: entry}, nil
|
||||
}
|
||||
return nil, status.Error(codes.NotFound, "no entry is found")
|
||||
}
|
||||
|
||||
func (f *copyTestFiler) LookupVolume(_ context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
|
||||
resp := &filer_pb.LookupVolumeResponse{LocationsMap: make(map[string]*filer_pb.Locations)}
|
||||
for _, vid := range req.VolumeIds {
|
||||
resp.LocationsMap[vid] = &filer_pb.Locations{Locations: []*filer_pb.Location{{Url: f.volumeUrl}}}
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func newCopyTestServer(t *testing.T, volumeUrl string, entries map[string]*filer_pb.Entry) *SftpServer {
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
grpcServer := grpc.NewServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(grpcServer, ©TestFiler{entries: entries, volumeUrl: volumeUrl})
|
||||
go func() { _ = grpcServer.Serve(lis) }()
|
||||
t.Cleanup(grpcServer.Stop)
|
||||
|
||||
_, port, _ := net.SplitHostPort(lis.Addr().String())
|
||||
return &SftpServer{
|
||||
filerAddr: pb.ServerAddress("127.0.0.1:0." + port),
|
||||
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
user: &user.User{Username: "alice", Uid: 1000, HomeDir: "/"},
|
||||
}
|
||||
}
|
||||
|
||||
func chunkedEntry(size int64, chunks ...*filer_pb.FileChunk) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: uint64(size)},
|
||||
Chunks: chunks,
|
||||
IsDirectory: false,
|
||||
}
|
||||
}
|
||||
|
||||
func fileChunk(fileId string, offset, size int64) *filer_pb.FileChunk {
|
||||
return &filer_pb.FileChunk{FileId: fileId, Offset: offset, Size: uint64(size), ModifiedTsNs: 1}
|
||||
}
|
||||
|
||||
// A non-truncating write must keep untouched bytes, including a sparse file's
|
||||
// zeros between chunks. Stopping at the first gap would drop the tail.
|
||||
func TestCopyExistingContentFillsSparseGaps(t *testing.T) {
|
||||
chunkA := []byte("AAAAAAAAAA")
|
||||
chunkB := []byte("BBBBBBBBBB")
|
||||
objects := map[string][]byte{"3,01": chunkA, "3,02": chunkB}
|
||||
volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
data, ok := objects[r.URL.Path[1:]]
|
||||
if !ok {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(data)
|
||||
}))
|
||||
t.Cleanup(volume.Close)
|
||||
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/home/sparse.bin": chunkedEntry(100, fileChunk("3,01", 0, 10), fileChunk("3,02", 90, 10)),
|
||||
}
|
||||
fs := newCopyTestServer(t, volume.Listener.Addr().String(), entries)
|
||||
|
||||
var dst bytes.Buffer
|
||||
if err := fs.copyExistingContent("/home/sparse.bin", &dst); err != nil {
|
||||
t.Fatalf("copyExistingContent: %v", err)
|
||||
}
|
||||
want := append(append(append([]byte{}, chunkA...), make([]byte, 80)...), chunkB...)
|
||||
if !bytes.Equal(dst.Bytes(), want) {
|
||||
t.Fatalf("copied %d bytes, want %d with a zero-filled gap; tail present: %v", dst.Len(), len(want), bytes.Contains(dst.Bytes(), chunkB))
|
||||
}
|
||||
}
|
||||
|
||||
// A remote-only file has no local chunks: copying nothing would upload an
|
||||
// empty replacement on close. Refusing the read keeps the remote object.
|
||||
func TestCopyExistingContentRejectsUnreadableEntry(t *testing.T) {
|
||||
remoteOnly := chunkedEntry(0)
|
||||
remoteOnly.RemoteEntry = &filer_pb.RemoteEntry{RemoteSize: 100}
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/home/remote.bin": remoteOnly,
|
||||
}
|
||||
fs := newCopyTestServer(t, "", entries)
|
||||
|
||||
var dst bytes.Buffer
|
||||
if err := fs.copyExistingContent("/home/remote.bin", &dst); err == nil {
|
||||
t.Fatal("expected refusal when no local bytes exist")
|
||||
}
|
||||
}
|
||||
|
||||
// A locally shortened file still names its older, larger remote object. Once
|
||||
// the local entry is newer the remote size is stale: copying must not pad the
|
||||
// file back out to it.
|
||||
func TestCopyExistingContentIgnoresStaleRemoteSize(t *testing.T) {
|
||||
chunkA := []byte("AAAAAAAAAA")
|
||||
volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/3,01" {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(chunkA)
|
||||
}))
|
||||
t.Cleanup(volume.Close)
|
||||
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/home/short.bin": {
|
||||
Attributes: &filer_pb.FuseAttributes{FileSize: 10, Mtime: 200},
|
||||
Chunks: []*filer_pb.FileChunk{fileChunk("3,01", 0, 10)},
|
||||
RemoteEntry: &filer_pb.RemoteEntry{RemoteSize: 100, RemoteMtime: 100},
|
||||
},
|
||||
}
|
||||
fs := newCopyTestServer(t, volume.Listener.Addr().String(), entries)
|
||||
|
||||
var dst bytes.Buffer
|
||||
if err := fs.copyExistingContent("/home/short.bin", &dst); err != nil {
|
||||
t.Fatalf("copyExistingContent: %v", err)
|
||||
}
|
||||
if !bytes.Equal(dst.Bytes(), chunkA) {
|
||||
t.Fatalf("copied %d bytes, want the %d local bytes only", dst.Len(), len(chunkA))
|
||||
}
|
||||
}
|
||||
|
||||
// A file whose size is all holes still keeps its zeros.
|
||||
func TestCopyExistingContentAllHoles(t *testing.T) {
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/home/holes.bin": chunkedEntry(10),
|
||||
}
|
||||
fs := newCopyTestServer(t, "", entries)
|
||||
|
||||
var dst bytes.Buffer
|
||||
if err := fs.copyExistingContent("/home/holes.bin", &dst); err != nil {
|
||||
t.Fatalf("all-holes file: %v", err)
|
||||
}
|
||||
if dst.Len() != 10 || !bytes.Equal(dst.Bytes(), make([]byte, 10)) {
|
||||
t.Fatalf("wrote %d bytes, want 10 zeros", dst.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyExistingContentInlineAndMissing(t *testing.T) {
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/home/inline.txt": {Attributes: &filer_pb.FuseAttributes{FileSize: 5}, Content: []byte("hello")},
|
||||
}
|
||||
fs := newCopyTestServer(t, "", entries)
|
||||
|
||||
var dst bytes.Buffer
|
||||
if err := fs.copyExistingContent("/home/inline.txt", &dst); err != nil {
|
||||
t.Fatalf("inline content: %v", err)
|
||||
}
|
||||
if got := dst.String(); got != "hello" {
|
||||
t.Fatalf("inline content = %q, want %q", got, "hello")
|
||||
}
|
||||
|
||||
dst.Reset()
|
||||
if err := fs.copyExistingContent("/home/missing.txt", &dst); err != nil || dst.Len() != 0 {
|
||||
t.Fatalf("missing file: err=%v written=%d", err, dst.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyExistingContentTrailingGap(t *testing.T) {
|
||||
chunkA := []byte("AAAAAAAAAA")
|
||||
volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/3,01" {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(chunkA)
|
||||
}))
|
||||
t.Cleanup(volume.Close)
|
||||
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/home/trailing.bin": chunkedEntry(50, fileChunk("3,01", 0, 10)),
|
||||
}
|
||||
fs := newCopyTestServer(t, volume.Listener.Addr().String(), entries)
|
||||
|
||||
var dst bytes.Buffer
|
||||
if err := fs.copyExistingContent("/home/trailing.bin", &dst); err != nil {
|
||||
t.Fatalf("copyExistingContent: %v", err)
|
||||
}
|
||||
want := append(append([]byte{}, chunkA...), make([]byte, 40)...)
|
||||
if !bytes.Equal(dst.Bytes(), want) {
|
||||
t.Fatalf("copied %d bytes, want %d with a zero-filled tail", dst.Len(), len(want))
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
filer_pb "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -24,7 +23,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/sftpd/user"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
@@ -161,14 +159,6 @@ func (fs *SftpServer) newFileWriter(r *sftp.Request) (io.WriterAt, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temp file: %w", err)
|
||||
}
|
||||
// Without O_TRUNC the client writes into the existing file, so start from its current bytes.
|
||||
if !r.Pflags().Trunc {
|
||||
if err := fs.copyExistingContent(absPath, tmpFile); err != nil {
|
||||
tmpFile.Close()
|
||||
os.Remove(tmpFile.Name())
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return &SeaweedSftpFileWriter{
|
||||
fs: *fs,
|
||||
@@ -182,75 +172,6 @@ func (fs *SftpServer) newFileWriter(r *sftp.Request) (io.WriterAt, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fs *SftpServer) copyExistingContent(absPath string, dst io.Writer) error {
|
||||
entry, err := fs.getEntry(absPath)
|
||||
if err == os.ErrNotExist {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDirectory {
|
||||
return nil
|
||||
}
|
||||
size := int64(filer.FileSize(entry))
|
||||
remote := entry.GetRemoteEntry()
|
||||
if len(entry.Content) == 0 && len(entry.GetChunks()) == 0 {
|
||||
if remote != nil && remote.RemoteSize > 0 {
|
||||
return fmt.Errorf("existing content of %s is not readable", absPath)
|
||||
}
|
||||
if size == 0 {
|
||||
return nil
|
||||
}
|
||||
return writeZeros(dst, size)
|
||||
}
|
||||
if size == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(entry.Content) > 0 {
|
||||
if _, err := dst.Write(entry.Content); err != nil {
|
||||
return err
|
||||
}
|
||||
if remaining := size - int64(len(entry.Content)); remaining > 0 {
|
||||
return writeZeros(dst, remaining)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
streamFn, err := filer.PrepareStreamContent(lookupFileIdFn(filer.LookupFn(fs)), filer.JwtForVolumeServer, entry.GetChunks(), 0, size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read existing content of %s: %w", absPath, err)
|
||||
}
|
||||
if err := streamFn(dst); err != nil {
|
||||
return fmt.Errorf("read existing content of %s: %w", absPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookupFileIdFn adapts a filer LookupFn to the chunk streamer's provider
|
||||
// interface. PrepareStreamContent fills unstored gaps with zeros, so a sparse
|
||||
// file's tail survives the copy.
|
||||
type lookupFileIdFn wdclient.LookupFileIdFunctionType
|
||||
|
||||
func (fn lookupFileIdFn) GetLookupFileIdFunction() wdclient.LookupFileIdFunctionType {
|
||||
return wdclient.LookupFileIdFunctionType(fn)
|
||||
}
|
||||
|
||||
func writeZeros(w io.Writer, size int64) error {
|
||||
zeros := make([]byte, 8192)
|
||||
for size > 0 {
|
||||
n := int64(len(zeros))
|
||||
if size < n {
|
||||
n = size
|
||||
}
|
||||
written, err := w.Write(zeros[:n])
|
||||
size -= int64(written)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fs *SftpServer) removeEntry(absPath string) error {
|
||||
return fs.deleteEntry(absPath, false)
|
||||
}
|
||||
@@ -325,24 +246,21 @@ func (fs *SftpServer) listAllPages(dirPath string) (sftp.ListerAt, error) {
|
||||
var all []os.FileInfo
|
||||
last := ""
|
||||
for {
|
||||
page, lastName, received, err := fs.fetchDirectoryPage(dirPath, last)
|
||||
page, err := fs.fetchDirectoryPage(dirPath, last)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all = append(all, page...)
|
||||
if received < defaultListLimit {
|
||||
if len(page) < defaultListLimit {
|
||||
break
|
||||
}
|
||||
last = lastName
|
||||
last = page[len(page)-1].Name()
|
||||
}
|
||||
return listerat(all), nil
|
||||
}
|
||||
|
||||
// fetchDirectoryPage returns the entries the user may list, plus the last name and count the filer sent.
|
||||
func (fs *SftpServer) fetchDirectoryPage(dirPath, start string) ([]os.FileInfo, string, int, error) {
|
||||
func (fs *SftpServer) fetchDirectoryPage(dirPath, start string) ([]os.FileInfo, error) {
|
||||
var list []os.FileInfo
|
||||
lastName := ""
|
||||
received := 0
|
||||
err := fs.callWithClient(true, func(ctx context.Context, client filer_pb.SeaweedFilerClient) error {
|
||||
stream, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{Directory: dirPath, StartFromFileName: start, Limit: defaultListLimit})
|
||||
if err != nil {
|
||||
@@ -353,14 +271,9 @@ func (fs *SftpServer) fetchDirectoryPage(dirPath, start string) ([]os.FileInfo,
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.Entry == nil {
|
||||
if err != nil || r.Entry == nil {
|
||||
continue
|
||||
}
|
||||
received++
|
||||
lastName = r.Entry.Name
|
||||
p := path.Join(dirPath, r.Entry.Name)
|
||||
if err := fs.checkFilePermission(p, "list"); err != nil {
|
||||
continue
|
||||
@@ -369,7 +282,7 @@ func (fs *SftpServer) fetchDirectoryPage(dirPath, start string) ([]os.FileInfo,
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return list, lastName, received, err
|
||||
return list, err
|
||||
}
|
||||
|
||||
// makeDir creates a new directory with proper permissions.
|
||||
|
||||
@@ -1,24 +1,12 @@
|
||||
package sftpd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/sftp"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/sftpd/user"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// TestPutFileEscapesQueryInjection ensures a filename containing "?" cannot be
|
||||
@@ -49,132 +37,3 @@ func TestPutFileEscapesQueryInjection(t *testing.T) {
|
||||
t.Errorf("path not delivered literally: got %q, want %q", gotPath, malicious)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeListFiler struct {
|
||||
filer_pb.UnimplementedSeaweedFilerServer
|
||||
names []string
|
||||
failFrom int // ListEntries fails after sending this many entries, when > 0
|
||||
}
|
||||
|
||||
func (f *fakeListFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
|
||||
sent := 0
|
||||
for _, name := range f.names {
|
||||
if name <= req.StartFromFileName || sent >= int(req.Limit) {
|
||||
continue
|
||||
}
|
||||
if f.failFrom > 0 && sent == f.failFrom {
|
||||
return status.Error(codes.Unavailable, "filer went away")
|
||||
}
|
||||
if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}}}); err != nil {
|
||||
return err
|
||||
}
|
||||
sent++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeListFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: &filer_pb.Entry{Name: req.Name, Attributes: &filer_pb.FuseAttributes{}}}, nil
|
||||
}
|
||||
|
||||
func newListTestServer(t *testing.T, filer *fakeListFiler) *SftpServer {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(srv, filer)
|
||||
go srv.Serve(lis)
|
||||
t.Cleanup(srv.Stop)
|
||||
return &SftpServer{
|
||||
filerAddr: pb.ServerAddress(fmt.Sprintf("127.0.0.1:1.%d", lis.Addr().(*net.TCPAddr).Port)),
|
||||
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
user: &user.User{
|
||||
Username: "alice",
|
||||
HomeDir: "/",
|
||||
Uid: 1001,
|
||||
Gid: 1001,
|
||||
// every entry under /d is listable except f00000
|
||||
Permissions: map[string][]string{"/d": {"list"}, "/d/f00000": {}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func listNames(t *testing.T, fs *SftpServer) ([]string, error) {
|
||||
t.Helper()
|
||||
type result struct {
|
||||
lister sftp.ListerAt
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
lister, err := fs.listAllPages("/d")
|
||||
done <- result{lister, err}
|
||||
}()
|
||||
select {
|
||||
case r := <-done:
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
infos := make([]os.FileInfo, 10000)
|
||||
n, _ := r.lister.ListAt(infos, 0)
|
||||
var names []string
|
||||
for _, fi := range infos[:n] {
|
||||
names = append(names, fi.Name())
|
||||
}
|
||||
return names, nil
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("listing did not return")
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
func fileNames(n int) []string {
|
||||
names := make([]string, n)
|
||||
for i := range names {
|
||||
names[i] = fmt.Sprintf("f%05d", i)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestListAllPagesReturnsStreamError(t *testing.T) {
|
||||
fs := newListTestServer(t, &fakeListFiler{names: fileNames(10), failFrom: 3})
|
||||
_, err := listNames(t, fs)
|
||||
if status.Code(err) != codes.Unavailable {
|
||||
t.Fatalf("expected the stream error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAllPagesContinuesPastHiddenEntries(t *testing.T) {
|
||||
fs := newListTestServer(t, &fakeListFiler{names: fileNames(defaultListLimit + 500)})
|
||||
names, err := listNames(t, fs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(names) != defaultListLimit+499 {
|
||||
t.Fatalf("expected %d entries, got %d", defaultListLimit+499, len(names))
|
||||
}
|
||||
if names[0] != "f00001" || names[len(names)-1] != fmt.Sprintf("f%05d", defaultListLimit+499) {
|
||||
t.Fatalf("unexpected first/last entries %q, %q", names[0], names[len(names)-1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestListAllPagesContinuesPastFullyHiddenPage(t *testing.T) {
|
||||
const tail = 10
|
||||
fs := newListTestServer(t, &fakeListFiler{names: fileNames(defaultListLimit + tail)})
|
||||
// The whole first page is hidden from the user; only the tail is listable.
|
||||
for i := 0; i < defaultListLimit; i++ {
|
||||
fs.user.Permissions[fmt.Sprintf("/d/f%05d", i)] = []string{}
|
||||
}
|
||||
names, err := listNames(t, fs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(names) != tail {
|
||||
t.Fatalf("expected the %d entries after the hidden page, got %d", tail, len(names))
|
||||
}
|
||||
if names[0] != fmt.Sprintf("f%05d", defaultListLimit) {
|
||||
t.Fatalf("unexpected first entry %q", names[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -274,7 +273,7 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][
|
||||
}
|
||||
// most empty one is in the front
|
||||
slices.SortFunc(otherNodes, func(a, b *Node) int {
|
||||
return cmp.Compare(a.localVolumeRatio(maxVolumeCountFn), b.localVolumeRatio(maxVolumeCountFn))
|
||||
return int(a.localVolumeRatio(maxVolumeCountFn) - b.localVolumeRatio(maxVolumeCountFn))
|
||||
})
|
||||
for i := 0; i < len(otherNodes); i++ {
|
||||
emptyNode := otherNodes[i]
|
||||
|
||||
@@ -5,22 +5,6 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEvacuatePrefersLowerVolumeRatio(t *testing.T) {
|
||||
volumes := mkByteVolumes(1, 1, 1)
|
||||
source := makeByteNode("source", 10, 0, 0, volumes)
|
||||
fuller := makeByteNode("fuller", 10, 0, 0, mkByteVolumes(10, 4, 1))
|
||||
emptier := makeByteNode("emptier", 10, 0, 0, mkByteVolumes(20, 2, 1))
|
||||
|
||||
// Put the 40%-full server first: truncating 0.2 - 0.4 to int makes the ratios tie.
|
||||
moved, err := moveAwayOneNormalVolume(nil, nil, volumes[0], source, []*Node{fuller, emptier}, false)
|
||||
if err != nil || !moved {
|
||||
t.Fatalf("evacuate: moved=%v, err=%v", moved, err)
|
||||
}
|
||||
if _, found := emptier.selectedVolumes[volumes[0].Id]; !found {
|
||||
t.Fatal("expected evacuation to choose the server with lower occupancy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeServerEvacuate(t *testing.T) {
|
||||
targetServer := ""
|
||||
volumeRack := "rack1"
|
||||
|
||||
@@ -982,7 +982,7 @@ func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle.
|
||||
return types.Offset{}, 0, nil, fmt.Errorf("FindNeedleFromEcx: %w", err)
|
||||
}
|
||||
|
||||
intervals = ev.LocateEcShardNeedleInterval(version, offset.ToActualOffset(), size)
|
||||
intervals = ev.LocateEcShardNeedleInterval(version, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, version)))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -256,30 +256,3 @@ func TestEcVolumeDestroySweepsBothDirs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocateEcShardNeedleMatchesInterval(t *testing.T) {
|
||||
ecxFile, err := os.OpenFile("./test_files/389.ecx", os.O_RDONLY, 0)
|
||||
assert.NoError(t, err)
|
||||
defer ecxFile.Close()
|
||||
|
||||
stat, err := ecxFile.Stat()
|
||||
assert.NoError(t, err)
|
||||
fileSize := stat.Size()
|
||||
|
||||
ev := &EcVolume{
|
||||
VolumeId: needle.VolumeId(389),
|
||||
ecxFile: ecxFile,
|
||||
ecxFileSize: fileSize,
|
||||
Shards: []*EcVolumeShard{
|
||||
{ecdFileSize: 1118830592},
|
||||
},
|
||||
ECContext: NewDefaultECContext("", 389),
|
||||
}
|
||||
|
||||
needleId, _ := types.ParseNeedleId("0f0edb92")
|
||||
offset, size, intervals, err := ev.LocateEcShardNeedle(needleId, needle.GetCurrentVersion())
|
||||
assert.NoError(t, err)
|
||||
|
||||
expectedIntervals := ev.LocateEcShardNeedleInterval(needle.GetCurrentVersion(), offset.ToActualOffset(), size)
|
||||
assert.Equal(t, expectedIntervals, intervals)
|
||||
}
|
||||
|
||||
@@ -38,10 +38,6 @@ type Node struct {
|
||||
rack string // composite rack key (e.g. "dc1:rack1")
|
||||
|
||||
freeSlots int
|
||||
// noReceive marks a node that refused shards (e.g. answered "no space
|
||||
// left"): it stays usable as a move source but is never picked as a target,
|
||||
// regardless of what its freeSlots say.
|
||||
noReceive bool
|
||||
disks map[uint32]*disk
|
||||
shards map[volKey]*volumeShards
|
||||
}
|
||||
@@ -51,8 +47,7 @@ type disk struct {
|
||||
diskType string
|
||||
tags []string // placement tags, for preferred-tag tiering
|
||||
freeSlots int
|
||||
shardCount int // total EC shards on this disk across all volumes
|
||||
noReceive bool // this disk refused a shard; siblings may still receive
|
||||
shardCount int // total EC shards on this disk across all volumes
|
||||
}
|
||||
|
||||
type volumeShards struct {
|
||||
@@ -161,39 +156,6 @@ func (n *Node) SetHost(host string) {
|
||||
}
|
||||
}
|
||||
|
||||
// SetNoReceive marks the node ineligible as a move destination. Use when the
|
||||
// node itself refused shards (e.g. answered "no space left") so the heartbeat's
|
||||
// free-slot count cannot be trusted; the node can still shed shards as a source.
|
||||
func (n *Node) SetNoReceive() {
|
||||
n.noReceive = true
|
||||
}
|
||||
|
||||
// SetDiskNoReceive marks one disk on the node ineligible as a move destination.
|
||||
// An explicit-disk copy can be refused while the node's other disks still have
|
||||
// room; a disk_id 0 refusal means the server searched every disk and is a
|
||||
// whole-node refusal (SetNoReceive) instead.
|
||||
func (n *Node) SetDiskNoReceive(diskID uint32) {
|
||||
if d, ok := n.disks[diskID]; ok {
|
||||
d.noReceive = true
|
||||
}
|
||||
}
|
||||
|
||||
// receiveSlots is the node's receiving capacity: its aggregate free slots with
|
||||
// refused disks subtracted, and 0 for a refused node. Slots a disk just
|
||||
// refused are phantom capacity whatever the snapshot reported.
|
||||
func (n *Node) receiveSlots() int {
|
||||
if n.noReceive {
|
||||
return 0
|
||||
}
|
||||
free := n.freeSlots
|
||||
for _, d := range n.disks {
|
||||
if d.noReceive {
|
||||
free -= d.freeSlots
|
||||
}
|
||||
}
|
||||
return max(free, 0)
|
||||
}
|
||||
|
||||
// AddDisk registers a physical disk. shardCount is the disk's total EC shard
|
||||
// count across all volumes (used for disk scoring); freeSlots is the per-disk
|
||||
// free EC shard slots.
|
||||
@@ -334,7 +296,7 @@ func buildRacks(nodes map[string]*Node) map[string]*rack {
|
||||
racks[n.rack] = r
|
||||
}
|
||||
r.nodes[n.id] = n
|
||||
r.freeSlots += n.receiveSlots()
|
||||
r.freeSlots += n.freeSlots
|
||||
}
|
||||
return racks
|
||||
}
|
||||
@@ -478,7 +440,7 @@ func planRackTotalCap(vk volKey, racks map[string]*rack, rackShardCount map[stri
|
||||
continue
|
||||
}
|
||||
for _, n := range r.nodes {
|
||||
room[rk] += min(n.receiveSlots(), max(rp.SameRackCount-volumeShardCount(n, vk), 0))
|
||||
room[rk] += min(max(n.freeSlots, 0), max(rp.SameRackCount-volumeShardCount(n, vk), 0))
|
||||
}
|
||||
}
|
||||
return rackTotalCap(sortedKeys(racks), nil, room, totalShards)
|
||||
@@ -613,15 +575,12 @@ func pickBestNodeForVolumeOnDisk(nodes []*Node, vk volKey, rp *super_block.Repli
|
||||
// metadata is authoritative when present; without it the node's aggregate
|
||||
// stands in so the volume server's own disk selection can place the shard.
|
||||
func freeSlotsForType(n *Node, diskType string) int {
|
||||
if n.noReceive {
|
||||
return 0
|
||||
}
|
||||
if len(n.disks) == 0 {
|
||||
return n.freeSlots
|
||||
}
|
||||
free := 0
|
||||
for _, d := range n.disks {
|
||||
if d.freeSlots > 0 && !d.noReceive && (diskType == "" || d.diskType == diskType) {
|
||||
if d.freeSlots > 0 && (diskType == "" || d.diskType == diskType) {
|
||||
free += d.freeSlots
|
||||
}
|
||||
}
|
||||
@@ -645,7 +604,7 @@ func pickBestNodeForVolume(nodes []*Node, vk volKey, rp *super_block.ReplicaPlac
|
||||
bestFree := -1
|
||||
bestScore := -1
|
||||
for _, node := range nodes {
|
||||
if node.receiveSlots() <= 0 {
|
||||
if node.freeSlots <= 0 {
|
||||
continue
|
||||
}
|
||||
count := volumeShardCount(node, vk)
|
||||
@@ -950,7 +909,7 @@ func detectGlobalImbalance(nodes map[string]*Node, racks map[string]*rack, diskT
|
||||
continue
|
||||
}
|
||||
util := float64(count) / float64(capacity)
|
||||
if util < minUtil && node.receiveSlots() > 0 {
|
||||
if util < minUtil && node.freeSlots > 0 {
|
||||
minUtil, minCount, minNode = util, count, node
|
||||
}
|
||||
if util > maxUtil {
|
||||
@@ -1132,7 +1091,7 @@ func pickBestDiskOnNode(node *Node, vk volKey, diskType string, shardID, dataSha
|
||||
if diskType != "" && d.diskType != diskType {
|
||||
continue
|
||||
}
|
||||
if d.freeSlots <= 0 || d.noReceive {
|
||||
if d.freeSlots <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -926,67 +926,3 @@ func TestGlobalPrefersVolumeAbsentFromDestinationMachine(t *testing.T) {
|
||||
t.Errorf("first global move is volume %d, want 200 (vol100 already on the destination machine)", moves[0].volumeID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoReceiveNodeNeverTargeted: a node that refused a copy ("no space left")
|
||||
// must not be picked as a destination no matter how many free slots it reports,
|
||||
// but it still sheds its own over-cap shards as a source.
|
||||
func TestNoReceiveNodeNeverTargeted(t *testing.T) {
|
||||
const vid = uint32(1)
|
||||
topo := NewTopology()
|
||||
src := topo.AddNode("n1", "dc1", "dc1:r1", 50)
|
||||
src.AddShards(vid, "c", 0, bits(0, 1, 2, 3, 4, 5, 6, 7, 8, 9))
|
||||
full := topo.AddNode("n2", "dc1", "dc1:r1", 50)
|
||||
full.AddShards(vid, "c", 0, bits(10, 11, 12, 13)) // all parity, over the 2/node cap
|
||||
full.SetNoReceive()
|
||||
topo.AddNode("n3", "dc1", "dc1:r1", 50)
|
||||
|
||||
moves := Plan(topo, Options{Ratio: ratio(10, 4)})
|
||||
if len(moves) == 0 {
|
||||
t.Fatal("expected moves off the over-cap nodes")
|
||||
}
|
||||
shed := false
|
||||
for _, m := range moves {
|
||||
if m.TargetNode == "n2" {
|
||||
t.Fatalf("move targets the refusing node: %+v", m)
|
||||
}
|
||||
if m.SourceNode == "n2" {
|
||||
shed = true
|
||||
}
|
||||
}
|
||||
if !shed {
|
||||
t.Error("the refusing node should still shed its over-cap shards as a source")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoReceiveDiskLeavesSiblingsEligible: an explicit-disk refusal marks only
|
||||
// that disk, not the node — the planner may still target the node's other
|
||||
// disks, but never the refused one.
|
||||
func TestNoReceiveDiskLeavesSiblingsEligible(t *testing.T) {
|
||||
const vid = uint32(1)
|
||||
topo := NewTopology()
|
||||
src := topo.AddNode("n1", "dc1", "dc1:r1", 50)
|
||||
src.AddShards(vid, "c", 0, bits(0, 1, 2, 3, 4, 5, 6, 7, 8, 9))
|
||||
n2 := topo.AddNode("n2", "dc1", "dc1:r1", 50)
|
||||
// The refused disk scores better (empty); without the noReceive check the
|
||||
// picker would choose it again.
|
||||
n2.AddDisk(1, "hdd", 50, 0)
|
||||
n2.AddDisk(2, "hdd", 50, 50)
|
||||
n2.AddShards(vid, "c", 2, bits(10, 11, 12, 13))
|
||||
n2.SetDiskNoReceive(1)
|
||||
topo.AddNode("n3", "dc1", "dc1:r1", 50)
|
||||
|
||||
moves := Plan(topo, Options{Ratio: ratio(10, 4)})
|
||||
targeted := false
|
||||
for _, m := range moves {
|
||||
if m.TargetNode != "n2" {
|
||||
continue
|
||||
}
|
||||
targeted = true
|
||||
if m.TargetDisk == 1 {
|
||||
t.Fatalf("move targets the refused disk: %+v", m)
|
||||
}
|
||||
}
|
||||
if !targeted {
|
||||
t.Fatal("a node with one refused disk should still receive on its other disks")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,9 +430,6 @@ func rackTotalCap(rackKeys []string, held, room map[string]int, totalShards int)
|
||||
// its free slots, bounded by the per-disk durability cap (maxPerDisk shards of
|
||||
// one volume per disk; <= 0 disables it).
|
||||
func diskShardRoom(n *Node, d *disk, vk volKey, maxPerDisk int) int {
|
||||
if d.noReceive {
|
||||
return 0
|
||||
}
|
||||
room := d.freeSlots
|
||||
if maxPerDisk > 0 {
|
||||
held := 0
|
||||
@@ -447,16 +444,13 @@ func diskShardRoom(n *Node, d *disk, vk volKey, maxPerDisk int) int {
|
||||
// nodeShardRoom returns how many more shards of the volume the node's eligible
|
||||
// disks can take, bounded by the node's free slots.
|
||||
func nodeShardRoom(n *Node, vk volKey, eligible func(*disk) bool, maxPerDisk int) int {
|
||||
if n.noReceive {
|
||||
return 0
|
||||
}
|
||||
room := 0
|
||||
for _, d := range n.disks {
|
||||
if eligible(d) {
|
||||
room += diskShardRoom(n, d, vk, maxPerDisk)
|
||||
}
|
||||
}
|
||||
return max(min(room, n.receiveSlots()), 0)
|
||||
return max(min(room, n.freeSlots), 0)
|
||||
}
|
||||
|
||||
// rackShardRoom returns how many more shards of the volume the rack can take.
|
||||
@@ -538,7 +532,7 @@ func pickBestDiskEligible(node *Node, vk volKey, eligible func(*disk) bool, pref
|
||||
bestPreferred := false
|
||||
for _, diskID := range sortedDiskKeys(node.disks) {
|
||||
d := node.disks[diskID]
|
||||
if !eligible(d) || d.freeSlots <= 0 || d.noReceive {
|
||||
if !eligible(d) || d.freeSlots <= 0 {
|
||||
continue
|
||||
}
|
||||
existingShards := 0
|
||||
|
||||
@@ -343,19 +343,6 @@ func (dn *DataNode) GetRack() *Rack {
|
||||
return dn.Parent().(*NodeImpl).value.(*Rack)
|
||||
}
|
||||
|
||||
func (dn *DataNode) matchesLocation(option *VolumeGrowOption) bool {
|
||||
if option.DataCenter != "" && dn.GetDataCenter().Id() != NodeId(option.DataCenter) {
|
||||
return false
|
||||
}
|
||||
if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
|
||||
return false
|
||||
}
|
||||
if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (dn *DataNode) GetTopology() *Topology {
|
||||
p := dn.Parent()
|
||||
for p.Parent() != nil {
|
||||
|
||||
@@ -448,22 +448,6 @@ func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.Replic
|
||||
}).(*Collection).GetOrCreateVolumeLayout(rp, ttl, diskType)
|
||||
}
|
||||
|
||||
// FindVolumeLayout is the read-only counterpart of GetVolumeLayout: it returns
|
||||
// nil when the collection or its layout does not exist, instead of creating
|
||||
// one. Lets callers check a layout's state without polluting the collection
|
||||
// map with an empty collection.
|
||||
func (t *Topology) FindVolumeLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) *VolumeLayout {
|
||||
c, ok := t.collectionMap.Find(collectionName)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
vl, ok := c.(*Collection).GetVolumeLayout(rp, ttl, diskType)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return vl
|
||||
}
|
||||
|
||||
// DecayQuietVolumeSizes decays pending assign estimates across every layout.
|
||||
// A volume that changed reports within a pulse, so two quiet pulses mean the
|
||||
// size on record is the size there is.
|
||||
|
||||
@@ -650,7 +650,13 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi
|
||||
writableVolumeId := vl.writables[(start+i)%lenWriters]
|
||||
volumeLocationList := vl.vid2location[writableVolumeId]
|
||||
for _, dn := range volumeLocationList.list {
|
||||
if !dn.matchesLocation(option) {
|
||||
if option.DataCenter != "" && dn.GetDataCenter().Id() != NodeId(option.DataCenter) {
|
||||
continue
|
||||
}
|
||||
if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
|
||||
continue
|
||||
}
|
||||
if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
|
||||
continue
|
||||
}
|
||||
sample[found] = writableVolumeId
|
||||
@@ -756,45 +762,6 @@ func (vl *VolumeLayout) ShouldGrowVolumes() bool {
|
||||
return writable <= crowded
|
||||
}
|
||||
|
||||
// WritableAvgFill returns the average fill ratio of the writable volumes and
|
||||
// how many there are. The fill of a single volume is its tracked effective
|
||||
// size over the layout's volume size limit; volumes without a size-tracking
|
||||
// entry yet (freshly created, no heartbeat size report so far) count as
|
||||
// empty. A non-empty location filter in option narrows the measure to
|
||||
// writables serving that target, so a targeted grow is not blocked or
|
||||
// triggered by fill elsewhere in the layout.
|
||||
func (vl *VolumeLayout) WritableAvgFill(option *VolumeGrowOption) (avgFill float64, writableCount int) {
|
||||
vl.accessLock.RLock()
|
||||
defer vl.accessLock.RUnlock()
|
||||
|
||||
var total uint64
|
||||
for _, vid := range vl.writables {
|
||||
if option != nil && (option.DataCenter != "" || option.Rack != "" || option.DataNode != "") {
|
||||
matched := false
|
||||
if loc := vl.vid2location[vid]; loc != nil {
|
||||
for _, dn := range loc.list {
|
||||
if dn.matchesLocation(option) {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
}
|
||||
writableCount++
|
||||
if st := vl.sizeTracking[vid]; st != nil {
|
||||
total += st.effectiveSize
|
||||
}
|
||||
}
|
||||
if writableCount == 0 || vl.volumeSizeLimit == 0 {
|
||||
return 0, writableCount
|
||||
}
|
||||
avgFill = float64(total) / (float64(writableCount) * float64(vl.volumeSizeLimit))
|
||||
return avgFill, writableCount
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) ShouldGrowVolumesByDcAndRack(writables *[]needle.VolumeId, dcId NodeId, rackId NodeId) bool {
|
||||
// When replication spans multiple racks (DiffRackCount > 0), a writable
|
||||
// volume's replicas only cover some racks in a DC. It is wrong to
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
func TestWritableAvgFill(t *testing.T) {
|
||||
// 3 writable volumes at 20%, 50%, 80% fill (sizes 2000, 5000, 8000 of
|
||||
// limit 10000): average fill is 50%.
|
||||
layout := `
|
||||
{
|
||||
"dc1":{
|
||||
"rack1":{
|
||||
"server1":{
|
||||
"volumes":[
|
||||
{"id":1, "size":2000, "replication":"000"},
|
||||
{"id":2, "size":5000, "replication":"000"},
|
||||
{"id":3, "size":8000, "replication":"000"}
|
||||
],
|
||||
"limit":10
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
_, vl := setupPickTest(t, layout, 10000)
|
||||
|
||||
avgFill, writableCount := vl.WritableAvgFill(nil)
|
||||
if writableCount != 3 {
|
||||
t.Fatalf("expected 3 writable volumes, got %d", writableCount)
|
||||
}
|
||||
if math.Abs(avgFill-0.5) > 1e-9 {
|
||||
t.Errorf("expected avg fill 0.5, got %v", avgFill)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWritableAvgFillNoWritables(t *testing.T) {
|
||||
layout := `
|
||||
{
|
||||
"dc1":{
|
||||
"rack1":{
|
||||
"server1":{
|
||||
"volumes":[
|
||||
{"id":1, "size":2000, "replication":"000"}
|
||||
],
|
||||
"limit":10
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
_, vl := setupPickTest(t, layout, 10000)
|
||||
|
||||
vl.accessLock.Lock()
|
||||
vl.removeFromWritable(1)
|
||||
vl.accessLock.Unlock()
|
||||
|
||||
avgFill, writableCount := vl.WritableAvgFill(nil)
|
||||
if writableCount != 0 || avgFill != 0 {
|
||||
t.Errorf("expected (0, 0) with no writables, got (%v, %d)", avgFill, writableCount)
|
||||
}
|
||||
}
|
||||
|
||||
// A writable volume without a size-tracking entry yet — freshly created, no
|
||||
// heartbeat size report so far — counts as empty, not as an error.
|
||||
func TestWritableAvgFillUntrackedVolumeCountsAsEmpty(t *testing.T) {
|
||||
layout := `
|
||||
{
|
||||
"dc1":{
|
||||
"rack1":{
|
||||
"server1":{
|
||||
"volumes":[
|
||||
{"id":1, "size":5000, "replication":"000"},
|
||||
{"id":2, "size":5000, "replication":"000"}
|
||||
],
|
||||
"limit":10
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
_, vl := setupPickTest(t, layout, 10000)
|
||||
|
||||
vl.accessLock.Lock()
|
||||
delete(vl.sizeTracking, 2)
|
||||
vl.accessLock.Unlock()
|
||||
|
||||
avgFill, writableCount := vl.WritableAvgFill(nil)
|
||||
if writableCount != 2 {
|
||||
t.Fatalf("expected 2 writable volumes, got %d", writableCount)
|
||||
}
|
||||
if math.Abs(avgFill-0.25) > 1e-9 {
|
||||
t.Errorf("expected avg fill 0.25 (one of two volumes empty), got %v", avgFill)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWritableAvgFillScopedToTarget(t *testing.T) {
|
||||
layout := `
|
||||
{
|
||||
"dc1":{
|
||||
"rack1":{
|
||||
"server1":{
|
||||
"volumes":[
|
||||
{"id":1, "size":9000, "replication":"000"},
|
||||
{"id":2, "size":9000, "replication":"000"}
|
||||
],
|
||||
"limit":10
|
||||
}
|
||||
}
|
||||
},
|
||||
"dc2":{
|
||||
"rack2":{
|
||||
"server2":{
|
||||
"volumes":[
|
||||
{"id":3, "size":1000, "replication":"000"}
|
||||
],
|
||||
"limit":10
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
_, vl := setupPickTest(t, layout, 10000)
|
||||
|
||||
avgFill, writableCount := vl.WritableAvgFill(&VolumeGrowOption{DataCenter: "dc2"})
|
||||
if writableCount != 1 || math.Abs(avgFill-0.1) > 1e-9 {
|
||||
t.Fatalf("expected dc2 fill (0.1, 1), got (%v, %d)", avgFill, writableCount)
|
||||
}
|
||||
|
||||
// A target with no writable volumes reports none, so growth is not
|
||||
// blocked by fill elsewhere.
|
||||
avgFill, writableCount = vl.WritableAvgFill(&VolumeGrowOption{DataCenter: "dc2", DataNode: "missing"})
|
||||
if writableCount != 0 || avgFill != 0 {
|
||||
t.Fatalf("expected (0, 0) for a target with no writables, got (%v, %d)", avgFill, writableCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindVolumeLayout(t *testing.T) {
|
||||
layout := `
|
||||
{
|
||||
"dc1":{
|
||||
"rack1":{
|
||||
"server1":{
|
||||
"volumes":[
|
||||
{"id":1, "size":2000, "collection":"colA", "replication":"000"}
|
||||
],
|
||||
"limit":10
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
topo := setupWithLimit(t, layout, 10000)
|
||||
rp, _ := super_block.NewReplicaPlacementFromString("000")
|
||||
|
||||
// Existing layout is found.
|
||||
if vl := topo.FindVolumeLayout("colA", rp, needle.EMPTY_TTL, types.HardDriveType); vl == nil {
|
||||
t.Error("expected layout for colA to be found")
|
||||
}
|
||||
|
||||
// A missing collection resolves to nil and must not be created.
|
||||
if vl := topo.FindVolumeLayout("noSuchCollection", rp, needle.EMPTY_TTL, types.HardDriveType); vl != nil {
|
||||
t.Error("expected nil layout for unknown collection")
|
||||
}
|
||||
if _, ok := topo.collectionMap.Find("noSuchCollection"); ok {
|
||||
t.Error("FindVolumeLayout must not create the collection")
|
||||
}
|
||||
|
||||
// A missing layout within an existing collection also resolves to nil.
|
||||
rpOther, _ := super_block.NewReplicaPlacementFromString("001")
|
||||
if vl := topo.FindVolumeLayout("colA", rpOther, needle.EMPTY_TTL, types.HardDriveType); vl != nil {
|
||||
t.Error("expected nil layout for unused replication")
|
||||
}
|
||||
}
|
||||
@@ -7,10 +7,6 @@ const (
|
||||
ErrMsgOperationNotPermitted = "operation not permitted"
|
||||
ErrMsgBadDigest = "The Content-Md5 you specified did not match what we received."
|
||||
ErrMsgEntryNameTooLong = "entry name too long"
|
||||
// Unsupported write-mode combinations; deterministic client errors.
|
||||
ErrMsgAppendSaveInsideChunked = "append saved inside the entry to a chunked file is not supported"
|
||||
ErrMsgAppendInlineMode = "append to inline content with this storage mode is not supported"
|
||||
ErrMsgOffsetWriteInline = "offset write to inline small file is not supported yet"
|
||||
// Suffix appended to a path, e.g. "/buckets/b/foo/bar is a file"
|
||||
ErrMsgIsAFile = " is a file"
|
||||
// Prefix + suffix, e.g. "existing /buckets/b/foo is a directory"
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// HostKey is the hostname/IP of an "host:port" address (or the string itself
|
||||
// when it has no port). Compared case-insensitively so worker Hello
|
||||
// "10.87.0.3:33646" matches a volume replica on "10.87.0.3:8080".
|
||||
func HostKey(addr string) string {
|
||||
addr = strings.TrimSpace(addr)
|
||||
if addr == "" {
|
||||
return ""
|
||||
}
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return strings.ToLower(addr)
|
||||
}
|
||||
return strings.ToLower(host)
|
||||
}
|
||||
|
||||
// HostKeysFromAddresses returns unique HostKey values, skipping empties.
|
||||
func HostKeysFromAddresses(addrs []string) []string {
|
||||
seen := make(map[string]struct{}, len(addrs))
|
||||
var out []string
|
||||
for _, a := range addrs {
|
||||
h := HostKey(a)
|
||||
if h == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[h]; ok {
|
||||
continue
|
||||
}
|
||||
seen[h] = struct{}{}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AddressIsLocal reports whether addr's host is this process's machine
|
||||
// (loopback or a configured interface address). Used so an EC worker prefers
|
||||
// a replica on the same box after replica sync.
|
||||
func AddressIsLocal(addr string) bool {
|
||||
host := HostKey(addr)
|
||||
if host == "" {
|
||||
return false
|
||||
}
|
||||
if host == "localhost" || host == "127.0.0.1" || host == "::1" {
|
||||
return true
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip != nil && ip.IsLoopback() {
|
||||
return true
|
||||
}
|
||||
ifaces, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, ia := range ifaces {
|
||||
var got net.IP
|
||||
switch v := ia.(type) {
|
||||
case *net.IPNet:
|
||||
got = v.IP
|
||||
case *net.IPAddr:
|
||||
got = v.IP
|
||||
}
|
||||
if got == nil {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(got.String(), host) {
|
||||
return true
|
||||
}
|
||||
if ip != nil && got.Equal(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package util
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHostKeyStripsPort(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := HostKey("10.87.0.3:8080"); got != "10.87.0.3" {
|
||||
t.Fatalf("HostKey replica: got %q", got)
|
||||
}
|
||||
if got := HostKey("10.87.0.3:33646"); got != "10.87.0.3" {
|
||||
t.Fatalf("HostKey worker: got %q", got)
|
||||
}
|
||||
if HostKey("10.87.0.3:8080") != HostKey("10.87.0.3:33646") {
|
||||
t.Fatal("worker and volume server on the same IP must share a HostKey")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostKeysFromAddressesDedups(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := HostKeysFromAddresses([]string{"10.0.0.1:8080", "10.0.0.1:8081", "10.0.0.2:8080", ""})
|
||||
if len(got) != 2 || got[0] != "10.0.0.1" || got[1] != "10.0.0.2" {
|
||||
t.Fatalf("got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -116,29 +116,17 @@ func (t *ECBalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParam
|
||||
return nil
|
||||
}
|
||||
|
||||
move := volume_move.EcShardMove{
|
||||
err := mover.MoveEcShards(ctx, volume_move.EcShardMove{
|
||||
VolumeId: needle.VolumeId(params.VolumeId),
|
||||
Collection: params.Collection,
|
||||
ShardIds: shardIds,
|
||||
Source: sourceAddr,
|
||||
Target: targetAddr,
|
||||
TargetDisk: target.DiskId,
|
||||
}
|
||||
opts := volume_move.EcMoveOptions{
|
||||
}, volume_move.EcMoveOptions{
|
||||
IoBytePerSecond: ecParams.GetIoBytePerSecond(),
|
||||
Progress: t.reportProgress,
|
||||
}
|
||||
err := mover.MoveEcShards(ctx, move, opts)
|
||||
// A "no space left" refusal to a pinned disk convicts only that disk;
|
||||
// retry once letting the server pick any disk on the same node. The
|
||||
// planner's node choice stands; only the intra-node disk defers to the
|
||||
// server's live view. When the node is truly full, the retry fails with
|
||||
// the same error.
|
||||
if err != nil && target.DiskId > 0 && volume_move.IsNoSpaceLeftError(err) {
|
||||
glog.Infof("EC balance: %s disk %d has no space left; retrying with server-picked disk", target.Node, target.DiskId)
|
||||
move.TargetDisk = 0
|
||||
err = mover.MoveEcShards(ctx, move, opts)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func TestECBalanceTaskRejectsOutOfRangeShardIds(t *testing.T) {
|
||||
@@ -27,32 +25,3 @@ func TestECBalanceTaskRejectsOutOfRangeShardIds(t *testing.T) {
|
||||
t.Fatalf("Execute: expected out-of-range rejection, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal against the pinned disk convicts only that disk, not the node:
|
||||
// the task retries once letting the server pick a disk, and the move completes.
|
||||
func TestECBalanceTaskRetriesServerPickOnPinnedDiskRefusal(t *testing.T) {
|
||||
source := startFakeEcVolumeServer(t, 7, 5)
|
||||
target := startFakeEcVolumeServer(t, 7)
|
||||
target.refuseCopyOnDisk = 2
|
||||
|
||||
task := NewECBalanceTask("t2", 7, "c1",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
params := &worker_pb.TaskParams{
|
||||
VolumeId: 7,
|
||||
Sources: []*worker_pb.TaskSource{{Node: source.address(), ShardIds: []uint32{5}}},
|
||||
Targets: []*worker_pb.TaskTarget{{Node: target.address(), DiskId: 2, ShardIds: []uint32{5}}},
|
||||
}
|
||||
|
||||
if err := task.Execute(context.Background(), params); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if got := target.copyDiskIds; len(got) != 2 || got[0] != 2 || got[1] != 0 {
|
||||
t.Fatalf("expected copies to disk 2 then server-picked 0, got %v", got)
|
||||
}
|
||||
if source.has(7, 5) {
|
||||
t.Fatalf("source still holds shard 5")
|
||||
}
|
||||
if !target.has(7, 5) {
|
||||
t.Fatalf("target never received shard 5")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,12 +32,6 @@ type fakeEcVolumeServer struct {
|
||||
// the request even when it was a no-op against the inventory.
|
||||
deleted []string
|
||||
unmounted []string
|
||||
// refuseCopyOnDisk, when nonzero, makes VolumeEcShardsCopy fail requests
|
||||
// pinning that disk id with "no space left" — a full disk; a request with
|
||||
// DiskId 0 (server picks) still succeeds.
|
||||
refuseCopyOnDisk uint32
|
||||
// copyDiskIds records the DiskId each copy request carried.
|
||||
copyDiskIds []uint32
|
||||
// grpcAddr is host:port; the code under test dials by address string.
|
||||
grpcAddr string
|
||||
stop func()
|
||||
@@ -52,31 +46,11 @@ func (f *fakeEcVolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume
|
||||
VolumeId: req.VolumeId,
|
||||
ShardId: sid,
|
||||
Collection: f.collection,
|
||||
Size: 1,
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// VolumeEcShardsCopy records the request's DiskId and adds the shards to this
|
||||
// server's inventory, unless the pinned disk is the configured refusing disk.
|
||||
func (f *fakeEcVolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.copyDiskIds = append(f.copyDiskIds, req.DiskId)
|
||||
if f.refuseCopyOnDisk != 0 && req.DiskId == f.refuseCopyOnDisk {
|
||||
return nil, fmt.Errorf("disk %d: no space left", req.DiskId)
|
||||
}
|
||||
for _, sid := range req.ShardIds {
|
||||
f.shards[req.VolumeId][sid] = true
|
||||
}
|
||||
return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEcVolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest) (*volume_server_pb.VolumeEcShardsMountResponse, error) {
|
||||
return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeEcVolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
storagetypes "github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
|
||||
@@ -37,9 +36,6 @@ type ErasureCodingTask struct {
|
||||
workDir string
|
||||
progress float64
|
||||
grpcDialOption grpc.DialOption
|
||||
// workerAddress is weed worker -address (host:port). Not a volume URL;
|
||||
// late-bind matches replica hostnames against HostKey(workerAddress).
|
||||
workerAddress string
|
||||
|
||||
// EC parameters
|
||||
dataShards int32
|
||||
@@ -407,75 +403,9 @@ func (t *ErasureCodingTask) syncAndSelectSourceReplica() error {
|
||||
glog.Infof("EC encode: using best replica %s as source for volume %d (was %s)", best.Url, t.volumeID, t.server)
|
||||
t.server = best.Url
|
||||
}
|
||||
// Late-bind from after consistency: worker -address hostname, not Sources[0]
|
||||
// and not AddressIsLocal (replica URLs are hostnames like full-02:8081).
|
||||
workerAddr := strings.TrimSpace(t.workerAddress)
|
||||
if workerAddr == "" {
|
||||
workerAddr, _ = os.Hostname()
|
||||
}
|
||||
if local := preferWorkerHostReplicaIfComplete(t.grpcDialOption, needle.VolumeId(t.volumeID), locs, t.server, workerAddr); local != "" && local != t.server {
|
||||
glog.Infof("EC encode: preferring worker-host replica %s as source for volume %d (was %s)", local, t.volumeID, t.server)
|
||||
t.server = local
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type replicaFileCount struct {
|
||||
url string
|
||||
fileCount uint64
|
||||
err error
|
||||
}
|
||||
|
||||
// preferWorkerHostReplicaIfComplete rebinds from to a same-hostname replica
|
||||
// only when live FileCounts are equal. Diverged counts keep current (max).
|
||||
func preferWorkerHostReplicaIfComplete(dial grpc.DialOption, vid needle.VolumeId, locs []wdclient.Location, current, workerAddress string) string {
|
||||
statuses := volume_replica.GetReplicaStatuses(dial, vid, locs)
|
||||
replicas := make([]replicaFileCount, 0, len(statuses))
|
||||
for _, s := range statuses {
|
||||
replicas = append(replicas, replicaFileCount{url: s.Location.Url, fileCount: s.FileCount, err: s.Error})
|
||||
}
|
||||
return lateBindSourceToWorkerHost(current, workerAddress, replicas)
|
||||
}
|
||||
|
||||
// lateBindSourceToWorkerHost: equal FileCount → replica on worker -address host
|
||||
// (any port). Diverge → keep current (already max FileCount). Worker address
|
||||
// is not a volume URL.
|
||||
func lateBindSourceToWorkerHost(current, workerAddress string, replicas []replicaFileCount) string {
|
||||
workerHost := util.HostKey(workerAddress)
|
||||
if workerHost == "" {
|
||||
return current
|
||||
}
|
||||
var live []replicaFileCount
|
||||
for _, r := range replicas {
|
||||
if r.err != nil || r.url == "" {
|
||||
continue
|
||||
}
|
||||
live = append(live, r)
|
||||
}
|
||||
if len(live) == 0 {
|
||||
return current
|
||||
}
|
||||
maxCount := live[0].fileCount
|
||||
allSame := true
|
||||
for _, r := range live[1:] {
|
||||
if r.fileCount != live[0].fileCount {
|
||||
allSame = false
|
||||
}
|
||||
if r.fileCount > maxCount {
|
||||
maxCount = r.fileCount
|
||||
}
|
||||
}
|
||||
if !allSame {
|
||||
return current
|
||||
}
|
||||
for _, r := range live {
|
||||
if util.HostKey(r.url) == workerHost {
|
||||
return r.url
|
||||
}
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
// rollbackReadonly is a best-effort restore of every replica markReplicasReadonly
|
||||
// touched, used when the EC task fails before the originals are deleted. Logs but
|
||||
// does not return errors; uses a fresh context since the caller's may be cancelled.
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package erasure_coding
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestLateBindSourceToWorkerHostEqualFileCountPrefersWorkerHost(t *testing.T) {
|
||||
a := replicaFileCount{url: "full-01:8080", fileCount: 21}
|
||||
b := replicaFileCount{url: "full-02:8081", fileCount: 21}
|
||||
worker := "full-02:9334"
|
||||
|
||||
// Sources [A,B] — topology-first is A; worker on B must still copy B.
|
||||
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != b.url {
|
||||
t.Fatalf("Sources [A,B] worker B: got %s want %s", got, b.url)
|
||||
}
|
||||
// Invert Sources — still B.
|
||||
if got := lateBindSourceToWorkerHost(b.url, worker, []replicaFileCount{b, a}); got != b.url {
|
||||
t.Fatalf("Sources [B,A] worker B: got %s want %s", got, b.url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateBindSourceToWorkerHostDivergeKeepsMaxFileCount(t *testing.T) {
|
||||
a := replicaFileCount{url: "full-01:8080", fileCount: 40}
|
||||
b := replicaFileCount{url: "full-02:8081", fileCount: 21}
|
||||
worker := "full-02:9334"
|
||||
|
||||
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{a, b}); got != a.url {
|
||||
t.Fatalf("diverge [A,B]: got %s want max FileCount %s", got, a.url)
|
||||
}
|
||||
if got := lateBindSourceToWorkerHost(a.url, worker, []replicaFileCount{b, a}); got != a.url {
|
||||
t.Fatalf("diverge [B,A]: got %s want max FileCount %s", got, a.url)
|
||||
}
|
||||
}
|
||||
@@ -26,9 +26,7 @@ func init() {
|
||||
Category: pluginworker.CategoryHeavy,
|
||||
Aliases: []string{"erasure-coding", "erasure.coding", "ec"},
|
||||
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
|
||||
h := NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir)
|
||||
h.workerAddress = strings.TrimSpace(opts.WorkerAddress)
|
||||
return h, nil
|
||||
return NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir), nil
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -41,7 +39,6 @@ type erasureCodingWorkerConfig struct {
|
||||
type ErasureCodingHandler struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
workingDir string
|
||||
workerAddress string
|
||||
}
|
||||
|
||||
func NewErasureCodingHandler(grpcDialOption grpc.DialOption, workingDir string) *ErasureCodingHandler {
|
||||
@@ -522,7 +519,6 @@ func (h *ErasureCodingHandler) Execute(
|
||||
params.Collection,
|
||||
h.grpcDialOption,
|
||||
)
|
||||
task.workerAddress = h.workerAddress
|
||||
execCtx, execCancel := context.WithCancel(ctx)
|
||||
defer execCancel()
|
||||
task.SetProgressCallback(func(progress float64, stage string) {
|
||||
@@ -674,15 +670,8 @@ func buildErasureCodingProposal(
|
||||
}
|
||||
|
||||
sourceNode := ""
|
||||
var replicaServers []string
|
||||
for _, s := range params.Sources {
|
||||
if s == nil || strings.TrimSpace(s.Node) == "" || len(s.ShardIds) > 0 {
|
||||
continue
|
||||
}
|
||||
replicaServers = append(replicaServers, s.Node)
|
||||
if sourceNode == "" {
|
||||
sourceNode = strings.TrimSpace(s.Node)
|
||||
}
|
||||
if len(params.Sources) > 0 {
|
||||
sourceNode = strings.TrimSpace(params.Sources[0].Node)
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("Erasure code volume %d", result.VolumeID)
|
||||
@@ -713,9 +702,6 @@ func buildErasureCodingProposal(
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
|
||||
},
|
||||
"replica_servers": {
|
||||
Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: replicaServers}},
|
||||
},
|
||||
"collection": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user