mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-19 13:02:35 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  Chris LuandDevin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
|
d6335c5e94 |
@@ -10,5 +10,3 @@ updates:
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: weekly
|
||||
ignore:
|
||||
- dependency-name: "github.com/seaweedfs/goexif"
|
||||
|
||||
@@ -1,240 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check JWT extraction and, with --context, real Helm upgrade persistence."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # CI also exercises Python 3.10.
|
||||
import tomli as tomllib
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CHART = ROOT / "k8s/charts/seaweedfs"
|
||||
SECTIONS = ("jwt.signing", "jwt.signing.read", "jwt.filer_signing", "jwt.filer_signing.read")
|
||||
KEYS = {section: f"active-{index}" for index, section in enumerate(SECTIONS)}
|
||||
ESCAPED_HEADER = '["jw\\u0074".signing]\nkey = "existing"'
|
||||
|
||||
|
||||
def run(*args):
|
||||
return subprocess.run(args, check=True, text=True, capture_output=True).stdout
|
||||
|
||||
|
||||
def keys(raw):
|
||||
document = tomllib.loads(raw)
|
||||
result = {}
|
||||
for section in SECTIONS:
|
||||
table = document
|
||||
for part in section.split("."):
|
||||
table = table.get(part, {})
|
||||
if "key" in table:
|
||||
result[section] = table["key"]
|
||||
return result
|
||||
|
||||
|
||||
def fixtures():
|
||||
canonical = "\n".join(f'[{section}]\nkey = "{value}"' for section, value in KEYS.items())
|
||||
yield "canonical / final line without newline", canonical
|
||||
yield "commented stale keys", canonical.replace("key =", '# key = "stale"\nkey =')
|
||||
yield "commented headers / mixed line endings", "\n".join(
|
||||
f'# [{section}]\r\n# key = "stale"\r\n[{section}]\nkey = "{value}"'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "indented headers and keys / trailing comments", "\n".join(
|
||||
f' \t[ {section} ] \t# header [note]\n \tkey \t= \t"{value}" # key comment'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "CRLF", canonical.replace("\n", "\r\n")
|
||||
yield "quoted and spaced section names", "\n".join(
|
||||
f'["{section.split(".")[0]}" . \'{section.split(".")[1]}\''
|
||||
+ (f' . "{section.split(".")[2]}"' if section.count(".") == 2 else "")
|
||||
+ f'] # original table\nkey = "{value}"'
|
||||
for section, value in KEYS.items()
|
||||
)
|
||||
yield "unrelated quoted header before JWT keys", '["custom section"]\nkey = "other"\n' + canonical
|
||||
yield "brackets in comments", canonical.replace("key =", "# consult [notes]\nkey =")
|
||||
yield "quoted values and quoted key names", "\n".join((
|
||||
'[jwt.signing]\n"key" = "brackets[inside]#value"',
|
||||
"[jwt.signing.read]\n'key' = 'literal\\path[#value]'",
|
||||
r'[jwt.filer_signing]' + '\n' + r'key = "escaped\"quote\\slash\u0041"',
|
||||
'[jwt.filer_signing.read]\nkey = ""',
|
||||
))
|
||||
yield "absent keys and sections / unrelated key", '\n'.join((
|
||||
'[jwt.signing]\nexpires_after_seconds = 10',
|
||||
'[jwt.signing.read]\nkey = "read-only"',
|
||||
'[unrelated]\nkey = "not-a-jwt-key"',
|
||||
'# [jwt.filer_signing]\n# key = "not-active"',
|
||||
))
|
||||
yield "no existing security.toml", ""
|
||||
|
||||
|
||||
def check_generated(value):
|
||||
decoded = base64.b64decode(value, validate=True).decode("ascii")
|
||||
assert len(decoded) == 10 and decoded.isascii() and decoded.isalnum(), "invalid generated JWT key"
|
||||
|
||||
|
||||
def check_helpers(helm, reference_helm):
|
||||
# Exercise the real helper, not a second implementation of its matching rules.
|
||||
with tempfile.TemporaryDirectory(prefix="helm-jwt-helper-") as directory:
|
||||
chart = Path(directory)
|
||||
(chart / "templates").mkdir()
|
||||
(chart / "Chart.yaml").write_text("apiVersion: v2\nname: jwt-regression\nversion: 0.0.0\n")
|
||||
shutil.copyfile(CHART / "templates/shared/_helpers.tpl", chart / "templates/_helpers.tpl")
|
||||
entries = [
|
||||
json.dumps(section) + ': {{ include "seaweedfs.existingTomlKey" (list '
|
||||
+ json.dumps(section) + ' .Values.raw) | toJson }}'
|
||||
for section in SECTIONS
|
||||
]
|
||||
template = chart / "templates/keys.yaml"
|
||||
prefix = '{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"keys"},"data":{'
|
||||
helper_template = prefix + ",".join(entries) + "}}"
|
||||
reference_entries = [
|
||||
json.dumps(section) + ': {{ dig '
|
||||
+ " ".join(json.dumps(part) for part in section.split("."))
|
||||
+ ' "key" "__ABSENT__" (fromToml .Values.raw) | toJson }}'
|
||||
for section in SECTIONS
|
||||
]
|
||||
reference_template = prefix + ",".join(reference_entries) + "}}"
|
||||
|
||||
def render(binary, source, raw):
|
||||
template.write_text(source)
|
||||
values = chart / "input.json"
|
||||
values.write_text(json.dumps({"raw": raw}))
|
||||
output = run(binary, "template", "keys", str(chart), "-f", str(values))
|
||||
return json.loads(output[output.index("{"):])["data"]
|
||||
|
||||
for name, raw in fixtures():
|
||||
expected = keys(raw)
|
||||
tokens = render(helm, helper_template, raw)
|
||||
actual = {section: tomllib.loads("key = " + token)["key"]
|
||||
for section, token in tokens.items() if token != ""}
|
||||
assert actual == expected, f"{name}: extracted keys differ from stored TOML"
|
||||
if reference_helm:
|
||||
reference = render(reference_helm, reference_template, raw)
|
||||
reference = {section: value for section, value in reference.items() if value != "__ABSENT__"}
|
||||
assert actual == reference, f"{name}: keys differ from fromToml/dig"
|
||||
print(f"PASS helper: {name}")
|
||||
|
||||
# A present but unsupported value must not silently become a fresh key.
|
||||
try:
|
||||
render(helm, helper_template, '[jwt.signing]\nkey = """multi\nline"""')
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "refusing to replace an existing key" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("multiline existing key was silently accepted or replaced")
|
||||
print("PASS helper: unsupported existing value fails without rotation")
|
||||
|
||||
try:
|
||||
render(helm, helper_template, ESCAPED_HEADER)
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "unsupported quoted section header" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("unsupported quoted header silently rotated its key")
|
||||
print("PASS helper: unsupported quoted header fails without rotation")
|
||||
|
||||
|
||||
def check_upgrades(helm, context):
|
||||
namespace = "jwt-key-persist-" + uuid.uuid4().hex[:8]
|
||||
current = "jk-seaweedfs-security-config"
|
||||
legacy = "seaweedfs-security-config"
|
||||
kubectl = ["kubectl", "--context", context, "-n", namespace]
|
||||
release_args = ["jk", str(CHART), "--kube-context", context, "-n", namespace]
|
||||
# No workload is needed to exercise Helm's real ConfigMap lookup and update.
|
||||
for setting in (
|
||||
"master.enabled=false", "volume.enabled=false", "filer.enabled=false",
|
||||
"global.seaweedfs.createClusterRole=false",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.volumeRead=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerWrite=true",
|
||||
"global.seaweedfs.securityConfig.jwtSigning.filerRead=true",
|
||||
):
|
||||
release_args += ["--set", setting]
|
||||
|
||||
def stored():
|
||||
cm = json.loads(run(*kubectl, "get", "configmap", current, "-o", "json"))
|
||||
return keys(cm["data"]["security.toml"])
|
||||
|
||||
def upgrade():
|
||||
run(helm, "upgrade", *release_args)
|
||||
return stored()
|
||||
|
||||
def patch(raw):
|
||||
# Seed previous-release content without taking Helm 4's SSA ownership.
|
||||
run(*kubectl, "patch", "configmap", current, "--type=merge", "--field-manager=helm", "-p",
|
||||
json.dumps({"data": {"security.toml": raw}}))
|
||||
|
||||
run(*kubectl, "create", "namespace", namespace)
|
||||
try:
|
||||
run(helm, "install", *release_args)
|
||||
initial = stored()
|
||||
assert set(initial) == set(SECTIONS), "install omitted a JWT section"
|
||||
for value in initial.values():
|
||||
check_generated(value)
|
||||
assert upgrade() == initial, "no-op upgrade changed an existing key"
|
||||
print("PASS upgrade: all four generated keys persist")
|
||||
|
||||
for name, raw in fixtures():
|
||||
patch(raw)
|
||||
actual = upgrade()
|
||||
expected = keys(raw)
|
||||
assert set(actual) == set(SECTIONS), f"{name}: upgrade omitted a JWT section"
|
||||
for section in SECTIONS:
|
||||
if section in expected:
|
||||
assert actual[section] == expected[section], f"{name}: changed {section}"
|
||||
else:
|
||||
check_generated(actual[section])
|
||||
assert upgrade() == actual, f"{name}: subsequent upgrade changed a key"
|
||||
print(f"PASS upgrade: {name}")
|
||||
|
||||
# Migration from the old chart name, followed by precedence of the current name.
|
||||
legacy_raw = "\n".join(f'[{section}]\nkey = "legacy-{index}"'
|
||||
for index, section in enumerate(SECTIONS))
|
||||
run(*kubectl, "create", "configmap", legacy, "--from-literal=security.toml=" + legacy_raw)
|
||||
run(*kubectl, "delete", "configmap", current)
|
||||
assert upgrade() == keys(legacy_raw), "legacy ConfigMap keys were not preserved"
|
||||
print("PASS upgrade: legacy ConfigMap migration")
|
||||
current_raw = next(fixtures())[1]
|
||||
patch(current_raw)
|
||||
assert upgrade() == keys(current_raw), "legacy ConfigMap overrode current ConfigMap"
|
||||
print("PASS upgrade: current ConfigMap takes precedence")
|
||||
|
||||
patch(ESCAPED_HEADER)
|
||||
try:
|
||||
upgrade()
|
||||
except subprocess.CalledProcessError as error:
|
||||
assert "unsupported quoted section header" in error.stderr, error.stderr
|
||||
else:
|
||||
raise AssertionError("unsupported quoted header silently rotated its key")
|
||||
assert keys(run(*kubectl, "get", "configmap", current, "-o",
|
||||
"jsonpath={.data.security\\.toml}"))["jwt.signing"] == "existing"
|
||||
print("PASS upgrade: unsupported quoted header leaves stored key untouched")
|
||||
finally:
|
||||
run(*kubectl, "delete", "namespace", namespace, "--wait=false")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--helm", default="helm")
|
||||
parser.add_argument("--reference-helm", help="Helm >=3.17 binary for differential checks")
|
||||
parser.add_argument("--context", help="explicit disposable Kubernetes context for live upgrade checks")
|
||||
args = parser.parse_args()
|
||||
print(run(args.helm, "version", "--short").strip())
|
||||
check_helpers(args.helm, args.reference_helm)
|
||||
if args.context:
|
||||
check_upgrades(args.helm, args.context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except subprocess.CalledProcessError as error:
|
||||
print(error.stderr, file=sys.stderr)
|
||||
sys.exit(error.returncode)
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4.38.3
|
||||
uses: github/codeql-action/init@v4.38.2
|
||||
# Override language selection by uncommenting this and choosing your languages
|
||||
with:
|
||||
languages: go
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below).
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4.38.3
|
||||
uses: github/codeql-action/autobuild@v4.38.2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -49,4 +49,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4.38.3
|
||||
uses: github/codeql-action/analyze@v4.38.2
|
||||
|
||||
@@ -405,7 +405,7 @@ jobs:
|
||||
output: trivy-results.sarif
|
||||
exit-code: '0'
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v4.38.3
|
||||
uses: github/codeql-action/upload-sarif@v4.38.2
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
|
||||
@@ -460,7 +460,7 @@ jobs:
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security
|
||||
if: always()
|
||||
uses: github/codeql-action/upload-sarif@v4.38.3
|
||||
uses: github/codeql-action/upload-sarif@v4.38.2
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: trivy-${{ matrix.variant }}
|
||||
|
||||
@@ -148,21 +148,3 @@ jobs:
|
||||
# -short skips the e2e suites already covered on amd64.
|
||||
- name: Test linux/386
|
||||
run: cd weed; GOOS=linux GOARCH=386 go test -short ./...
|
||||
|
||||
build-azurekms:
|
||||
name: Build and test with azurekms tag
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v7
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
# weed/kms/azure is excluded from every default build, so without this
|
||||
# nothing compiles it and tag-gated code silently rots.
|
||||
- name: Build and test with azurekms
|
||||
run: |
|
||||
cd weed
|
||||
go build -tags azurekms ./...
|
||||
go test -tags azurekms ./kms/...
|
||||
|
||||
@@ -3,10 +3,10 @@ name: "helm: lint and test charts"
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
|
||||
paths: ['k8s/**', '.github/workflows/helm_ci.yml']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py']
|
||||
paths: ['k8s/**', '.github/workflows/helm_ci.yml']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -20,14 +20,6 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm before fromToml was available
|
||||
uses: azure/setup-helm@v5
|
||||
with:
|
||||
version: v3.16.3
|
||||
|
||||
- name: Record legacy Helm binary
|
||||
run: echo "HELM_LEGACY=$(command -v helm)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v5
|
||||
with:
|
||||
@@ -39,7 +31,7 @@ jobs:
|
||||
check-latest: true
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@v2.9.0
|
||||
uses: helm/chart-testing-action@v2.8.0
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
@@ -52,17 +44,6 @@ jobs:
|
||||
- name: Run chart-testing (lint)
|
||||
run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
|
||||
|
||||
- name: Verify legacy Helm rendering
|
||||
run: |
|
||||
"$HELM_LEGACY" lint k8s/charts/seaweedfs
|
||||
"$HELM_LEGACY" template test k8s/charts/seaweedfs > "$RUNNER_TEMP/legacy-default.yaml"
|
||||
"$HELM_LEGACY" template test k8s/charts/seaweedfs \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
|
||||
--set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
|
||||
> "$RUNNER_TEMP/legacy-jwt.yaml"
|
||||
|
||||
|
||||
- name: Verify template rendering
|
||||
run: |
|
||||
set -e
|
||||
@@ -76,60 +57,7 @@ jobs:
|
||||
helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
|
||||
grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
|
||||
echo "S3 deployment renders correctly"
|
||||
|
||||
echo "=== Testing S3 rollout settings ==="
|
||||
helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true > /tmp/s3-rollout-defaults.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 10$" /tmp/s3-rollout-defaults.yaml
|
||||
test "$(grep -cE '^ strategy:|^ +lifecycle:' /tmp/s3-rollout-defaults.yaml)" -eq 0
|
||||
helm template test $CHART_DIR --show-only templates/s3/s3-deployment.yaml \
|
||||
--set s3.enabled=true \
|
||||
--set s3.terminationGracePeriodSeconds=30 \
|
||||
--set s3.updateStrategy.type=RollingUpdate \
|
||||
--set s3.updateStrategy.rollingUpdate.maxUnavailable=0 \
|
||||
--set-string 's3.lifecycle.preStop.exec.command={sleep,5}' \
|
||||
> /tmp/s3-rollout.yaml
|
||||
grep -q "terminationGracePeriodSeconds: 30$" /tmp/s3-rollout.yaml
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "maxUnavailable: 0"
|
||||
grep -A 4 "^ strategy:" /tmp/s3-rollout.yaml | grep -q "type: RollingUpdate"
|
||||
grep -A 5 "^ lifecycle:" /tmp/s3-rollout.yaml | grep -q -- "- sleep"
|
||||
echo "S3 rollout settings render correctly"
|
||||
|
||||
echo "=== Testing the S3 config checksum is stable across renders ==="
|
||||
# s3-secret.yaml keeps its generated keys with lookup. Hashing the rendered
|
||||
# Secret drew new keys for the checksum whenever lookup found nothing (on
|
||||
# install), so the first upgrade changed the checksum and restarted the
|
||||
# filer and S3 pods although nothing had changed.
|
||||
s3_checksum() { # <template> <helm args...>: the checksum/s3config of one workload
|
||||
local template=$1; shift
|
||||
local sum
|
||||
sum=$(helm template test $CHART_DIR --show-only "$template" "$@" \
|
||||
| grep 'checksum/s3config:' | awk '{print $2}')
|
||||
test -n "$sum" || { echo "no checksum/s3config in $template" >&2; return 1; }
|
||||
echo "$sum"
|
||||
}
|
||||
s3_auth_args=(--set s3.enabled=true --set s3.enableAuth=true)
|
||||
s3_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")
|
||||
filer_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}")
|
||||
test "$s3_sum" = "$filer_sum" # one Secret, one checksum
|
||||
test "$s3_sum" = "$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}")"
|
||||
s3_creds_sum=$(s3_checksum templates/s3/s3-deployment.yaml "${s3_auth_args[@]}" \
|
||||
--set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
|
||||
test "$s3_sum" != "$s3_creds_sum"
|
||||
filer_audit_sum=$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
|
||||
--set-json 'filer.s3.auditLogConfig={"fluent_port":24224}')
|
||||
test "$filer_sum" != "$filer_audit_sum"
|
||||
# Settings the Secret does not read must not restart the S3 and filer pods.
|
||||
test "$filer_sum" = "$(s3_checksum templates/filer/filer-statefulset.yaml "${s3_auth_args[@]}" \
|
||||
--set s3.replicas=3 --set s3.resources.limits.cpu=1)"
|
||||
all_in_one_args=(--set allInOne.enabled=true --set allInOne.s3.enabled=true --set allInOne.s3.enableAuth=true)
|
||||
aio_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")
|
||||
test "$aio_sum" = "$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}")"
|
||||
aio_creds_sum=$(s3_checksum templates/all-in-one/all-in-one-deployment.yaml "${all_in_one_args[@]}" \
|
||||
--set s3.credentials.admin.accessKey=a --set s3.credentials.admin.secretKey=b)
|
||||
test "$aio_sum" != "$aio_creds_sum"
|
||||
echo "S3 config checksum is stable and follows only what the S3 Secret reads"
|
||||
|
||||
|
||||
echo "=== Testing S3 credentials from an existing secret ==="
|
||||
credential_args=(
|
||||
--set s3.credentials.admin.existingSecret=minio-root
|
||||
@@ -168,23 +96,6 @@ jobs:
|
||||
grep -A 12 "^kind: Ingress" /tmp/s3-ingress-labels.yaml | grep -q "^ external-dns: s3"
|
||||
echo "S3 ingress renders custom labels"
|
||||
|
||||
echo "=== Testing HTTPRoute ==="
|
||||
helm template test $CHART_DIR --show-only templates/s3/s3-httproute.yaml \
|
||||
--set s3.enabled=true,s3.httpRoute.enabled=true \
|
||||
--set 's3.httpRoute.parentRefs[0].name=gateway' > /tmp/s3-httproute.yaml
|
||||
grep -q "^kind: HTTPRoute" /tmp/s3-httproute.yaml
|
||||
grep -A 4 -- '- group: ""' /tmp/s3-httproute.yaml | grep -q "name: test-seaweedfs-s3$"
|
||||
grep -A 4 -- '- group: ""' /tmp/s3-httproute.yaml | grep -q "port: 8333$"
|
||||
echo "S3 HTTPRoute routes to the s3 service by default"
|
||||
helm template test $CHART_DIR --show-only templates/s3/s3-httproute.yaml \
|
||||
--set filer.s3.enabled=true,filer.s3.port=8334,s3.httpRoute.enabled=true > /tmp/filer-s3-httproute.yaml
|
||||
grep -A 4 -- '- group: ""' /tmp/filer-s3-httproute.yaml | grep -q "port: 8334$"
|
||||
echo "S3 on filer HTTPRoute uses filer.s3.port"
|
||||
helm template test $CHART_DIR --show-only templates/master/master-httproute.yaml \
|
||||
--set allInOne.enabled=true,master.enabled=false,master.httpRoute.enabled=true > /tmp/allinone-master-httproute.yaml
|
||||
grep -A 4 -- '- group: ""' /tmp/allinone-master-httproute.yaml | grep -q "name: test-seaweedfs-all-in-one$"
|
||||
echo "All-in-one master HTTPRoute routes to the all-in-one service"
|
||||
|
||||
echo "=== Testing with all-in-one mode ==="
|
||||
helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
|
||||
grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
|
||||
@@ -205,21 +116,6 @@ jobs:
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "Security configuration renders correctly"
|
||||
|
||||
echo "=== Testing secure bucket-creation hook certificate mounts ==="
|
||||
helm template test $CHART_DIR \
|
||||
--show-only templates/shared/post-install-bucket-hook.yaml \
|
||||
--set s3.enabled=true \
|
||||
--set 's3.createBuckets[0].name=data' \
|
||||
--set global.seaweedfs.enableSecurity=true \
|
||||
> /tmp/security-bucket-hook.yaml
|
||||
test "$(grep -cE '^[[:space:]]*- name: ca-cert$' /tmp/security-bucket-hook.yaml)" -eq 2
|
||||
test "$(grep -cE '^[[:space:]]*- name: client-cert$' /tmp/security-bucket-hook.yaml)" -eq 2
|
||||
grep -q 'mountPath: /usr/local/share/ca-certificates/ca/' /tmp/security-bucket-hook.yaml
|
||||
grep -q 'mountPath: /usr/local/share/ca-certificates/client/' /tmp/security-bucket-hook.yaml
|
||||
grep -q 'secretName: test-seaweedfs-ca-cert' /tmp/security-bucket-hook.yaml
|
||||
grep -q 'secretName: test-seaweedfs-client-cert' /tmp/security-bucket-hook.yaml
|
||||
echo "Secure bucket-creation hook mounts its CA and client certificate"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing admin.allowInsecureBind satisfies the admin auth render guard ==="
|
||||
helm template test $CHART_DIR --set admin.enabled=true --set admin.allowInsecureBind=true \
|
||||
@@ -822,168 +718,6 @@ jobs:
|
||||
helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
|
||||
grep -q "seaweedfs-cosi" /tmp/cosi.yaml
|
||||
echo "COSI driver renders correctly"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing configurable pod and container security contexts ==="
|
||||
helm template test $CHART_DIR > /tmp/security-context-defaults.yaml
|
||||
|
||||
security_context_args=()
|
||||
for component in master volume filer s3 sftp admin worker allInOne cosi; do
|
||||
security_context_args+=(
|
||||
--set "$component.podSecurityContext.enabled=true"
|
||||
--set "$component.podSecurityContext.seccompProfile.type=RuntimeDefault"
|
||||
--set "$component.containerSecurityContext.enabled=true"
|
||||
--set "$component.containerSecurityContext.privileged=false"
|
||||
--set "$component.containerSecurityContext.allowPrivilegeEscalation=false"
|
||||
--set "$component.containerSecurityContext.readOnlyRootFilesystem=true"
|
||||
--set "$component.containerSecurityContext.capabilities.drop[0]=ALL"
|
||||
--set "$component.containerSecurityContext.seccompProfile.type=RuntimeDefault"
|
||||
)
|
||||
done
|
||||
|
||||
helm template test $CHART_DIR \
|
||||
"${security_context_args[@]}" \
|
||||
--set s3.enabled=true \
|
||||
--set s3.createBuckets[0].name=test \
|
||||
--set sftp.enabled=true \
|
||||
--set admin.enabled=true \
|
||||
--set admin.secret.adminPassword=ci-admin-password \
|
||||
--set worker.enabled=true \
|
||||
--set volume.idx.type=hostPath \
|
||||
--set volume.idx.hostPathPrefix=/tmp \
|
||||
--set cosi.enabled=true \
|
||||
--set global.seaweedfs.tmpDir.sizeLimit=64Mi > /tmp/security-contexts.yaml
|
||||
helm template test $CHART_DIR \
|
||||
"${security_context_args[@]}" \
|
||||
--set allInOne.enabled=true \
|
||||
--set master.enabled=false \
|
||||
--set volume.enabled=false \
|
||||
--set filer.enabled=false \
|
||||
--set global.seaweedfs.tmpDir.sizeLimit=64Mi > /tmp/security-contexts-aio.yaml
|
||||
python3 - /tmp/security-context-defaults.yaml /tmp/security-contexts.yaml /tmp/security-contexts-aio.yaml <<'PYEOF'
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
errors = []
|
||||
workloads = 0
|
||||
containers = 0
|
||||
chart_managed_init_containers = 0
|
||||
components = set()
|
||||
|
||||
def validate_container(workload_name, container):
|
||||
context = container.get("securityContext", {})
|
||||
prefix = f"{workload_name}/{container['name']}"
|
||||
if "enabled" in context:
|
||||
errors.append(f"{prefix}: internal enabled flag leaked into container securityContext")
|
||||
if context.get("privileged") is not False:
|
||||
errors.append(f"{prefix}: privileged is not false")
|
||||
if context.get("allowPrivilegeEscalation") is not False:
|
||||
errors.append(f"{prefix}: allowPrivilegeEscalation is not false")
|
||||
if context.get("readOnlyRootFilesystem") is not True:
|
||||
errors.append(f"{prefix}: readOnlyRootFilesystem is not true")
|
||||
if context.get("capabilities", {}).get("drop") != ["ALL"]:
|
||||
errors.append(f"{prefix}: capabilities.drop is not exactly [ALL]")
|
||||
if context.get("seccompProfile", {}).get("type") != "RuntimeDefault":
|
||||
errors.append(f"{prefix}: seccompProfile is not RuntimeDefault")
|
||||
mounts = {mount["name"]: mount for mount in container.get("volumeMounts", [])}
|
||||
if mounts.get("seaweedfs-tmp", {}).get("mountPath") != "/tmp":
|
||||
errors.append(f"{prefix}: writable temporary volume is not mounted at /tmp")
|
||||
|
||||
with open(sys.argv[1]) as stream:
|
||||
default_documents = [document for document in yaml.safe_load_all(stream) if document]
|
||||
for document in default_documents:
|
||||
if document.get("kind") not in ("Deployment", "StatefulSet", "Job"):
|
||||
continue
|
||||
name = document["metadata"]["name"]
|
||||
pod = document["spec"]["template"]["spec"]
|
||||
if "securityContext" in pod:
|
||||
errors.append(f"{name}: pod securityContext should be absent by default")
|
||||
if any(volume["name"] == "seaweedfs-tmp" for volume in pod.get("volumes", [])):
|
||||
errors.append(f"{name}: writable /tmp volume should be absent by default")
|
||||
for container in pod.get("containers", []):
|
||||
if "securityContext" in container:
|
||||
errors.append(
|
||||
f"{name}/{container['name']}: container securityContext "
|
||||
f"should be absent by default"
|
||||
)
|
||||
if any(
|
||||
mount["name"] == "seaweedfs-tmp"
|
||||
for mount in container.get("volumeMounts", [])
|
||||
):
|
||||
errors.append(
|
||||
f"{name}/{container['name']}: writable /tmp mount "
|
||||
f"should be absent by default"
|
||||
)
|
||||
|
||||
for path in sys.argv[2:]:
|
||||
with open(path) as stream:
|
||||
documents = [document for document in yaml.safe_load_all(stream) if document]
|
||||
for document in documents:
|
||||
if document.get("kind") not in ("Deployment", "StatefulSet", "Job"):
|
||||
continue
|
||||
workloads += 1
|
||||
name = document["metadata"]["name"]
|
||||
pod = document["spec"]["template"]["spec"]
|
||||
volumes = {volume["name"]: volume for volume in pod.get("volumes", [])}
|
||||
temporary = volumes.get("seaweedfs-tmp", {}).get("emptyDir")
|
||||
if temporary is None:
|
||||
errors.append(f"{name}: writable /tmp emptyDir is missing")
|
||||
elif temporary.get("sizeLimit") != "64Mi":
|
||||
errors.append(f"{name}: temporary volume sizeLimit is not 64Mi")
|
||||
component = document["spec"]["template"]["metadata"]["labels"].get("app.kubernetes.io/component")
|
||||
if component:
|
||||
components.add(component)
|
||||
else:
|
||||
errors.append(f"{name}: app.kubernetes.io/component label is missing")
|
||||
pod_context = pod.get("securityContext", {})
|
||||
if "enabled" in pod_context:
|
||||
errors.append(f"{name}: internal enabled flag leaked into pod securityContext")
|
||||
if pod_context.get("seccompProfile", {}).get("type") != "RuntimeDefault":
|
||||
errors.append(f"{name}: pod seccompProfile is not RuntimeDefault")
|
||||
for container in pod.get("containers", []):
|
||||
containers += 1
|
||||
validate_container(name, container)
|
||||
for container in pod.get("initContainers", []):
|
||||
if container["name"] != "seaweedfs-vol-move-idx":
|
||||
continue
|
||||
chart_managed_init_containers += 1
|
||||
validate_container(name, container)
|
||||
|
||||
expected_components = {
|
||||
"master", "volume", "filer", "s3", "sftp", "admin", "worker",
|
||||
"objectstorage-provisioner", "bucket-hook", "seaweedfs-all-in-one",
|
||||
}
|
||||
if components != expected_components:
|
||||
errors.append(
|
||||
f"security context workload coverage is incomplete: "
|
||||
f"expected {sorted(expected_components)}, got {sorted(components)}"
|
||||
)
|
||||
if workloads != 10 or containers != 12:
|
||||
errors.append(
|
||||
f"expected 10 workloads and 12 containers, got "
|
||||
f"{workloads} workloads and {containers} containers"
|
||||
)
|
||||
if chart_managed_init_containers != 1:
|
||||
errors.append(
|
||||
f"expected one chart-managed seaweedfs-vol-move-idx init container, "
|
||||
f"got {chart_managed_init_containers}"
|
||||
)
|
||||
|
||||
if errors:
|
||||
print("\n".join(f"FAIL: {error}" for error in errors), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"Validated security contexts on {workloads} workloads and {containers} containers")
|
||||
PYEOF
|
||||
|
||||
# The resize hook depends on lookup finding a live StatefulSet and
|
||||
# therefore cannot render during helm template. Keep static coverage
|
||||
# for both security-context blocks.
|
||||
grep -Fqx ' securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }}' \
|
||||
"$CHART_DIR/templates/volume/volume-resize-hook.yaml"
|
||||
grep -Fqx ' securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}' \
|
||||
"$CHART_DIR/templates/volume/volume-resize-hook.yaml"
|
||||
echo "Volume resize hook security contexts are covered"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing long release name: service names match DNS references ==="
|
||||
@@ -1869,79 +1603,7 @@ jobs:
|
||||
echo "IMAGE_TAG=$TAG" >> $GITHUB_ENV
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.15.1
|
||||
|
||||
- name: Verify volume resize hook security contexts
|
||||
run: |
|
||||
set -e
|
||||
CHART_DIR="k8s/charts/seaweedfs"
|
||||
NS="resize-hook-security"
|
||||
kubectl create namespace "$NS"
|
||||
kubectl apply -n "$NS" -f - <<'EOF'
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: data1-resize-seaweedfs-volume-0
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
EOF
|
||||
|
||||
helm template resize "$CHART_DIR" -n "$NS" --dry-run=server \
|
||||
--set volume.dataDirs[0].name=data1 \
|
||||
--set volume.dataDirs[0].type=persistentVolumeClaim \
|
||||
--set volume.dataDirs[0].size=2Gi \
|
||||
--set volume.podSecurityContext.enabled=true \
|
||||
--set volume.podSecurityContext.seccompProfile.type=RuntimeDefault \
|
||||
--set volume.containerSecurityContext.enabled=true \
|
||||
--set volume.containerSecurityContext.privileged=false \
|
||||
--set volume.containerSecurityContext.allowPrivilegeEscalation=false \
|
||||
--set volume.containerSecurityContext.readOnlyRootFilesystem=true \
|
||||
--set volume.containerSecurityContext.capabilities.drop[0]=ALL \
|
||||
--set volume.containerSecurityContext.seccompProfile.type=RuntimeDefault \
|
||||
--set global.seaweedfs.tmpDir.sizeLimit=64Mi \
|
||||
> /tmp/security-context-resize-hook.yaml
|
||||
|
||||
python3 - /tmp/security-context-resize-hook.yaml <<'PYEOF'
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
with open(sys.argv[1]) as stream:
|
||||
documents = [document for document in yaml.safe_load_all(stream) if document]
|
||||
|
||||
jobs = [
|
||||
document for document in documents
|
||||
if document.get("kind") == "Job"
|
||||
and document["metadata"]["name"].endswith("-volume-resize-hook")
|
||||
]
|
||||
if len(jobs) != 1:
|
||||
raise AssertionError(f"expected one volume resize hook Job, got {len(jobs)}")
|
||||
|
||||
pod = jobs[0]["spec"]["template"]["spec"]
|
||||
assert pod["securityContext"] == {
|
||||
"seccompProfile": {"type": "RuntimeDefault"},
|
||||
}
|
||||
assert len(pod["containers"]) == 1
|
||||
assert pod["containers"][0]["securityContext"] == {
|
||||
"allowPrivilegeEscalation": False,
|
||||
"capabilities": {"drop": ["ALL"]},
|
||||
"privileged": False,
|
||||
"readOnlyRootFilesystem": True,
|
||||
"seccompProfile": {"type": "RuntimeDefault"},
|
||||
}
|
||||
assert pod["containers"][0]["volumeMounts"] == [
|
||||
{"mountPath": "/tmp", "name": "seaweedfs-tmp"},
|
||||
]
|
||||
assert pod["volumes"] == [
|
||||
{"emptyDir": {"sizeLimit": "64Mi"}, "name": "seaweedfs-tmp"},
|
||||
]
|
||||
PYEOF
|
||||
kubectl delete namespace "$NS"
|
||||
echo "Volume resize hook security contexts render correctly"
|
||||
uses: helm/kind-action@v1.15.0
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
run: |
|
||||
@@ -1993,16 +1655,6 @@ jobs:
|
||||
kubectl delete namespace "$NS"
|
||||
echo "SFTP host key lifecycle tests passed"
|
||||
|
||||
- name: Verify JWT signing key persistence across upgrades
|
||||
run: |
|
||||
# chart-testing puts its pip-less venv first on PATH; use setup-python.
|
||||
PYTHON="$pythonLocation/bin/python3"
|
||||
"$PYTHON" -m pip install tomli==2.2.1
|
||||
CONTEXT=$(kubectl config current-context)
|
||||
"$PYTHON" .github/scripts/helm_jwt_keys.py --context "$CONTEXT"
|
||||
"$PYTHON" .github/scripts/helm_jwt_keys.py --helm "$HELM_LEGACY" \
|
||||
--reference-helm helm --context "$CONTEXT"
|
||||
|
||||
- name: Verify install into a default-deny namespace
|
||||
run: |
|
||||
set -e
|
||||
|
||||
@@ -139,8 +139,7 @@ jobs:
|
||||
unit-tests:
|
||||
name: Go Unit Tests (Implicit Directory)
|
||||
runs-on: ubuntu-latest
|
||||
# Leave time for setup and the focused run before the nine-minute suite.
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -160,5 +159,5 @@ jobs:
|
||||
- name: Run all S3 API tests
|
||||
run: |
|
||||
cd weed/s3api
|
||||
go test -v -timeout 9m
|
||||
go test -v -timeout 5m
|
||||
|
||||
|
||||
@@ -289,10 +289,6 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_object_write_check_etag \
|
||||
s3tests/functional/test_s3.py::test_object_write_cache_control \
|
||||
s3tests/functional/test_s3.py::test_object_write_expires \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_s3 \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_bad_enc_kms \
|
||||
s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
|
||||
s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
|
||||
s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
|
||||
@@ -1155,10 +1151,6 @@ jobs:
|
||||
s3tests/functional/test_s3.py::test_object_write_check_etag \
|
||||
s3tests/functional/test_s3.py::test_object_write_cache_control \
|
||||
s3tests/functional/test_s3.py::test_object_write_expires \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_s3 \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_c_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms \
|
||||
s3tests/functional/test_s3.py::test_put_obj_enc_conflict_bad_enc_kms \
|
||||
s3tests/functional/test_s3.py::test_object_content_encoding_aws_chunked \
|
||||
s3tests/functional/test_s3.py::test_object_write_read_update_read_delete \
|
||||
s3tests/functional/test_s3.py::test_object_metadata_replaced_on_put \
|
||||
|
||||
@@ -82,8 +82,6 @@ AWS_ACCESS_KEY_ID=admin AWS_SECRET_ACCESS_KEY=secret \
|
||||
|
||||
The same process also runs the master, a volume server, the filer, WebDAV, the Iceberg REST catalog, and the Admin UI. Add `S3_TABLE_BUCKET=warehouse` to also create an Iceberg table bucket, or `warehouse:LANCE` for a Lance one. Drop the AWS keys to run without authentication for development.
|
||||
|
||||
Without Admin authentication or mTLS, `weed mini` binds the Admin UI/API and its worker gRPC control plane to loopback rather than `-ip.bind`. Set `WEED_ADMIN_PASSWORD` or configure `https.admin` mTLS to keep the network bind. Remote workers must opt in with `-admin.worker.ip=<address>` and should configure `grpc.admin` mTLS. `-admin.allowInsecureBind` restores the legacy unauthenticated network bind and should only be used on an isolated network.
|
||||
|
||||
> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
|
||||
|
||||
`weed mini` is auto-tuned for one node and is fine for single-node production, such as an S3 gateway that issues presigned URLs. See [Quick Start with weed mini][WeedMini].
|
||||
|
||||
@@ -169,7 +169,6 @@ seaweed-volume/
|
||||
- `--memprofile`
|
||||
- `--compactionMBps` (default 0)
|
||||
- `--maintenanceMBps` (default 0)
|
||||
- `--compactionMaxParallelCommits` (default 0)
|
||||
- `--fileSizeLimitMB` (default 256)
|
||||
- `--concurrentUploadLimitMB` (default 0)
|
||||
- `--concurrentDownloadLimitMB` (default 0)
|
||||
|
||||
@@ -543,7 +543,7 @@ a language-agnostic gRPC stream that external maintenance workers connect on: th
|
||||
opens `WorkerStream`, sends `WorkerHello` with the job types it can `detect` and `execute`,
|
||||
answers `RequestConfigSchema` with a `JobTypeDescriptor`, replies to `RunDetectionRequest`
|
||||
with `JobProposal`s and to `ExecuteJobRequest` with `JobProgressUpdate`s and `JobCompleted`.
|
||||
`weed worker -admin=host:12535` is the Go reference implementation of exactly that contract,
|
||||
`weed worker -admin=host:23646` is the Go reference implementation of exactly that contract,
|
||||
from outside the admin process.
|
||||
|
||||
Nothing in it is Go-specific, and the Rust toolchain is already in the tree.
|
||||
|
||||
@@ -21,7 +21,7 @@ The testing environment uses **official SeaweedFS commands** and includes:
|
||||
- **1 Master Server** (port 9333) - Coordinates the cluster with 50MB volume size limit
|
||||
- **6 Volume Servers** (ports 8080-8085) - Distributed across 2 data centers and 3 racks for diversity
|
||||
- **1 Filer** (port 8888) - Provides file system interface
|
||||
- **1 Admin Server** (port 12535) - Detects volumes needing EC and manages workers using official `admin` command
|
||||
- **1 Admin Server** (port 23646) - Detects volumes needing EC and manages workers using official `admin` command
|
||||
- **3 EC Workers** - Execute erasure coding tasks using official `worker` command with task-specific working directories
|
||||
- **1 Load Generator** - Continuously writes and deletes files using SeaweedFS shell commands
|
||||
- **1 Monitor** - Tracks cluster health and EC progress using shell scripts
|
||||
@@ -47,7 +47,7 @@ This provides:
|
||||
- Docker and Docker Compose installed
|
||||
- GNU Make installed
|
||||
- At least 4GB RAM available for containers
|
||||
- Ports 8080-8085, 8888, 9333, 12535 available
|
||||
- Ports 8080-8085, 8888, 9333, 23646 available
|
||||
|
||||
### Start the Environment
|
||||
|
||||
@@ -127,7 +127,7 @@ Run `make help` to see all available targets:
|
||||
|---------|-----|-------------|
|
||||
| Master UI | http://localhost:9333 | Cluster status and topology |
|
||||
| Filer | http://localhost:8888 | File operations |
|
||||
| Admin Server | http://localhost:12535/ | Task management |
|
||||
| Admin Server | http://localhost:23646/ | Task management |
|
||||
| Monitor | http://localhost:9999/status | Complete cluster monitoring |
|
||||
| Volume Servers | http://localhost:8080-8085/status | Individual volume server stats |
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ build-and-restart: build ## Build with latest changes and restart services
|
||||
@echo "2️⃣ Recreating workers to reconnect..."
|
||||
@docker-compose -f $(COMPOSE_FILE) up -d worker1 worker2 worker3
|
||||
@echo "✅ All services recreated with latest changes!"
|
||||
@echo "🌐 Admin UI: http://localhost:12535/"
|
||||
@echo "🌐 Admin UI: http://localhost:23646/"
|
||||
@echo "💡 Workers will reconnect to the new admin server"
|
||||
|
||||
restart-workers: ## Restart all workers to reconnect to admin server
|
||||
@@ -64,7 +64,7 @@ start: ## Start the complete SeaweedFS cluster with admin and workers
|
||||
@echo "✅ Cluster started!"
|
||||
@echo ""
|
||||
@echo "📊 Access points:"
|
||||
@echo " • Admin UI: http://localhost:12535/"
|
||||
@echo " • Admin UI: http://localhost:23646/"
|
||||
@echo " • Master UI: http://localhost:9333/"
|
||||
@echo " • Filer: http://localhost:8888/"
|
||||
@echo ""
|
||||
@@ -107,7 +107,7 @@ start-staged: ## Start services in proper order with delays
|
||||
@echo "✅ All services started!"
|
||||
@echo ""
|
||||
@echo "📊 Access points:"
|
||||
@echo " • Admin UI: http://localhost:12535/"
|
||||
@echo " • Admin UI: http://localhost:23646/"
|
||||
@echo " • Master UI: http://localhost:9333/"
|
||||
@echo " • Filer: http://localhost:8888/"
|
||||
@echo ""
|
||||
@@ -136,7 +136,7 @@ status: ## Check the status of all services
|
||||
@echo "Master:"
|
||||
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' 2>/dev/null || echo " ❌ Master not ready"
|
||||
@echo "Admin:"
|
||||
@curl -s http://localhost:12535/ | grep -q "Admin" && echo " ✅ Admin ready" || echo " ❌ Admin not ready"
|
||||
@curl -s http://localhost:23646/ | grep -q "Admin" && echo " ✅ Admin ready" || echo " ❌ Admin not ready"
|
||||
|
||||
logs: ## Show logs from all services
|
||||
@echo "📜 Following logs from all services..."
|
||||
@@ -157,7 +157,7 @@ master-logs: ## Show logs from master server
|
||||
|
||||
admin-ui: ## Open admin UI in browser (macOS)
|
||||
@echo "🌐 Opening admin UI in browser..."
|
||||
@open http://localhost:12535/ || echo "💡 Manually open: http://localhost:12535/"
|
||||
@open http://localhost:23646/ || echo "💡 Manually open: http://localhost:23646/"
|
||||
|
||||
test: ## Run integration test to verify task assignment and completion
|
||||
@echo "🧪 Running Admin-Worker Integration Test"
|
||||
@@ -166,11 +166,11 @@ test: ## Run integration test to verify task assignment and completion
|
||||
@echo "1️⃣ Checking cluster health..."
|
||||
@sleep 5
|
||||
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master healthy" || echo "❌ Master not ready"
|
||||
@curl -s http://localhost:12535/ | grep -q "Admin" && echo "✅ Admin healthy" || echo "❌ Admin not ready"
|
||||
@curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin healthy" || echo "❌ Admin not ready"
|
||||
@echo ""
|
||||
@echo "2️⃣ Checking worker registration..."
|
||||
@sleep 10
|
||||
@echo "💡 Check admin UI for connected workers: http://localhost:12535/"
|
||||
@echo "💡 Check admin UI for connected workers: http://localhost:23646/"
|
||||
@echo ""
|
||||
@echo "3️⃣ Generating load to trigger EC tasks..."
|
||||
@echo "📝 Creating test files to fill volumes..."
|
||||
@@ -193,13 +193,13 @@ test: ## Run integration test to verify task assignment and completion
|
||||
@echo " • EC shards being distributed"
|
||||
@echo ""
|
||||
@echo "✅ Integration test setup complete!"
|
||||
@echo "📊 Monitor progress at: http://localhost:12535/"
|
||||
@echo "📊 Monitor progress at: http://localhost:23646/"
|
||||
|
||||
quick-test: ## Quick verification that core services are running
|
||||
@echo "⚡ Quick Health Check"
|
||||
@echo "===================="
|
||||
@echo "Master: $$(curl -s http://localhost:9333/cluster/status | jq -r '.IsLeader // "not ready"')"
|
||||
@echo "Admin: $$(curl -s http://localhost:12535/ | grep -q "Admin" && echo "ready" || echo "not ready")"
|
||||
@echo "Admin: $$(curl -s http://localhost:23646/ | grep -q "Admin" && echo "ready" || echo "not ready")"
|
||||
@echo "Workers: $$(docker-compose -f $(COMPOSE_FILE) ps worker1 worker2 worker3 | grep -c Up) running"
|
||||
|
||||
validate: ## Validate integration test configuration
|
||||
@@ -223,7 +223,7 @@ demo: start ## Start cluster and run demonstration
|
||||
@echo " • Load generator (creates files continuously)"
|
||||
@echo ""
|
||||
@echo "📊 Watch the process:"
|
||||
@echo " 1. Visit: http://localhost:12535/"
|
||||
@echo " 1. Visit: http://localhost:23646/"
|
||||
@echo " 2. Observe workers connecting"
|
||||
@echo " 3. Watch tasks being created and assigned"
|
||||
@echo " 4. See tasks progress from pending → completed"
|
||||
@@ -244,13 +244,13 @@ vacuum-test: ## Create test data with garbage and verify vacuum detection
|
||||
@echo ""
|
||||
@echo "1️⃣ Checking cluster health..."
|
||||
@curl -s http://localhost:9333/cluster/status | jq '.IsLeader' > /dev/null && echo "✅ Master ready" || (echo "❌ Master not ready. Run 'make start' first." && exit 1)
|
||||
@curl -s http://localhost:12535/ | grep -q "Admin" && echo "✅ Admin ready" || (echo "❌ Admin not ready. Run 'make start' first." && exit 1)
|
||||
@curl -s http://localhost:23646/ | grep -q "Admin" && echo "✅ Admin ready" || (echo "❌ Admin not ready. Run 'make start' first." && exit 1)
|
||||
@echo ""
|
||||
@echo "2️⃣ Creating test data with garbage..."
|
||||
@docker-compose -f $(COMPOSE_FILE) exec vacuum-tester go run create_vacuum_test_data.go -files=25 -delete=0.5 -size=200
|
||||
@echo ""
|
||||
@echo "3️⃣ Configuration Instructions:"
|
||||
@echo " Visit: http://localhost:12535/maintenance/config/vacuum"
|
||||
@echo " Visit: http://localhost:23646/maintenance/config/vacuum"
|
||||
@echo " Set for testing:"
|
||||
@echo " • Enable Vacuum Tasks: ✅ Checked"
|
||||
@echo " • Garbage Threshold: 0.20 (20%)"
|
||||
@@ -258,7 +258,7 @@ vacuum-test: ## Create test data with garbage and verify vacuum detection
|
||||
@echo " • Min Volume Age: [0] [Minutes]"
|
||||
@echo " • Max Concurrent: 2"
|
||||
@echo ""
|
||||
@echo "4️⃣ Monitor vacuum tasks at: http://localhost:12535/maintenance"
|
||||
@echo "4️⃣ Monitor vacuum tasks at: http://localhost:23646/maintenance"
|
||||
@echo ""
|
||||
@echo "💡 Use 'make vacuum-status' to check volume garbage ratios"
|
||||
|
||||
@@ -335,8 +335,8 @@ vacuum-help: ## Show vacuum testing help and examples
|
||||
@echo " make vacuum-demo # Full guided demonstration"
|
||||
@echo ""
|
||||
@echo "Configuration:"
|
||||
@echo " Visit: http://localhost:12535/maintenance/config/vacuum"
|
||||
@echo " Monitor: http://localhost:12535/maintenance"
|
||||
@echo " Visit: http://localhost:23646/maintenance/config/vacuum"
|
||||
@echo " Monitor: http://localhost:23646/maintenance"
|
||||
@echo ""
|
||||
@echo "Custom Parameters:"
|
||||
@echo " make vacuum-data FILES=50 DELETE=0.8 SIZE=200"
|
||||
|
||||
@@ -243,7 +243,7 @@ func printTestingInstructions() {
|
||||
fmt.Println("🧪 Testing Instructions:")
|
||||
fmt.Println()
|
||||
fmt.Println("1. Configure Vacuum for Testing:")
|
||||
fmt.Println(" Visit: http://localhost:12535/maintenance/config/vacuum")
|
||||
fmt.Println(" Visit: http://localhost:23646/maintenance/config/vacuum")
|
||||
fmt.Println(" Set:")
|
||||
fmt.Printf(" - Garbage Percentage Threshold: 20 (20%% - lower than default 30)\n")
|
||||
fmt.Printf(" - Scan Interval: [30] [Seconds] (faster than default)\n")
|
||||
@@ -253,7 +253,7 @@ func printTestingInstructions() {
|
||||
fmt.Println()
|
||||
|
||||
fmt.Println("2. Monitor Vacuum Tasks:")
|
||||
fmt.Println(" Visit: http://localhost:12535/maintenance")
|
||||
fmt.Println(" Visit: http://localhost:23646/maintenance")
|
||||
fmt.Println(" Watch for vacuum tasks to appear in the queue")
|
||||
fmt.Println()
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ echo ""
|
||||
# Check if SeaweedFS is running
|
||||
echo "📋 Checking SeaweedFS status..."
|
||||
MASTER_URL="${MASTER_HOST:-master:9333}"
|
||||
ADMIN_URL="${ADMIN_HOST:-admin:12535}"
|
||||
ADMIN_URL="${ADMIN_HOST:-admin:23646}"
|
||||
|
||||
if ! curl -s http://$MASTER_URL/cluster/status > /dev/null; then
|
||||
echo "❌ SeaweedFS master not running at $MASTER_URL"
|
||||
@@ -42,7 +42,7 @@ echo ""
|
||||
|
||||
# Phase 3: Configure vacuum
|
||||
echo "⚙️ Phase 3: Vacuum configuration instructions..."
|
||||
echo " 1. Visit: http://localhost:12535/maintenance/config/vacuum"
|
||||
echo " 1. Visit: http://localhost:23646/maintenance/config/vacuum"
|
||||
echo " 2. Set these values for testing:"
|
||||
echo " - Enable Vacuum Tasks: ✅ Checked"
|
||||
echo " - Garbage Threshold: 0.30"
|
||||
@@ -57,7 +57,7 @@ echo ""
|
||||
|
||||
# Phase 4: Monitor tasks
|
||||
echo "🎯 Phase 4: Monitoring vacuum tasks..."
|
||||
echo " Visit: http://localhost:12535/maintenance"
|
||||
echo " Visit: http://localhost:23646/maintenance"
|
||||
echo " You should see vacuum tasks appear within 30 seconds"
|
||||
echo ""
|
||||
|
||||
@@ -93,9 +93,9 @@ echo ""
|
||||
echo "🎉 Demo Complete!"
|
||||
echo ""
|
||||
echo "🔍 Things to check:"
|
||||
echo " 1. Maintenance Queue: http://localhost:12535/maintenance"
|
||||
echo " 1. Maintenance Queue: http://localhost:23646/maintenance"
|
||||
echo " 2. Volume Status: http://localhost:9333/vol/status"
|
||||
echo " 3. Admin Dashboard: http://localhost:12535"
|
||||
echo " 3. Admin Dashboard: http://localhost:23646"
|
||||
echo ""
|
||||
echo "💡 Next Steps:"
|
||||
echo " - Try different garbage thresholds (0.10, 0.50, 0.80)"
|
||||
|
||||
@@ -114,9 +114,9 @@ services:
|
||||
admin:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
- "12535:12535" # HTTP admin interface (default port)
|
||||
- "22535:22535" # gRPC worker communication (12535 + 10000)
|
||||
command: "-v=2 admin -port=12535 -masters=master:9333 -dataDir=/data"
|
||||
- "23646:23646" # HTTP admin interface (default port)
|
||||
- "33646:33646" # gRPC worker communication (23646 + 10000)
|
||||
command: "-v=2 admin -port=23646 -masters=master:9333 -dataDir=/data"
|
||||
depends_on:
|
||||
- master
|
||||
- filer
|
||||
@@ -127,7 +127,7 @@ services:
|
||||
|
||||
worker1:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
depends_on:
|
||||
- admin
|
||||
volumes:
|
||||
@@ -139,7 +139,7 @@ services:
|
||||
|
||||
worker2:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
depends_on:
|
||||
- admin
|
||||
volumes:
|
||||
@@ -151,7 +151,7 @@ services:
|
||||
|
||||
worker3:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=2 worker -admin=admin:12535 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
||||
depends_on:
|
||||
- admin
|
||||
volumes:
|
||||
@@ -199,7 +199,7 @@ services:
|
||||
curl -s http://master:9333/cluster/status | jq '.IsLeader, .Peers' || echo 'Master not ready';
|
||||
echo;
|
||||
echo 'Admin status:';
|
||||
curl -s http://admin:12535/ | grep -o 'Admin.*Interface' || echo 'Admin not ready';
|
||||
curl -s http://admin:23646/ | grep -o 'Admin.*Interface' || echo 'Admin not ready';
|
||||
echo;
|
||||
echo 'Volume count by server:';
|
||||
curl -s http://master:9333/vol/status | jq '.Volumes | length' || echo 'Volumes not ready';
|
||||
@@ -237,4 +237,4 @@ services:
|
||||
- seaweed_net
|
||||
environment:
|
||||
- MASTER_HOST=master:9333
|
||||
- ADMIN_HOST=admin:12535
|
||||
- ADMIN_HOST=admin:23646
|
||||
@@ -23,7 +23,7 @@ else
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}2. Checking if required ports are available...${NC}"
|
||||
for port in 9333 8080 8081 8082 8083 8084 8085 8888 12535; do
|
||||
for port in 9333 8080 8081 8082 8083 8084 8085 8888 23646; do
|
||||
if lsof -i :$port > /dev/null 2>&1; then
|
||||
echo -e "${YELLOW}⚠️ Port $port is in use${NC}"
|
||||
else
|
||||
@@ -41,7 +41,7 @@ else
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}4. Verifying admin server configuration...${NC}"
|
||||
if docker-compose -f docker-compose-ec-test.yml config | grep -q "admin:12535"; then
|
||||
if docker-compose -f docker-compose-ec-test.yml config | grep -q "admin:23646"; then
|
||||
echo -e "${GREEN}✅ Admin server port configuration is correct${NC}"
|
||||
else
|
||||
echo -e "${RED}❌ Admin server port configuration is incorrect${NC}"
|
||||
|
||||
@@ -45,7 +45,7 @@ services:
|
||||
admin:
|
||||
image: chrislusf/seaweedfs:dev # use a remote dev image
|
||||
ports:
|
||||
- 12535:12535
|
||||
- 23646:23646
|
||||
command: 'admin -master=master:9333'
|
||||
depends_on:
|
||||
- master
|
||||
|
||||
@@ -23,8 +23,8 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.18.0
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.17.0
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
github.com/go-zookeeper/zk v1.0.4 // indirect
|
||||
github.com/golang/protobuf v1.5.4
|
||||
@@ -44,7 +44,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.20.1
|
||||
github.com/klauspost/compress v1.19.2
|
||||
github.com/klauspost/reedsolomon v1.14.2
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.8
|
||||
@@ -60,12 +60,12 @@ require (
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.24.1
|
||||
github.com/prometheus/client_model v0.6.3
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.22.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.2.2
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible
|
||||
github.com/seaweedfs/raft v1.2.1
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
@@ -73,7 +73,7 @@ require (
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.19.0
|
||||
github.com/tidwall/gjson v1.18.0
|
||||
github.com/tidwall/match v1.2.0
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
@@ -90,14 +90,14 @@ require (
|
||||
gocloud.dev v0.46.0
|
||||
gocloud.dev/pubsub/natspubsub v0.46.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.46.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597
|
||||
golang.org/x/image v0.46.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/net v0.58.0
|
||||
golang.org/x/oauth2 v0.37.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
golang.org/x/tools v0.51.0 // indirect
|
||||
golang.org/x/sys v0.48.0
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
golang.org/x/tools v0.49.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.297.0
|
||||
google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect
|
||||
@@ -116,16 +116,16 @@ require (
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.2
|
||||
github.com/a-h/templ v0.3.1070
|
||||
github.com/a-h/templ v0.3.1020
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.1.2
|
||||
github.com/apache/iceberg-go v0.7.0
|
||||
github.com/apache/iceberg-go v0.6.1-0.20260817192109-c2105090c9e2
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.3
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.35
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.4
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/cognusion/imaging v1.0.4
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
@@ -134,16 +134,16 @@ require (
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/hashicorp/raft v1.8.0
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.4.2
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.23.0
|
||||
github.com/jhump/protoreflect v1.18.1
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.32.0
|
||||
github.com/pkg/sftp v1.13.11
|
||||
github.com/rabbitmq/amqp091-go v1.15.0
|
||||
github.com/rabbitmq/amqp091-go v1.14.0
|
||||
github.com/rclone/rclone v1.75.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.22.0
|
||||
@@ -156,11 +156,11 @@ require (
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/twmb/avro v1.9.0
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.155.1
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.151.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1
|
||||
go.uber.org/atomic v1.12.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/sync v0.23.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0
|
||||
)
|
||||
@@ -168,22 +168,24 @@ require (
|
||||
require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
|
||||
|
||||
require (
|
||||
atomicgo.dev/cursor v0.2.0 // indirect
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v1.2.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.6.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/internal v0.1.0 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v3 v3.4.1 // indirect
|
||||
github.com/RoaringBitmap/roaring/v2 v2.26.0 // indirect
|
||||
github.com/RoaringBitmap/roaring/v2 v2.24.0 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/adrg/xdg v0.5.3 // indirect
|
||||
github.com/anchore/go-lzo v0.1.1 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.8.0 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.7.0 // indirect
|
||||
github.com/apache/thrift v0.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
@@ -198,6 +200,7 @@ require (
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect
|
||||
github.com/containerd/console v1.0.5 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
@@ -216,6 +219,7 @@ require (
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/gookit/color v1.6.0 // indirect
|
||||
github.com/gopherjs/gopherjs v1.17.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
@@ -233,6 +237,7 @@ require (
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lib/pq v1.12.0 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/lpar/calendar v0.2.0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
@@ -255,10 +260,12 @@ require (
|
||||
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/pterm/pterm v0.12.83 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/rclone/Proton-API-Bridge v1.0.5 // indirect
|
||||
github.com/rclone/go-proton-api v1.0.4 // indirect
|
||||
github.com/rogpeppe/go-internal v1.15.0 // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
@@ -274,6 +281,7 @@ require (
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 // indirect
|
||||
@@ -294,10 +302,10 @@ require (
|
||||
cloud.google.com/go/iam v1.12.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.30.0 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.2
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
|
||||
@@ -314,26 +322,26 @@ require (
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.12.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.6 // indirect
|
||||
github.com/andybalholm/brotli v1.2.2 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.4 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.24 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1
|
||||
github.com/aws/smithy-go v1.28.5
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.0
|
||||
github.com/aws/smithy-go v1.28.1
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
@@ -357,7 +365,7 @@ require (
|
||||
github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.39.1-0.20260819172001-e6e3fd93e4be // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
|
||||
github.com/fatih/color v1.19.0 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
@@ -389,10 +397,10 @@ require (
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-metrics v0.7.0 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.5 // indirect
|
||||
github.com/hashicorp/go-metrics v0.5.4 // indirect
|
||||
github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
|
||||
github.com/hashicorp/golang-lru v1.0.2 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
@@ -470,10 +478,10 @@ require (
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20240819112322-98a264d392f6 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810122915-65bfd5c4b705 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.5 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.2 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
@@ -484,15 +492,15 @@ require (
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect
|
||||
go.opentelemetry.io/otel v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.45.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/time v0.16.0
|
||||
golang.org/x/term v0.46.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
|
||||
+1
-1
@@ -350,7 +350,7 @@ main() {
|
||||
info " weed volume -mserver=localhost:9333 # Start Go volume server"
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
|
||||
info " weed-worker --admin localhost:23646 # Start the Rust maintenance worker"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@@ -3,4 +3,4 @@ description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.48"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.48.2
|
||||
version: 4.48.0
|
||||
|
||||
@@ -27,19 +27,6 @@ so your deployment will be spread/HA.
|
||||
* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
The chart's templates render with Helm 3.16.3 and newer versions tested in CI.
|
||||
Earlier Helm versions are not covered by this chart's compatibility checks.
|
||||
When JWT signing is enabled, a live upgrade reuses keys from the existing
|
||||
security ConfigMap (including the legacy ConfigMap name); a key absent from
|
||||
that ConfigMap is generated. Plain `helm template` does not read cluster state;
|
||||
use an install and upgrade against a cluster to check key persistence.
|
||||
|
||||
The key reader supports single-line quoted TOML strings in JWT sections,
|
||||
including indented assignments, quoted key names, and bare or simply quoted
|
||||
section-name segments. An unsupported value or quoted header fails the upgrade
|
||||
rather than silently rotating a key.
|
||||
|
||||
### Database
|
||||
|
||||
leveldb is the default database, this supports multiple filer replicas that will [sync automatically](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication), with some [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation).
|
||||
@@ -345,9 +332,8 @@ To enable the admin interface, add the following to your values.yaml:
|
||||
```yaml
|
||||
admin:
|
||||
enabled: true
|
||||
port: 12535
|
||||
grpcPort: 22535 # For worker connections
|
||||
legacyPort: 23646 # Compat: also serve http + gRPC (+10000) for pre-12535 clients
|
||||
port: 23646
|
||||
grpcPort: 33646 # For worker connections
|
||||
secret:
|
||||
adminUser: "admin"
|
||||
adminPassword: "your-secure-password" # Leave empty to disable auth
|
||||
@@ -365,9 +351,7 @@ admin:
|
||||
className: "nginx"
|
||||
```
|
||||
|
||||
The admin interface will be available at `http://<admin-service>:12535` (or via ingress). Workers connect to the admin server via gRPC on port `22535`.
|
||||
|
||||
For backward compatibility with older clients and workers, admin also serves the previous default ports: HTTP on `23646` and gRPC on `33646` (legacyPort + 10000). The Service and NetworkPolicy expose both. Set `admin.legacyPort: 0` to disable.
|
||||
The admin interface will be available at `http://<admin-service>:23646` (or via ingress). Workers connect to the admin server via gRPC on port `33646`.
|
||||
|
||||
### Admin Authentication
|
||||
|
||||
@@ -439,7 +423,7 @@ worker:
|
||||
# Auto-discovery only works if the admin is in the same namespace and same Helm release.
|
||||
# For cross-namespace or separate release scenarios, explicitly set this value.
|
||||
# Example: If main SeaweedFS is deployed in "production" namespace:
|
||||
# adminServer: "seaweedfs-admin.production.svc:22535"
|
||||
# adminServer: "seaweedfs-admin.production.svc:33646"
|
||||
adminServer: ""
|
||||
|
||||
# Workers need storage for task execution
|
||||
@@ -505,8 +489,8 @@ worker:
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
# adminServer: "seaweedfs-admin.production.svc:22535"
|
||||
adminServer: "seaweedfs-admin.<namespace>.svc:22535"
|
||||
# adminServer: "seaweedfs-admin.production.svc:33646"
|
||||
adminServer: "seaweedfs-admin.<namespace>.svc:33646"
|
||||
```
|
||||
|
||||
**values-worker-balance.yaml** (for balance operations):
|
||||
@@ -531,8 +515,8 @@ worker:
|
||||
# REQUIRED: Point to the admin service of your main SeaweedFS release
|
||||
# Replace <namespace> with the namespace where your main seaweedfs is deployed
|
||||
# Example: If deploying in namespace "production":
|
||||
# adminServer: "seaweedfs-admin.production.svc:22535"
|
||||
adminServer: "seaweedfs-admin.<namespace>.svc:22535"
|
||||
# adminServer: "seaweedfs-admin.production.svc:33646"
|
||||
adminServer: "seaweedfs-admin.<namespace>.svc:33646"
|
||||
```
|
||||
|
||||
Deploy the specialized workers as separate releases:
|
||||
@@ -545,26 +529,6 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum
|
||||
helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml
|
||||
```
|
||||
|
||||
## Gateway API
|
||||
|
||||
Every component with an `ingress` block (master, volume, filer, s3 and admin) can also be exposed through a [Gateway API](https://gateway-api.sigs.k8s.io/) `HTTPRoute`, for clusters that route through a Gateway instead of an Ingress controller. `<component>.httpRoute` sits next to `<component>.ingress` and is disabled by default. The chart does not create the Gateway, so point `parentRefs` at one that already exists, and the `gateway.networking.k8s.io/v1` CRDs must be installed.
|
||||
|
||||
```yaml
|
||||
s3:
|
||||
httpRoute:
|
||||
enabled: true
|
||||
parentRefs:
|
||||
- group: gateway.networking.k8s.io
|
||||
kind: Gateway
|
||||
name: my-gateway
|
||||
namespace: gateway-system
|
||||
sectionName: https
|
||||
hostnames:
|
||||
- s3.example.com
|
||||
```
|
||||
|
||||
With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`.
|
||||
|
||||
## Network Policies
|
||||
|
||||
In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on.
|
||||
@@ -606,23 +570,6 @@ Two things worth knowing before you turn this on:
|
||||
|
||||
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
|
||||
|
||||
## Pod and container security contexts
|
||||
|
||||
Pod and container security contexts are configurable independently for every built-in workload and remain empty by default for backwards compatibility. The examples in `values.yaml` show how to enable a `RuntimeDefault` seccomp profile, disable privilege escalation and privileged mode, drop all Linux capabilities, and use a read-only root filesystem.
|
||||
|
||||
SeaweedFS uses `/tmp` for Unix sockets, temporary uploads, worker task files, and other runtime data. When `readOnlyRootFilesystem` is enabled for a built-in component, the chart mounts a writable `emptyDir` at `/tmp` for its chart-managed containers. Its optional size limit can be configured globally:
|
||||
|
||||
```yaml
|
||||
global:
|
||||
seaweedfs:
|
||||
tmpDir:
|
||||
sizeLimit: 1Gi
|
||||
```
|
||||
|
||||
The chart does not enable `runAsNonRoot` by default because its default `hostPath` storage may be owned by root. To enforce the Kubernetes `restricted` Pod Security Standard, use storage that is writable by a non-root user and configure `runAsNonRoot` or use the OpenShift overrides below.
|
||||
|
||||
Security contexts configured for a component also apply to the chart-managed helper containers for that component. User-provided init containers and sidecars must define their own container security context and writable mounts.
|
||||
|
||||
## OpenShift Support
|
||||
|
||||
SeaweedFS can be deployed on OpenShift or any cluster enforcing the Kubernetes "restricted" Pod Security Standard. By default, OpenShift blocks containers that run as root or use `hostPath` volumes.
|
||||
@@ -631,7 +578,6 @@ To deploy on OpenShift, use the provided `openshift-values.yaml` which overrides
|
||||
1. Use `PersistentVolumeClaims` instead of `hostPath`.
|
||||
2. Enable `runAsNonRoot` and omit hardcoded UIDs to allow OpenShift to assign valid UIDs automatically.
|
||||
3. Apply appropriate `seccompProfile` and drop capabilities.
|
||||
4. Use a read-only root filesystem with writable temporary storage at `/tmp`.
|
||||
|
||||
Usage:
|
||||
```bash
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
# automatically assign a valid UID from the namespace's allocated range.
|
||||
# 3. Dropping all Linux capabilities and setting allowPrivilegeEscalation: false
|
||||
# 4. Enabling RuntimeDefault seccompProfile
|
||||
# 5. Using a read-only root filesystem with writable temporary storage
|
||||
#
|
||||
# Usage:
|
||||
# helm install seaweedfs seaweedfs/seaweedfs \
|
||||
@@ -50,7 +49,6 @@ master:
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
@@ -78,7 +76,6 @@ volume:
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
@@ -104,7 +101,6 @@ filer:
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
@@ -129,7 +125,6 @@ s3:
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
runAsNonRoot: true
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
{{- if and .Values.admin.enabled .Values.admin.httpRoute.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "admin") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- with .Values.admin.httpRoute.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- with .Values.admin.httpRoute.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.admin.httpRoute.parentRefs }}
|
||||
parentRefs:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.admin.httpRoute.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.admin.httpRoute.rules | default (list dict) }}
|
||||
- backendRefs:
|
||||
{{- with .backendRefs }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- else }}
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: {{ include "seaweedfs.componentName" (list $ "admin") }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
port: {{ $.Values.admin.port }}
|
||||
weight: 1
|
||||
{{- end }}
|
||||
{{- with .matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .filters }}
|
||||
filters:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .timeouts }}
|
||||
timeouts:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -37,16 +37,6 @@ spec:
|
||||
nodePort: {{ $nodePorts.grpc }}
|
||||
{{- end }}
|
||||
protocol: TCP
|
||||
{{- if gt (int .Values.admin.legacyPort) 0 }}
|
||||
- name: "legacy-http"
|
||||
port: {{ .Values.admin.legacyPort }}
|
||||
targetPort: {{ .Values.admin.legacyPort }}
|
||||
protocol: TCP
|
||||
- name: "legacy-grpc"
|
||||
port: {{ add (int .Values.admin.legacyPort) 10000 }}
|
||||
targetPort: {{ add (int .Values.admin.legacyPort) 10000 }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
|
||||
@@ -170,9 +170,6 @@ spec:
|
||||
-ip={{ .Values.admin.ip | default "0.0.0.0" }} \
|
||||
-port={{ .Values.admin.port }} \
|
||||
-port.grpc={{ .Values.admin.grpcPort }} \
|
||||
{{- if ne (int .Values.admin.legacyPort) 23646 }}
|
||||
-port.legacy={{ .Values.admin.legacyPort }} \
|
||||
{{- end }}
|
||||
{{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }}
|
||||
-dataDir=/data \
|
||||
{{- else if .Values.admin.dataDir }}
|
||||
@@ -195,7 +192,6 @@ spec:
|
||||
{{ $arg }}{{- if lt $index (sub (len $.Values.admin.extraArgs) 1) }} \{{ end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.admin.containerSecurityContext (tpl (.Values.admin.extraVolumeMounts | default "") .) (tpl (.Values.admin.extraVolumes | default "") .)) | nindent 12 }}
|
||||
{{- if or (eq .Values.admin.data.type "hostPath") (eq .Values.admin.data.type "persistentVolumeClaim") (eq .Values.admin.data.type "emptyDir") (eq .Values.admin.data.type "existingClaim") }}
|
||||
- name: admin-data
|
||||
mountPath: /data
|
||||
@@ -236,12 +232,6 @@ spec:
|
||||
name: http
|
||||
- containerPort: {{ .Values.admin.grpcPort }}
|
||||
name: grpc
|
||||
{{- if gt (int .Values.admin.legacyPort) 0 }}
|
||||
- containerPort: {{ .Values.admin.legacyPort }}
|
||||
name: legacy-http
|
||||
- containerPort: {{ add (int .Values.admin.legacyPort) 10000 }}
|
||||
name: legacy-grpc
|
||||
{{- end }}
|
||||
{{- if .Values.admin.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
@@ -277,7 +267,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.admin.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.admin.containerSecurityContext (tpl (.Values.admin.extraVolumeMounts | default "") .) (tpl (.Values.admin.extraVolumes | default "") .) false) | nindent 8 }}
|
||||
{{- if eq .Values.admin.data.type "hostPath" }}
|
||||
- name: admin-data
|
||||
hostPath:
|
||||
|
||||
@@ -43,7 +43,7 @@ spec:
|
||||
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace $existingS3ConfigSecret) | default dict }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
|
||||
{{- else }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
|
||||
{{- end }}
|
||||
{{- $_ := set $podAnnotations "checksum/master-config" (include (print .Template.BasePath "/master/master-configmap.yaml") . | sha256sum) }}
|
||||
annotations:
|
||||
@@ -306,7 +306,6 @@ spec:
|
||||
{{- end }}
|
||||
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.allInOne.containerSecurityContext (tpl (.Values.allInOne.extraVolumeMounts | default "") .) (tpl (.Values.allInOne.extraVolumes | default "") .)) | nindent 12 }}
|
||||
- name: data
|
||||
mountPath: /data
|
||||
{{- if and .Values.allInOne.s3.enabled (or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth) }}
|
||||
@@ -434,7 +433,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.allInOne.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.allInOne.containerSecurityContext (tpl (.Values.allInOne.extraVolumeMounts | default "") .) (tpl (.Values.allInOne.extraVolumes | default "") .) false) | nindent 8 }}
|
||||
{{- include "seaweedfs.licenseVolume" . | nindent 8 }}
|
||||
- name: data
|
||||
{{- if eq .Values.allInOne.data.type "hostPath" }}
|
||||
|
||||
@@ -28,13 +28,6 @@ rules:
|
||||
- "update"
|
||||
- "create"
|
||||
- "delete"
|
||||
- apiGroups: ["objectstorage.k8s.io"]
|
||||
resources:
|
||||
- "bucketclasses"
|
||||
verbs:
|
||||
- "get"
|
||||
- "list"
|
||||
- "watch"
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs:
|
||||
@@ -68,7 +61,7 @@ metadata:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner
|
||||
name: {{ .Values.global.seaweedfs.serviceAccountName }}-objectstorage-provisioner
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
|
||||
@@ -61,7 +61,7 @@ spec:
|
||||
schedulerName: {{ .Values.cosi.schedulerName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner
|
||||
serviceAccountName: {{ include "seaweedfs.componentName" (list . "objectstorage-provisioner") }}
|
||||
{{- if .Values.cosi.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.cosi.initContainers . | nindent 8 | trim }}
|
||||
@@ -113,7 +113,6 @@ spec:
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.cosi.containerSecurityContext (tpl (.Values.cosi.extraVolumeMounts | default "") .) (tpl (.Values.cosi.extraVolumes | default "") .)) | nindent 12 }}
|
||||
- mountPath: /var/lib/cosi
|
||||
name: socket
|
||||
{{- if .Values.cosi.enableAuth }}
|
||||
@@ -149,9 +148,6 @@ spec:
|
||||
resources:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.cosi.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.cosi.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
- name: seaweedfs-cosi-sidecar
|
||||
image: "{{ .Values.cosi.sidecar.image }}"
|
||||
imagePullPolicy: {{ default "IfNotPresent" .Values.global.seaweedfs.imagePullPolicy }}
|
||||
@@ -163,7 +159,6 @@ spec:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.cosi.containerSecurityContext "" "") | nindent 12 }}
|
||||
- mountPath: /var/lib/cosi
|
||||
name: socket
|
||||
{{- with .Values.cosi.sidecar.resources }}
|
||||
@@ -177,7 +172,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.cosi.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.cosi.containerSecurityContext (tpl (.Values.cosi.extraVolumeMounts | default "") .) (tpl (.Values.cosi.extraVolumes | default "") .) true) | nindent 8 }}
|
||||
- name: socket
|
||||
emptyDir: {}
|
||||
{{- if .Values.cosi.enableAuth }}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner
|
||||
name: {{ .Values.global.seaweedfs.serviceAccountName }}-objectstorage-provisioner
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
{{- $filerEnabled := or .Values.filer.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $filerEnabled .Values.filer.httpRoute.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "filer")) .Values.allInOne.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "filer") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- with .Values.filer.httpRoute.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: filer
|
||||
{{- with .Values.filer.httpRoute.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.filer.httpRoute.parentRefs }}
|
||||
parentRefs:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.filer.httpRoute.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.filer.httpRoute.rules | default (list dict) }}
|
||||
- backendRefs:
|
||||
{{- with .backendRefs }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- else }}
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: {{ $serviceName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
port: {{ $.Values.filer.port }}
|
||||
weight: 1
|
||||
{{- end }}
|
||||
{{- with .matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .filters }}
|
||||
filters:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .timeouts }}
|
||||
timeouts:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -48,7 +48,7 @@ spec:
|
||||
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.filer.s3.existingConfigSecret) | default dict }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
|
||||
{{- else }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
@@ -229,7 +229,6 @@ spec:
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.filer.containerSecurityContext (tpl (.Values.filer.extraVolumeMounts | default "") .) (tpl (.Values.filer.extraVolumes | default "") .)) | nindent 12 }}
|
||||
{{- if (or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "persistentVolumeClaim") (eq .Values.filer.logs.type "emptyDir")) }}
|
||||
- name: seaweedfs-filer-log-volume
|
||||
mountPath: "/logs/"
|
||||
@@ -338,7 +337,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.filer.containerSecurityContext (tpl (.Values.filer.extraVolumeMounts | default "") .) (tpl (.Values.filer.extraVolumes | default "") .) false) | nindent 8 }}
|
||||
{{- if eq .Values.filer.logs.type "hostPath" }}
|
||||
- name: seaweedfs-filer-log-volume
|
||||
hostPath:
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
{{- $masterEnabled := or .Values.master.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $masterEnabled .Values.master.httpRoute.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "master")) .Values.allInOne.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "master") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- with .Values.master.httpRoute.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: master
|
||||
{{- with .Values.master.httpRoute.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.master.httpRoute.parentRefs }}
|
||||
parentRefs:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.master.httpRoute.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.master.httpRoute.rules | default (list dict) }}
|
||||
- backendRefs:
|
||||
{{- with .backendRefs }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- else }}
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: {{ $serviceName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
port: {{ $.Values.master.port }}
|
||||
weight: 1
|
||||
{{- end }}
|
||||
{{- with .matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .filters }}
|
||||
filters:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .timeouts }}
|
||||
timeouts:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -179,7 +179,6 @@ spec:
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.master.containerSecurityContext (tpl (.Values.master.extraVolumeMounts | default "") .) (tpl (.Values.master.extraVolumes | default "") .)) | nindent 12 }}
|
||||
- name : data-{{ .Release.Namespace }}
|
||||
mountPath: /data
|
||||
{{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "persistentVolumeClaim") (eq .Values.master.logs.type "emptyDir") }}
|
||||
@@ -259,7 +258,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.master.containerSecurityContext (tpl (.Values.master.extraVolumeMounts | default "") .) (tpl (.Values.master.extraVolumes | default "") .) false) | nindent 8 }}
|
||||
{{- include "seaweedfs.licenseVolume" . | nindent 8 }}
|
||||
{{- if eq .Values.master.logs.type "hostPath" }}
|
||||
- name: seaweedfs-master-log-volume
|
||||
|
||||
@@ -17,10 +17,6 @@ metadata:
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.s3.replicas }}
|
||||
{{- with .Values.s3.updateStrategy }}
|
||||
strategy:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
@@ -44,7 +40,7 @@ spec:
|
||||
{{- $configSecret := (lookup "v1" "Secret" .Release.Namespace .Values.s3.existingConfigSecret) | default dict }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" ($configSecret | toYaml | sha256sum) }}
|
||||
{{- else }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include "seaweedfs.s3.configChecksum" .) }}
|
||||
{{- $_ := set $podAnnotations "checksum/s3config" (include (print .Template.BasePath "/s3/s3-secret.yaml") . | sha256sum) }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
@@ -63,7 +59,7 @@ spec:
|
||||
{{ tpl .Values.s3.tolerations . | nindent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
terminationGracePeriodSeconds: {{ .Values.s3.terminationGracePeriodSeconds }}
|
||||
terminationGracePeriodSeconds: 10
|
||||
{{- if .Values.s3.priorityClassName }}
|
||||
priorityClassName: {{ .Values.s3.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
@@ -161,7 +157,6 @@ spec:
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.s3.containerSecurityContext (tpl (.Values.s3.extraVolumeMounts | default "") .) (tpl (.Values.s3.extraVolumes | default "") .)) | nindent 12 }}
|
||||
{{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
|
||||
- name: logs
|
||||
mountPath: "/logs/"
|
||||
@@ -239,10 +234,6 @@ spec:
|
||||
failureThreshold: {{ .Values.s3.livenessProbe.failureThreshold }}
|
||||
timeoutSeconds: {{ .Values.s3.livenessProbe.timeoutSeconds }}
|
||||
{{- end }}
|
||||
{{- with .Values.s3.lifecycle }}
|
||||
lifecycle:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.s3.resources }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
@@ -254,7 +245,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.s3.containerSecurityContext (tpl (.Values.s3.extraVolumeMounts | default "") .) (tpl (.Values.s3.extraVolumes | default "") .) false) | nindent 8 }}
|
||||
{{- if .Values.s3.enableAuth }}
|
||||
- name: config-users
|
||||
secret:
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
{{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }}
|
||||
{{- if and $s3Enabled .Values.s3.httpRoute.enabled }}
|
||||
{{- $serviceName := include "seaweedfs.componentName" (list . "s3") }}
|
||||
{{- $s3Port := .Values.filer.s3.port }}
|
||||
{{- if and .Values.allInOne.enabled .Values.allInOne.s3.enabled }}
|
||||
{{- $serviceName = include "seaweedfs.componentName" (list . "all-in-one") }}
|
||||
{{- $s3Port = .Values.allInOne.s3.port | default .Values.s3.port }}
|
||||
{{- else if .Values.s3.enabled }}
|
||||
{{- $s3Port = .Values.s3.port }}
|
||||
{{- end }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "s3") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- with .Values.s3.httpRoute.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
{{- with .Values.s3.httpRoute.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.s3.httpRoute.parentRefs }}
|
||||
parentRefs:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.s3.httpRoute.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.s3.httpRoute.rules | default (list dict) }}
|
||||
- backendRefs:
|
||||
{{- with .backendRefs }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- else }}
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: {{ $serviceName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
port: {{ $s3Port }}
|
||||
weight: 1
|
||||
{{- end }}
|
||||
{{- with .matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .filters }}
|
||||
filters:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .timeouts }}
|
||||
timeouts:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -170,7 +170,6 @@ spec:
|
||||
-userStoreFile=/etc/sw/seaweedfs_sftp_config \
|
||||
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ .Release.Namespace }}:{{ .Values.filer.port }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.sftp.containerSecurityContext (tpl (.Values.sftp.extraVolumeMounts | default "") .) (tpl (.Values.sftp.extraVolumes | default "") .)) | nindent 12 }}
|
||||
{{- if or (eq .Values.sftp.logs.type "hostPath") (eq .Values.sftp.logs.type "emptyDir") }}
|
||||
- name: logs
|
||||
mountPath: "/logs/"
|
||||
@@ -250,7 +249,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.sftp.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.sftp.containerSecurityContext (tpl (.Values.sftp.extraVolumeMounts | default "") .) (tpl (.Values.sftp.extraVolumes | default "") .) false) | nindent 8 }}
|
||||
{{- if .Values.sftp.enableAuth }}
|
||||
- name: config-users
|
||||
secret:
|
||||
|
||||
@@ -76,43 +76,6 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Writable temporary directory for containers using a read-only root filesystem.
|
||||
Input: list of the root context, the component container security context, the
|
||||
rendered extraVolumeMounts and extraVolumes, and whether the pod has secondary
|
||||
chart-managed containers that mount seaweedfs-tmp. A user-supplied /tmp mount
|
||||
only covers the main container, so the volume is still emitted for secondaries;
|
||||
a user-supplied seaweedfs-tmp volume is reused rather than duplicated.
|
||||
*/}}
|
||||
{{- define "seaweedfs.tmpDirCovered" -}}
|
||||
{{- regexMatch `(?m)^\s*-?\s*mountPath:\s*['"]?/tmp/?['"]?\s*(#.*)?$` (index . 2) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "seaweedfs.tmpDirVolume" -}}
|
||||
{{- $root := index . 0 -}}
|
||||
{{- $securityContext := index . 1 -}}
|
||||
{{- if and $securityContext.enabled $securityContext.readOnlyRootFilesystem
|
||||
(or (index . 4) (ne (include "seaweedfs.tmpDirCovered" .) "true"))
|
||||
(not (regexMatch `(?m)^\s*-?\s*name:\s*['"]?seaweedfs-tmp['"]?\s*(#.*)?$` (index . 3))) }}
|
||||
- name: seaweedfs-tmp
|
||||
{{- with $root.Values.global.seaweedfs.tmpDir.sizeLimit }}
|
||||
emptyDir:
|
||||
sizeLimit: {{ . | quote }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "seaweedfs.tmpDirVolumeMount" -}}
|
||||
{{- $securityContext := index . 1 -}}
|
||||
{{- if and $securityContext.enabled $securityContext.readOnlyRootFilesystem
|
||||
(ne (include "seaweedfs.tmpDirCovered" .) "true") }}
|
||||
- name: seaweedfs-tmp
|
||||
mountPath: /tmp
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Whether the mysql filer store is selected; a flag the chart cannot read counts as selected. */}}
|
||||
{{- define "seaweedfs.filer.mysqlEnabled" -}}
|
||||
{{- $merged := dict -}}
|
||||
@@ -444,46 +407,6 @@ true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Read a JWT key from the chart's existing security.toml without fromToml
|
||||
(which requires Helm >=3.17). Args: (list "<section>" $raw).
|
||||
Return the single-line TOML string token, including its quotes, so escapes
|
||||
and explicitly empty values survive unchanged. No output means absent.
|
||||
Accept bare or simply quoted dotted header segments. Fail on other quoted
|
||||
headers rather than risk rotating a key hidden by unsupported syntax.
|
||||
This reads the chart's section/key layout, not arbitrary TOML syntax. */}}
|
||||
{{- define "seaweedfs.existingTomlKey" -}}
|
||||
{{- $section := index . 0 -}}
|
||||
{{- $raw := index . 1 -}}
|
||||
{{- $parts := list -}}
|
||||
{{- range $part := splitList "." $section -}}
|
||||
{{- $escaped := regexQuoteMeta $part -}}
|
||||
{{- $parts = append $parts (printf `(?:%s|"%s"|'%s')` $escaped $escaped $escaped) -}}
|
||||
{{- end -}}
|
||||
{{- $header := printf `^\[[ \t]*%s[ \t]*\][ \t]*(#.*)?$` (join `[ \t]*\.[ \t]*` $parts) -}}
|
||||
{{- $segment := `(?:[A-Za-z0-9_-]+|"[^"\\]*"|'[^']*')` -}}
|
||||
{{- $simpleHeader := printf `^\[[ \t]*%s(?:[ \t]*\.[ \t]*%s)*[ \t]*\][ \t]*(#.*)?$` $segment $segment -}}
|
||||
{{- $assignment := `^(key|"key"|'key')[ \t]*=[ \t]*` -}}
|
||||
{{- $string := `"([^"\\]|\\.)*"|'[^']*'` -}}
|
||||
{{- $active := false -}}
|
||||
{{- $key := "" -}}
|
||||
{{- range $rawLine := splitList "\n" $raw -}}
|
||||
{{- $line := trim $rawLine -}}
|
||||
{{- if hasPrefix "[" $line -}}
|
||||
{{- if and (regexMatch `^\[[^]]*["']` $line) (not (regexMatch $simpleHeader $line)) (eq $key "") -}}
|
||||
{{- fail (printf "security.toml has an unsupported quoted section header; refusing to replace [%s].key" $section) -}}
|
||||
{{- end -}}
|
||||
{{- $active = regexMatch $header $line -}}
|
||||
{{- else if and $active (eq $key "") (regexMatch $assignment $line) -}}
|
||||
{{- $value := regexReplaceAll $assignment $line "" -}}
|
||||
{{- if not (regexMatch (printf `^(%s)[ \t]*(#.*)?$` $string) $value) -}}
|
||||
{{- fail (printf "security.toml [%s].key must be a single-line quoted TOML string; refusing to replace an existing key" $section) -}}
|
||||
{{- end -}}
|
||||
{{- $key = regexFind $string $value -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- $key -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* True when the post-install bucket hook Job renders: an S3 endpoint, plus
|
||||
buckets to create on it. Read by the Job itself and by its NetworkPolicy,
|
||||
which has to appear exactly when the Job does - a Job without its policy
|
||||
@@ -544,9 +467,8 @@ true
|
||||
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
|
||||
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
|
||||
{{- if $currentPVC }}
|
||||
{{- /* include returns a string such as "6.442450944e+10"; convert back to a number, or gt compares lexically */}}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage | float64 }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize | float64 }}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
|
||||
{{- if gt $newSize $oldSize }}
|
||||
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
|
||||
{{- end }}
|
||||
@@ -692,28 +614,3 @@ Usage: {{ include "seaweedfs.service.loadBalancerFields" .Values.s3.service }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Checksum that restarts the S3-serving pods when the generated S3 config changes. It hashes the
|
||||
values s3-secret.yaml is built from, not the rendered Secret: that template keeps its generated keys
|
||||
with lookup, so rendering it again for a checksum draws new random keys whenever lookup finds
|
||||
nothing (on install), and the first upgrade then changed the checksum and restarted the pods
|
||||
although nothing had changed. Only the values that template reads are hashed, so scaling the S3
|
||||
gateway or changing its resources does not restart the filer. Keep this list in step with
|
||||
s3-secret.yaml.
|
||||
Keys the chart generated are not part of it. Deleting the Secret to rotate them regenerates them on
|
||||
the next upgrade without restarting the pods; restart them afterwards (kubectl rollout restart).
|
||||
Folding the stored keys in would bring the first-upgrade restart back, since lookup finds nothing
|
||||
on install.
|
||||
*/}}
|
||||
{{- define "seaweedfs.s3.configChecksum" -}}
|
||||
{{- dict
|
||||
"secretName" (include "seaweedfs.fullname" .)
|
||||
"enabled" (list .Values.s3.enabled .Values.filer.s3.enabled .Values.allInOne.enabled .Values.allInOne.s3.enabled)
|
||||
"enableAuth" (list .Values.s3.enableAuth .Values.filer.s3.enableAuth .Values.allInOne.s3.enableAuth)
|
||||
"existingConfigSecret" (list .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret .Values.allInOne.s3.existingConfigSecret)
|
||||
"credentials" .Values.s3.credentials
|
||||
"legacySecret" (list .Values.s3.reuseLegacySecret .Values.s3.legacySecretName)
|
||||
"auditLogConfig" (list .Values.s3.auditLogConfig .Values.filer.s3.auditLogConfig)
|
||||
| toJson | sha256sum -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -93,11 +93,7 @@
|
||||
ClusterRole under global.seaweedfs.createClusterRole is a leftover from a
|
||||
migration and is not exercised by any component. */}}
|
||||
{{- if .Values.admin.enabled }}
|
||||
{{- $ports := list .Values.admin.port .Values.admin.grpcPort }}
|
||||
{{- if gt (int .Values.admin.legacyPort) 0 }}
|
||||
{{- $ports = concat $ports (list .Values.admin.legacyPort (add (int .Values.admin.legacyPort) 10000)) }}
|
||||
{{- end }}
|
||||
{{- $targets = append $targets (dict "component" "admin" "ports" $ports) }}
|
||||
{{- $targets = append $targets (dict "component" "admin" "ports" (list .Values.admin.port .Values.admin.grpcPort)) }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.enabled }}
|
||||
{{- $ports := list }}
|
||||
|
||||
@@ -16,20 +16,6 @@
|
||||
{{- $clusterUpper := upper $clusterAlias }}
|
||||
{{- $clusterMasterKey := printf "WEED_CLUSTER_%s_MASTER" $clusterUpper }}
|
||||
{{- $clusterFilerKey := printf "WEED_CLUSTER_%s_FILER" $clusterUpper }}
|
||||
{{- $podSecurityContext := .Values.filer.podSecurityContext }}
|
||||
{{- $containerSecurityContext := .Values.filer.containerSecurityContext }}
|
||||
{{- $nodeSelector := .Values.filer.nodeSelector }}
|
||||
{{- $tolerations := .Values.filer.tolerations }}
|
||||
{{- $createBucketsHook := .Values.filer.s3.createBucketsHook }}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
{{- $podSecurityContext = .Values.allInOne.podSecurityContext }}
|
||||
{{- $containerSecurityContext = .Values.allInOne.containerSecurityContext }}
|
||||
{{- $nodeSelector = .Values.allInOne.nodeSelector }}
|
||||
{{- $tolerations = .Values.allInOne.tolerations }}
|
||||
{{- $createBucketsHook = .Values.allInOne.s3.createBucketsHook }}
|
||||
{{- else if .Values.s3.createBuckets }}
|
||||
{{- $createBucketsHook = .Values.s3.createBucketsHook }}
|
||||
{{- end }}
|
||||
|
||||
{{- /* Check allInOne mode first */}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
@@ -84,17 +70,9 @@ spec:
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
app.kubernetes.io/component: bucket-hook
|
||||
spec:
|
||||
{{- with $createBucketsHook.affinity }}
|
||||
affinity:
|
||||
{{ tpl . $ | indent 8 | trim }}
|
||||
{{- end }}
|
||||
restartPolicy: Never
|
||||
{{- if $podSecurityContext.enabled }}
|
||||
securityContext: {{- omit $podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with coalesce $createBucketsHook.tolerations $tolerations }}
|
||||
tolerations:
|
||||
{{ tpl . $ | indent 8 | trim }}
|
||||
{{- if .Values.filer.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" $ | nindent 6 }}
|
||||
containers:
|
||||
@@ -224,9 +202,8 @@ spec:
|
||||
/usr/bin/weed shell
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if or (and $containerSecurityContext.enabled $containerSecurityContext.readOnlyRootFilesystem) $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
|
||||
{{- if or $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . $containerSecurityContext "" "") | nindent 10 }}
|
||||
{{- if $enableAuth }}
|
||||
- name: config-users
|
||||
mountPath: /etc/sw
|
||||
@@ -238,14 +215,6 @@ spec:
|
||||
mountPath: /etc/seaweedfs/security.toml
|
||||
subPath: security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.master.port }}
|
||||
@@ -260,12 +229,11 @@ spec:
|
||||
resources:
|
||||
{{- toYaml . | nindent 10 }}
|
||||
{{- end }}
|
||||
{{- if $containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit $containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- if .Values.filer.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if or (and $containerSecurityContext.enabled $containerSecurityContext.readOnlyRootFilesystem) $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
|
||||
{{- if or $enableAuth (include "seaweedfs.securityConfigEnabled" .) }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . $containerSecurityContext "" "" false) | nindent 8 }}
|
||||
{{- if $enableAuth }}
|
||||
- name: config-users
|
||||
secret:
|
||||
@@ -281,17 +249,5 @@ spec:
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-ca-cert
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with coalesce $createBucketsHook.nodeSelector $nodeSelector }}
|
||||
nodeSelector:
|
||||
{{ tpl . $ | indent 8 | trim }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -18,7 +18,7 @@ data:
|
||||
{{- $legacyName := printf "%s-%s" (include "seaweedfs.name" .) "security-config" }}
|
||||
{{- $existing = lookup "v1" "ConfigMap" .Release.Namespace $legacyName }}
|
||||
{{- end }}
|
||||
{{- $existingToml := dig "data" "security.toml" "" $existing }}
|
||||
{{- $securityConfig := fromToml (dig "data" "security.toml" "" $existing) }}
|
||||
{{- $securityConfigValues := .Values.global.seaweedfs.securityConfig | default dict }}
|
||||
{{- $jwtSigning := $securityConfigValues.jwtSigning | default dict }}
|
||||
{{- $expiresAfterSeconds := $jwtSigning.expiresAfterSeconds | default dict }}
|
||||
@@ -29,7 +29,7 @@ data:
|
||||
# the jwt signing key is read by master and volume server
|
||||
# the jwt defaults to expire after 10 seconds
|
||||
[jwt.signing]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
key = "{{ dig "jwt" "signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.volumeWrite) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.volumeWrite }}
|
||||
{{- end }}
|
||||
@@ -41,7 +41,7 @@ data:
|
||||
# - the Volume server validates the JWT on reading
|
||||
# the jwt defaults to expire after 60 seconds
|
||||
[jwt.signing.read]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
key = "{{ dig "jwt" "signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.volumeRead) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.volumeRead }}
|
||||
{{- end }}
|
||||
@@ -53,7 +53,7 @@ data:
|
||||
# - the Filer server validates the JWT on writing
|
||||
# the jwt defaults to expire after 10 seconds
|
||||
[jwt.filer_signing]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
key = "{{ dig "jwt" "filer_signing" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.filerWrite) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.filerWrite }}
|
||||
{{- end }}
|
||||
@@ -65,7 +65,7 @@ data:
|
||||
# - the Filer server validates the JWT on reading
|
||||
# the jwt defaults to expire after 60 seconds
|
||||
[jwt.filer_signing.read]
|
||||
key = {{ include "seaweedfs.existingTomlKey" (list "jwt.filer_signing.read" $existingToml) | default (randAlphaNum 10 | b64enc | quote) }}
|
||||
key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
{{- if gt (int $expiresAfterSeconds.filerRead) 0 }}
|
||||
expires_after_seconds = {{ int $expiresAfterSeconds.filerRead }}
|
||||
{{- end }}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $volumeEnabled .Values.volume.httpRoute.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }}
|
||||
apiVersion: gateway.networking.k8s.io/v1
|
||||
kind: HTTPRoute
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "volume") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- with .Values.volume.httpRoute.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
{{- with .Values.volume.httpRoute.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.volume.httpRoute.parentRefs }}
|
||||
parentRefs:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.volume.httpRoute.hostnames }}
|
||||
hostnames:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.volume.httpRoute.rules | default (list dict) }}
|
||||
- backendRefs:
|
||||
{{- with .backendRefs }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- else }}
|
||||
- group: ""
|
||||
kind: Service
|
||||
name: {{ $serviceName }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
port: {{ $.Values.volume.port }}
|
||||
weight: 1
|
||||
{{- end }}
|
||||
{{- with .matches }}
|
||||
matches:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .filters }}
|
||||
filters:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .timeouts }}
|
||||
timeouts:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -30,46 +30,15 @@ spec:
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
app.kubernetes.io/component: volume-resize-hook
|
||||
spec:
|
||||
{{- with .Values.volume.resizeHook.affinity }}
|
||||
affinity:
|
||||
{{ tpl (printf "{{ $volumeName := \"volume\" }}%s" .) $ | indent 8 | trim }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ $seaweedfsName }}-volume-resize-hook
|
||||
restartPolicy: Never
|
||||
{{- if .Values.volume.podSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with coalesce .Values.volume.resizeHook.tolerations .Values.volume.tolerations }}
|
||||
tolerations:
|
||||
{{ tpl (printf "{{ $volumeName := \"volume\" }}%s" .) $ | indent 8 | trim }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: resize
|
||||
image: {{ .Values.volume.resizeHook.image }}
|
||||
{{- if and .Values.volume.containerSecurityContext.enabled .Values.volume.containerSecurityContext.readOnlyRootFilesystem }}
|
||||
env:
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
{{- end }}
|
||||
command: ["sh", "-xec"]
|
||||
args:
|
||||
- |
|
||||
{{ $commands | indent 14 }}
|
||||
{{- if and .Values.volume.containerSecurityContext.enabled .Values.volume.containerSecurityContext.readOnlyRootFilesystem }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.volume.containerSecurityContext "" "") | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.volume.containerSecurityContext.enabled }}
|
||||
securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if and .Values.volume.containerSecurityContext.enabled .Values.volume.containerSecurityContext.readOnlyRootFilesystem }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.volume.containerSecurityContext "" "" false) | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with coalesce .Values.volume.resizeHook.nodeSelector .Values.volume.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{ tpl (printf "{{ $volumeName := \"volume\" }}%s" .) $ | indent 8 | trim }}
|
||||
{{- end }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
|
||||
@@ -104,7 +104,6 @@ spec:
|
||||
fi
|
||||
done
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list $ $volume.containerSecurityContext "" "") | nindent 12 }}
|
||||
- name: idx
|
||||
mountPath: /idx
|
||||
{{- range $dir := $volume.dataDirs }}
|
||||
@@ -225,7 +224,6 @@ spec:
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list $ $volume.containerSecurityContext (tpl (printf "{{ $volumeName := \"%s\" }}%s" $volumeName ($volume.extraVolumeMounts | default "")) $) (tpl ($volume.extraVolumes | default "") $)) | nindent 12 }}
|
||||
{{- range $dir := $volume.dataDirs }}
|
||||
{{- if not ( eq $dir.type "custom" ) }}
|
||||
- name: {{ $dir.name }}
|
||||
@@ -308,7 +306,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" (printf "{{ $volumeName := \"%s\" }}%s" $volumeName $volume.sidecars) "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list $ $volume.containerSecurityContext (tpl (printf "{{ $volumeName := \"%s\" }}%s" $volumeName ($volume.extraVolumeMounts | default "")) $) (tpl ($volume.extraVolumes | default "") $) (and $initContainers_exists $volume.idx)) | nindent 8 }}
|
||||
|
||||
{{- range $dir := $volume.dataDirs }}
|
||||
|
||||
|
||||
@@ -144,7 +144,6 @@ spec:
|
||||
{{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.worker.containerSecurityContext (tpl (.Values.worker.extraVolumeMounts | default "") .) (tpl (.Values.worker.extraVolumes | default "") .)) | nindent 12 }}
|
||||
{{- if or (eq .Values.worker.data.type "hostPath") (eq .Values.worker.data.type "emptyDir") (eq .Values.worker.data.type "existingClaim") }}
|
||||
- name: worker-data
|
||||
mountPath: {{ .Values.worker.workingDir }}
|
||||
@@ -263,16 +262,15 @@ spec:
|
||||
--metrics-ip=0.0.0.0 \
|
||||
{{- end }}
|
||||
--max-concurrency={{ .Values.worker.maxExecute }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
volumeMounts:
|
||||
{{- include "seaweedfs.tmpDirVolumeMount" (list . .Values.worker.containerSecurityContext "" "") | nindent 12 }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
- name: ca-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/ca/
|
||||
- name: worker-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/worker/
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.worker.lanceMetricsPort }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.worker.lanceMetricsPort }}
|
||||
@@ -304,7 +302,6 @@ spec:
|
||||
{{- include "seaweedfs.tplvalues.render" (dict "value" .Values.worker.sidecars "context" $) | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- include "seaweedfs.tmpDirVolume" (list . .Values.worker.containerSecurityContext (tpl (.Values.worker.extraVolumeMounts | default "") .) (tpl (.Values.worker.extraVolumes | default "") .) (ne (include "seaweedfs.worker.lanceNamespaceUrl" .) "")) | nindent 8 }}
|
||||
{{- if eq .Values.worker.data.type "hostPath" }}
|
||||
- name: worker-data
|
||||
hostPath:
|
||||
|
||||
@@ -25,9 +25,6 @@ global:
|
||||
imagePullPolicy: IfNotPresent
|
||||
restartPolicy: Always
|
||||
loggingLevel: 1
|
||||
# Writable temporary storage used when containers run with a read-only root filesystem.
|
||||
tmpDir:
|
||||
sizeLimit: ""
|
||||
enableSecurity: false
|
||||
masterServer: null
|
||||
# filerWrite: true mounts security.toml on filer + admin without needing
|
||||
@@ -263,8 +260,6 @@ master:
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
@@ -273,16 +268,7 @@ master:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
ingress:
|
||||
@@ -311,24 +297,6 @@ master:
|
||||
# sub_filter_once off;
|
||||
tls: []
|
||||
|
||||
# Gateway API HTTPRoute, an alternative to the ingress above.
|
||||
httpRoute:
|
||||
enabled: false
|
||||
labels: {}
|
||||
annotations: {}
|
||||
# Gateways this route attaches to, for example:
|
||||
# - group: gateway.networking.k8s.io
|
||||
# kind: Gateway
|
||||
# name: my-gateway
|
||||
# namespace: gateway-system
|
||||
# sectionName: https
|
||||
parentRefs: []
|
||||
hostnames: []
|
||||
# Each rule may set matches, filters, timeouts and backendRefs.
|
||||
# A rule without backendRefs routes to the master service.
|
||||
# An empty list routes all traffic there.
|
||||
rules: []
|
||||
|
||||
extraEnvironmentVars:
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_1: "7"
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_2: "6"
|
||||
@@ -437,10 +405,6 @@ volume:
|
||||
resizeHook:
|
||||
enabled: true
|
||||
image: alpine/k8s:1.28.4
|
||||
# nodeSelector and tolerations fall back to the volume server's when unset
|
||||
nodeSelector: ""
|
||||
tolerations: ""
|
||||
affinity: ""
|
||||
|
||||
# idx (the volume index) may use a separate PERSISTENT volume:
|
||||
#
|
||||
@@ -596,8 +560,6 @@ volume:
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
@@ -606,16 +568,7 @@ volume:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
# used to configure livenessProbe on volume-server containers
|
||||
@@ -669,24 +622,6 @@ volume:
|
||||
# sub_filter '/seaweedfsstatic' './seaweedfsstatic';
|
||||
# sub_filter_once off;
|
||||
|
||||
# Gateway API HTTPRoute, an alternative to the ingress above.
|
||||
httpRoute:
|
||||
enabled: false
|
||||
labels: {}
|
||||
annotations: {}
|
||||
# Gateways this route attaches to, for example:
|
||||
# - group: gateway.networking.k8s.io
|
||||
# kind: Gateway
|
||||
# name: my-gateway
|
||||
# namespace: gateway-system
|
||||
# sectionName: https
|
||||
parentRefs: []
|
||||
hostnames: []
|
||||
# Each rule may set matches, filters, timeouts and backendRefs.
|
||||
# A rule without backendRefs routes to the volume server service.
|
||||
# An empty list routes all traffic there.
|
||||
rules: []
|
||||
|
||||
# Map of named volume groups for topology-aware deployments.
|
||||
# Each key inherits all fields from the `volume` section but can override
|
||||
# them locally—for example, replicas, nodeSelector, dataCenter, etc.
|
||||
@@ -910,8 +845,6 @@ filer:
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
@@ -920,16 +853,7 @@ filer:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
ingresses:
|
||||
@@ -979,24 +903,6 @@ filer:
|
||||
# requires ingress-nginx to run with --enable-ssl-passthrough.
|
||||
tls: []
|
||||
|
||||
# Gateway API HTTPRoute, an alternative to the ingress above.
|
||||
httpRoute:
|
||||
enabled: false
|
||||
labels: {}
|
||||
annotations: {}
|
||||
# Gateways this route attaches to, for example:
|
||||
# - group: gateway.networking.k8s.io
|
||||
# kind: Gateway
|
||||
# name: my-gateway
|
||||
# namespace: gateway-system
|
||||
# sectionName: https
|
||||
parentRefs: []
|
||||
hostnames: []
|
||||
# Each rule may set matches, filters, timeouts and backendRefs.
|
||||
# A rule without backendRefs routes to the filer service.
|
||||
# An empty list routes all traffic there.
|
||||
rules: []
|
||||
|
||||
# extraEnvVars is a list of extra environment variables to set with the stateful set.
|
||||
extraEnvironmentVars:
|
||||
# the WEED_MYSQL_* keys and the db credential secret only render while this is "true"
|
||||
@@ -1087,27 +993,12 @@ filer:
|
||||
# anonymousRead: false
|
||||
createBucketsHook:
|
||||
resources: {}
|
||||
# nodeSelector and tolerations fall back to filer's when unset
|
||||
nodeSelector: ""
|
||||
tolerations: ""
|
||||
affinity: ""
|
||||
|
||||
s3:
|
||||
enabled: false
|
||||
imageOverride: null
|
||||
restartPolicy: null
|
||||
replicas: 1
|
||||
# Deployment update strategy, rendered as the Deployment's strategy
|
||||
# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
|
||||
# Example:
|
||||
# updateStrategy:
|
||||
# type: RollingUpdate
|
||||
# rollingUpdate:
|
||||
# maxUnavailable: 0
|
||||
# maxSurge: 25%
|
||||
updateStrategy: {}
|
||||
# Time the s3 pod is given to shut down, including any preStop hook
|
||||
terminationGracePeriodSeconds: 10
|
||||
bindAddress: 0.0.0.0
|
||||
port: 8333
|
||||
# add additional https port
|
||||
@@ -1236,8 +1127,6 @@ s3:
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
@@ -1246,16 +1135,7 @@ s3:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
# You can also use emptyDir storage:
|
||||
@@ -1300,23 +1180,8 @@ s3:
|
||||
failureThreshold: 100
|
||||
timeoutSeconds: 10
|
||||
|
||||
# Lifecycle hooks for the s3 container. A preStop sleep lets the pod leave
|
||||
# the Service endpoints before it receives SIGTERM; keep it shorter than
|
||||
# terminationGracePeriodSeconds.
|
||||
# ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
|
||||
# Example:
|
||||
# lifecycle:
|
||||
# preStop:
|
||||
# exec:
|
||||
# command: ["sleep", "5"]
|
||||
lifecycle: {}
|
||||
|
||||
createBucketsHook:
|
||||
resources: {}
|
||||
# nodeSelector and tolerations fall back to filer's when unset
|
||||
nodeSelector: ""
|
||||
tolerations: ""
|
||||
affinity: ""
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
@@ -1330,24 +1195,6 @@ s3:
|
||||
annotations: {}
|
||||
tls: []
|
||||
|
||||
# Gateway API HTTPRoute, an alternative to the ingress above.
|
||||
httpRoute:
|
||||
enabled: false
|
||||
labels: {}
|
||||
annotations: {}
|
||||
# Gateways this route attaches to, for example:
|
||||
# - group: gateway.networking.k8s.io
|
||||
# kind: Gateway
|
||||
# name: my-gateway
|
||||
# namespace: gateway-system
|
||||
# sectionName: https
|
||||
parentRefs: []
|
||||
hostnames: []
|
||||
# Each rule may set matches, filters, timeouts and backendRefs.
|
||||
# A rule without backendRefs routes to the s3 service.
|
||||
# An empty list routes all traffic there.
|
||||
rules: []
|
||||
|
||||
# Service settings
|
||||
service:
|
||||
type: ClusterIP
|
||||
@@ -1440,33 +1287,7 @@ sftp:
|
||||
priorityClassName: ""
|
||||
schedulerName: ""
|
||||
serviceAccountName: ""
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
logs:
|
||||
@@ -1511,14 +1332,8 @@ admin:
|
||||
imageOverride: null
|
||||
restartPolicy: null
|
||||
replicas: 1
|
||||
port: 12535 # Default admin port
|
||||
grpcPort: 22535 # Default gRPC port for worker connections
|
||||
# Legacy compat port: admin also serves HTTP and worker gRPC (legacyPort +
|
||||
# 10000) on this port so workers and clients from before the 23646 -> 12535
|
||||
# default change keep connecting during migration. 0 disables. The flag is
|
||||
# only passed to the binary when the value differs from its 23646 default,
|
||||
# so the chart keeps installing on releases that predate -port.legacy.
|
||||
legacyPort: 23646
|
||||
port: 23646 # Default admin port
|
||||
grpcPort: 33646 # Default gRPC port for worker connections
|
||||
# IP address the admin server listens on. Since `weed admin` 4.46 defaults to
|
||||
# loopback (127.0.0.1), the chart must bind a non-loopback address for the
|
||||
# kubelet's httpGet readiness/liveness probes (which dial the pod IP) to ever
|
||||
@@ -1619,33 +1434,7 @@ admin:
|
||||
priorityClassName: ""
|
||||
schedulerName: ""
|
||||
serviceAccountName: ""
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
extraEnvironmentVars: {}
|
||||
@@ -1691,24 +1480,6 @@ admin:
|
||||
annotations: {}
|
||||
tls: []
|
||||
|
||||
# Gateway API HTTPRoute, an alternative to the ingress above.
|
||||
httpRoute:
|
||||
enabled: false
|
||||
labels: {}
|
||||
annotations: {}
|
||||
# Gateways this route attaches to, for example:
|
||||
# - group: gateway.networking.k8s.io
|
||||
# kind: Gateway
|
||||
# name: my-gateway
|
||||
# namespace: gateway-system
|
||||
# sectionName: https
|
||||
parentRefs: []
|
||||
hostnames: []
|
||||
# Each rule may set matches, filters, timeouts and backendRefs.
|
||||
# A rule without backendRefs routes to the admin service.
|
||||
# An empty list routes all traffic there.
|
||||
rules: []
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
annotations: {}
|
||||
@@ -1820,33 +1591,7 @@ worker:
|
||||
priorityClassName: ""
|
||||
schedulerName: ""
|
||||
serviceAccountName: ""
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
extraEnvironmentVars: {}
|
||||
@@ -1945,10 +1690,6 @@ allInOne:
|
||||
# anonymousRead: false
|
||||
createBucketsHook:
|
||||
resources: {}
|
||||
# nodeSelector and tolerations fall back to allInOne's when unset
|
||||
nodeSelector: ""
|
||||
tolerations: ""
|
||||
affinity: ""
|
||||
|
||||
# SFTP server configuration
|
||||
# Note: Most parameters below default to null, which means they inherit from
|
||||
@@ -2110,8 +1851,6 @@ allInOne:
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
|
||||
# Configure security context for Container
|
||||
@@ -2120,16 +1859,7 @@ allInOne:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
# Resource management
|
||||
@@ -2168,33 +1898,7 @@ cosi:
|
||||
# should have a secret key called seaweedfs_s3_config with an inline json configure
|
||||
existingConfigSecret: null
|
||||
|
||||
# Configure security context for Pod
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# podSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 1000
|
||||
# runAsGroup: 3000
|
||||
# fsGroup: 2000
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
podSecurityContext: {}
|
||||
# Configure security context for Container
|
||||
# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
|
||||
# Example:
|
||||
# containerSecurityContext:
|
||||
# enabled: true
|
||||
# runAsUser: 2000
|
||||
# runAsGroup: 3000
|
||||
# runAsNonRoot: true
|
||||
# privileged: false
|
||||
# allowPrivilegeEscalation: false
|
||||
# readOnlyRootFilesystem: true
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# seccompProfile:
|
||||
# type: RuntimeDefault
|
||||
containerSecurityContext: {}
|
||||
|
||||
# used to assign a custom scheduler to cosi pods
|
||||
|
||||
+1005
-1067
File diff suppressed because it is too large
Load Diff
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 53 KiB |
@@ -527,10 +527,6 @@ message AtomicRenameEntryRequest {
|
||||
string new_directory = 3;
|
||||
string new_name = 4;
|
||||
repeated int32 signatures = 5;
|
||||
// When true, the rename fails with ErrEntryAlreadyExists if the target
|
||||
// already exists. Lets a caller that authorized the rename without delete
|
||||
// access on an absent target stay safe if one is created concurrently.
|
||||
bool fail_if_target_exists = 6;
|
||||
}
|
||||
|
||||
message AtomicRenameEntryResponse {
|
||||
|
||||
Generated
+2
-2
@@ -148,9 +148,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
version = "0.23.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
|
||||
Generated
+8
-15
@@ -234,9 +234,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.18.1"
|
||||
version = "1.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
|
||||
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"zeroize",
|
||||
@@ -244,15 +244,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.45.0"
|
||||
version = "0.39.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
|
||||
checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
"dunce",
|
||||
"fs_extra",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2678,12 +2677,6 @@ dependencies = [
|
||||
"spki 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.18.1"
|
||||
@@ -3396,9 +3389,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
version = "0.23.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -3443,9 +3436,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
version = "0.103.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
|
||||
@@ -64,7 +64,6 @@ seaweed-volume --port 8080 --master localhost:9333 --dir /data/vol1 --max 7 \
|
||||
| `--whiteList` | | Comma-separated IPs with write permission |
|
||||
| `--preStopSeconds` | `10` | Graceful drain period before shutdown |
|
||||
| `--compactionMBps` | `0` (unlimited) | Compaction I/O rate limit |
|
||||
| `--compactionMaxParallelCommits` | `0` (unlimited) | Max concurrent vacuum commits |
|
||||
| `--pprof` | `false` | Enable pprof HTTP handlers |
|
||||
|
||||
Set `RUST_LOG=debug` (or `trace`, `info`, `warn`) for log level control.
|
||||
|
||||
@@ -59,8 +59,6 @@ service Seaweed {
|
||||
}
|
||||
rpc RaftLeadershipTransfer (RaftLeadershipTransferRequest) returns (RaftLeadershipTransferResponse) {
|
||||
}
|
||||
rpc ResumeVolumeGrowth (ResumeVolumeGrowthRequest) returns (ResumeVolumeGrowthResponse) {
|
||||
}
|
||||
rpc VolumeGrow (VolumeGrowRequest) returns (VolumeGrowResponse) {
|
||||
}
|
||||
rpc CollectionStatistics (CollectionStatisticsRequest) returns (CollectionStatisticsResponse) {
|
||||
@@ -121,12 +119,6 @@ message Heartbeat {
|
||||
// place of `volumes`. A master that does not understand this never sets
|
||||
// volume_digest_supported, so it keeps being sent the whole list.
|
||||
repeated VolumeInformationMessage changed_volumes = 28;
|
||||
|
||||
// Highest volume id this server has ever observed, persisted locally. Fed
|
||||
// by HeartbeatResponse.max_volume_id echoes and by AllocateVolume requests.
|
||||
uint32 max_volume_id_seen = 29;
|
||||
// Topology id of the master cluster this server last heartbeated to.
|
||||
string last_seen_topology_id = 30;
|
||||
}
|
||||
|
||||
message HeartbeatResponse {
|
||||
@@ -143,12 +135,6 @@ message HeartbeatResponse {
|
||||
// The master compares volume digests, so a server that reports one may send
|
||||
// changed_volumes in place of its whole list.
|
||||
bool volume_digest_supported = 9;
|
||||
// The leader's current allocation bound; servers persist it and report it
|
||||
// back as max_volume_id_seen so the bound survives master state loss.
|
||||
uint32 max_volume_id = 10;
|
||||
// The leader's cluster identity; a different value reported back later
|
||||
// means the master's state was rebuilt.
|
||||
string topology_id = 11;
|
||||
}
|
||||
|
||||
message VolumeInformationMessage {
|
||||
@@ -227,9 +213,6 @@ message KeepConnectedRequest {
|
||||
string filer_group = 5;
|
||||
string data_center = 6;
|
||||
string rack = 7;
|
||||
// A draining filer leaves the lock ring but stays a cluster member, so peers
|
||||
// keep following its metadata until the stream closes.
|
||||
bool leave_lock_ring = 8;
|
||||
}
|
||||
|
||||
message VolumeLocation {
|
||||
@@ -319,18 +302,6 @@ message VolumeGrowRequest {
|
||||
string data_node = 7;
|
||||
uint32 memory_map_max_size_mb = 8;
|
||||
string disk_type = 9;
|
||||
// If > 0, grow only when the average fill ratio of the layout's writable
|
||||
// volumes has reached this threshold (e.g. 0.6). Lets callers pre-grow on
|
||||
// fill level with one call, instead of listing volumes to compute the
|
||||
// ratio themselves. Ignored when there are no writable volumes.
|
||||
float grow_avg_fill_threshold = 10;
|
||||
}
|
||||
|
||||
message ResumeVolumeGrowthRequest {
|
||||
}
|
||||
|
||||
message ResumeVolumeGrowthResponse {
|
||||
bool was_frozen = 1;
|
||||
}
|
||||
|
||||
message AssignResponse {
|
||||
@@ -637,9 +608,4 @@ message RaftLeadershipTransferResponse {
|
||||
}
|
||||
|
||||
message VolumeGrowResponse {
|
||||
// Average fill ratio of the layout's writable volumes at request time
|
||||
// (0 when there are none). Reported whether or not growth was triggered.
|
||||
float avg_fill = 1;
|
||||
// Number of writable volumes at request time.
|
||||
uint32 writable_count = 2;
|
||||
}
|
||||
|
||||
@@ -571,7 +571,6 @@ message ReadVolumeFileStatusResponse {
|
||||
string disk_type = 9;
|
||||
VolumeInfo volume_info = 10;
|
||||
uint32 version = 11;
|
||||
uint64 expire_at_sec = 12; // TTL expiry counted from the last write; 0 when the volume never expires
|
||||
}
|
||||
|
||||
message DiskStatus {
|
||||
|
||||
@@ -125,11 +125,6 @@ pub struct Cli {
|
||||
#[arg(long = "maintenanceMBps", default_value_t = 0)]
|
||||
pub maintenance_mb_per_second: u32,
|
||||
|
||||
/// Limit how many vacuum commits run at once on this server; queued
|
||||
/// commits hold no volume lock. 0 means unlimited.
|
||||
#[arg(long = "compactionMaxParallelCommits", default_value_t = 0)]
|
||||
pub max_parallel_vacuum_commits: usize,
|
||||
|
||||
/// Limit file size to avoid out of memory.
|
||||
#[arg(long = "fileSizeLimitMB", default_value_t = 256)]
|
||||
pub file_size_limit_mb: u32,
|
||||
@@ -268,8 +263,6 @@ pub struct VolumeServerConfig {
|
||||
pub mem_profile: String,
|
||||
pub compaction_byte_per_second: i64,
|
||||
pub maintenance_byte_per_second: i64,
|
||||
/// Max concurrent vacuum commits; 0 means unlimited.
|
||||
pub max_parallel_vacuum_commits: usize,
|
||||
pub file_size_limit_bytes: i64,
|
||||
pub concurrent_upload_limit: i64,
|
||||
pub concurrent_download_limit: i64,
|
||||
@@ -934,7 +927,6 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig {
|
||||
mem_profile: cli.mem_profile,
|
||||
compaction_byte_per_second: cli.compaction_mb_per_second as i64 * 1024 * 1024,
|
||||
maintenance_byte_per_second: cli.maintenance_mb_per_second as i64 * 1024 * 1024,
|
||||
max_parallel_vacuum_commits: cli.max_parallel_vacuum_commits,
|
||||
file_size_limit_bytes: cli.file_size_limit_mb as i64 * 1024 * 1024,
|
||||
concurrent_upload_limit: cli.concurrent_upload_limit_mb as i64 * 1024 * 1024,
|
||||
concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024,
|
||||
|
||||
@@ -340,14 +340,6 @@ async fn run(
|
||||
rack: config.rack.clone(),
|
||||
file_size_limit_bytes: config.file_size_limit_bytes,
|
||||
maintenance_byte_per_second: config.maintenance_byte_per_second,
|
||||
vacuum_commit_slots: if config.max_parallel_vacuum_commits > 0 {
|
||||
Some(tokio::sync::Semaphore::new(
|
||||
config.max_parallel_vacuum_commits,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
},
|
||||
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
|
||||
// Go sets isHeartbeating: true unconditionally at startup
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
|
||||
has_master: !config.masters.is_empty(),
|
||||
@@ -379,9 +371,6 @@ async fn run(
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
},
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail_notify: tokio::sync::Notify::new(),
|
||||
});
|
||||
|
||||
// Load persisted state from disk if it exists (matches Go's State.Load on startup)
|
||||
|
||||
@@ -397,7 +397,6 @@ mod tests {
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
max_parallel_vacuum_commits: 0,
|
||||
file_size_limit_bytes: 0,
|
||||
concurrent_upload_limit: 0,
|
||||
concurrent_download_limit: 0,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+608
-2667
File diff suppressed because it is too large
Load Diff
@@ -474,14 +474,6 @@ async fn do_heartbeat(
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let echo_max_volume_id = hb_resp.max_volume_id;
|
||||
let echo_topology_id = hb_resp.topology_id.clone();
|
||||
off_runtime(config, state, move |_, state| {
|
||||
let store = state.store.read().unwrap();
|
||||
store.update_max_volume_id_seen(echo_max_volume_id);
|
||||
store.update_last_seen_topology_id(&echo_topology_id);
|
||||
})
|
||||
.await?;
|
||||
let options_changed =
|
||||
volume_options_pass(config, state, &hb_resp).await?;
|
||||
if options_changed && maybe_adjust_volume_max(config, state).await? {
|
||||
@@ -974,7 +966,7 @@ fn apply_volume_actions_to(store: &mut Store, actions: VolumeActions) {
|
||||
if let Some(vol) = store
|
||||
.locations
|
||||
.get_mut(disk_id)
|
||||
.and_then(|loc| loc.find_volume(vid))
|
||||
.and_then(|loc| loc.find_volume_mut(vid))
|
||||
&& vol.should_quarantine()
|
||||
{
|
||||
vol.set_no_write_or_delete(true);
|
||||
@@ -1215,10 +1207,10 @@ fn build_heartbeat_with_ec_status(
|
||||
deleted_byte_count: vol.deleted_size(),
|
||||
read_only: vol.is_read_only(),
|
||||
read_only_can_delete: vol.is_no_write_can_delete(),
|
||||
replica_placement: vol.super_block().replica_placement.to_byte() as u32,
|
||||
version: vol.super_block().version.0 as u32,
|
||||
ttl: vol.super_block().ttl.to_u32(),
|
||||
compact_revision: vol.super_block().compaction_revision as u32,
|
||||
replica_placement: vol.super_block.replica_placement.to_byte() as u32,
|
||||
version: vol.super_block.version.0 as u32,
|
||||
ttl: vol.super_block.ttl.to_u32(),
|
||||
compact_revision: vol.super_block.compaction_revision as u32,
|
||||
// The .dat mtime, as Go reports: the shell's quiet-period
|
||||
// gates read this as "last touched", which a delete has to
|
||||
// count towards even though the TTL clock ignores it.
|
||||
@@ -1368,8 +1360,6 @@ fn build_heartbeat_with_ec_status(
|
||||
grpc_port: config.grpc_port as u32,
|
||||
location_uuids,
|
||||
disk_tags,
|
||||
max_volume_id_seen: store.max_volume_id_seen(),
|
||||
last_seen_topology_id: store.last_seen_topology_id(),
|
||||
..Default::default()
|
||||
};
|
||||
(heartbeat, volumes, actions)
|
||||
@@ -1495,8 +1485,6 @@ mod tests {
|
||||
rack: String::new(),
|
||||
file_size_limit_bytes: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
vacuum_commit_slots: None,
|
||||
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(false),
|
||||
has_master: true,
|
||||
pre_stop_seconds: 0,
|
||||
@@ -1520,9 +1508,6 @@ mod tests {
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail_notify: tokio::sync::Notify::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1888,7 +1873,7 @@ mod tests {
|
||||
.store(true, Ordering::Relaxed);
|
||||
|
||||
{
|
||||
let (_, volume) = store.find_volume(VolumeId(17)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(17)).unwrap();
|
||||
volume.set_read_only().unwrap();
|
||||
volume
|
||||
.update_remote_files(|files| files.push(Default::default()))
|
||||
@@ -1987,7 +1972,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
{
|
||||
let (_, volume) = store.find_volume(VolumeId(21)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(21)).unwrap();
|
||||
volume.set_read_only().unwrap();
|
||||
}
|
||||
|
||||
@@ -2165,7 +2150,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
let dat_path = {
|
||||
let (_, volume) = store.find_volume(VolumeId(41)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(41)).unwrap();
|
||||
volume.set_last_io_error_for_test(None);
|
||||
volume.set_last_modified_ts_for_test(
|
||||
SystemTime::now()
|
||||
@@ -2219,7 +2204,7 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let (_, volume) = store.find_volume(VolumeId(51)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(51)).unwrap();
|
||||
volume.set_last_io_error_for_test(Some("input/output error"));
|
||||
|
||||
let heartbeat = build_heartbeat(&test_config(), &mut store);
|
||||
@@ -2229,7 +2214,7 @@ mod tests {
|
||||
// advertised to the master.
|
||||
assert!(heartbeat.volumes.is_empty());
|
||||
assert!(store.has_volume(VolumeId(51)));
|
||||
let (_, volume) = store.find_volume(VolumeId(51)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(51)).unwrap();
|
||||
assert!(volume.is_no_write_or_delete());
|
||||
}
|
||||
|
||||
@@ -2525,7 +2510,7 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let (_, volume) = store.find_volume(VolumeId(id)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(id)).unwrap();
|
||||
volume.set_last_io_error_for_test(None);
|
||||
volume.set_last_modified_ts_for_test(now.saturating_sub(60 * 60));
|
||||
std::fs::OpenOptions::new()
|
||||
@@ -2545,7 +2530,7 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let (_, volume) = store.find_volume(VolumeId(64)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(64)).unwrap();
|
||||
volume.set_last_io_error_for_test(Some("input/output error"));
|
||||
|
||||
let (heartbeat, _, actions) =
|
||||
@@ -2797,7 +2782,7 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let (_, volume) = store.find_volume(VolumeId(71)).unwrap();
|
||||
let (_, volume) = store.find_volume_mut(VolumeId(71)).unwrap();
|
||||
volume
|
||||
.update_remote_files(|files| {
|
||||
files.push(crate::storage::volume::PbRemoteFile {
|
||||
|
||||
@@ -136,7 +136,6 @@ mod tests {
|
||||
mem_profile: String::new(),
|
||||
compaction_byte_per_second: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
max_parallel_vacuum_commits: 0,
|
||||
file_size_limit_bytes: 0,
|
||||
concurrent_upload_limit: 0,
|
||||
concurrent_download_limit: 0,
|
||||
|
||||
@@ -419,24 +419,13 @@ async fn delete_on_ec_shard_holders(
|
||||
|
||||
let mut last_err = None;
|
||||
if local_shards.contains(&shard_id) {
|
||||
// A decode in its publishing tail must not miss this delete. The
|
||||
// tail membership is verified again under the store write lock
|
||||
// inside journal_delete_local — WouldBlock means the decode claimed
|
||||
// it in the gap after this wait — so wait and retry.
|
||||
loop {
|
||||
crate::server::grpc_server::wait_ec_decode_tail(state, target.vid).await;
|
||||
match journal_delete_local(state, target.vid, target.needle_id) {
|
||||
Ok(()) => return Ok(true),
|
||||
Err(e) if e.kind() == io::ErrorKind::WouldBlock => continue,
|
||||
// Nothing was committed — the volume unmounted or remounted
|
||||
// without the needle — so it is safe to fall back to other
|
||||
// shard holders, unlike an RPC failure which may have landed.
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(e) => {
|
||||
last_err = Some(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
match journal_delete_local(state, target.vid, target.needle_id) {
|
||||
Ok(()) => return Ok(true),
|
||||
// Nothing was committed — the volume unmounted or remounted
|
||||
// without the needle — so it is safe to fall back to other
|
||||
// shard holders, unlike an RPC failure which may have landed.
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(e) => last_err = Some(e),
|
||||
}
|
||||
}
|
||||
if let Some(addrs) = addrs {
|
||||
@@ -498,15 +487,6 @@ fn journal_delete_local(
|
||||
needle_id: NeedleId,
|
||||
) -> io::Result<()> {
|
||||
let mut store = state.store.write().unwrap();
|
||||
// Membership is read under the write lock: a decode can claim the
|
||||
// publishing tail while this call waited for the decoder's read lock,
|
||||
// so a check taken earlier would be stale by commit time.
|
||||
if crate::server::grpc_server::ec_decode_tail_contains(state, vid) {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
format!("ec volume {} is in decode publishing tail", vid.0),
|
||||
));
|
||||
}
|
||||
let ecv = store.find_ec_volume_mut(vid).ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
@@ -1773,7 +1753,7 @@ async fn fetch_ec_index_from_one_peer(
|
||||
.await
|
||||
.map_err(|e| io::Error::other(format!("copy .ecx: {}", e)))?
|
||||
.into_inner();
|
||||
drain_copy_stream(stream, ecx_path).await?;
|
||||
drain_copy_stream(stream, ecx_path, false).await?;
|
||||
|
||||
let meta =
|
||||
fs::metadata(ecx_path).map_err(|e| io::Error::other(format!("stat copied .ecx: {}", e)))?;
|
||||
@@ -1786,30 +1766,15 @@ async fn fetch_ec_index_from_one_peer(
|
||||
)));
|
||||
}
|
||||
|
||||
// .ecj is the source peer's deletion journal; .vif carries EC params. Both
|
||||
// are best-effort: a missing .ecj is recreated at mount and a missing .vif
|
||||
// falls back to default EC parameters. The journal is a *set*: merge the
|
||||
// peer's ids into any local ones as a union instead of appending, so
|
||||
// a volume bounced between servers cannot double its journal. The merge
|
||||
// only appends whole records, so a failure leaves nothing to clean up.
|
||||
// .ecj is the source peer's deletion journal (appended); .vif carries EC
|
||||
// params. Both are best-effort: a missing .ecj is recreated at mount and a
|
||||
// missing .vif falls back to default EC parameters. A failed .ecj append
|
||||
// leaves a partial file, so drop it.
|
||||
match client.copy_file(copy_req(".ecj", true)).await {
|
||||
Ok(resp) => {
|
||||
let mut stream = resp.into_inner();
|
||||
let merged = match crate::server::grpc_server::receive_ecj_ids(&mut stream).await {
|
||||
Ok((ids, true)) => crate::server::grpc_server::merge_ecj_ids(
|
||||
state,
|
||||
m.vid,
|
||||
m.data_dir.clone(),
|
||||
ecj_path.to_string(),
|
||||
ids,
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
Ok((_, false)) => Ok(()),
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
if let Err(e) = merged {
|
||||
if let Err(e) = drain_copy_stream(resp.into_inner(), ecj_path, true).await {
|
||||
tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e);
|
||||
let _ = fs::remove_file(ecj_path);
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e),
|
||||
@@ -1817,7 +1782,7 @@ async fn fetch_ec_index_from_one_peer(
|
||||
|
||||
match client.copy_file(copy_req(".vif", true)).await {
|
||||
Ok(resp) => {
|
||||
if let Err(e) = drain_copy_stream(resp.into_inner(), vif_path).await {
|
||||
if let Err(e) = drain_copy_stream(resp.into_inner(), vif_path, false).await {
|
||||
tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .vif: {}", e);
|
||||
}
|
||||
}
|
||||
@@ -1827,14 +1792,22 @@ async fn fetch_ec_index_from_one_peer(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drain a CopyFile stream into a local file, truncating it first.
|
||||
/// Drain a CopyFile stream into a local file, appending or truncating.
|
||||
async fn drain_copy_stream(
|
||||
mut stream: tonic::Streaming<crate::pb::volume_server_pb::CopyFileResponse>,
|
||||
dest_path: &str,
|
||||
append: bool,
|
||||
) -> io::Result<()> {
|
||||
use std::io::Write;
|
||||
let mut file = fs::File::create(dest_path)
|
||||
.map_err(|e| io::Error::other(format!("create {}: {}", dest_path, e)))?;
|
||||
let mut file = if append {
|
||||
fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(dest_path)
|
||||
} else {
|
||||
fs::File::create(dest_path)
|
||||
}
|
||||
.map_err(|e| io::Error::other(format!("create {}: {}", dest_path, e)))?;
|
||||
while let Some(chunk) = stream
|
||||
.message()
|
||||
.await
|
||||
|
||||
@@ -400,7 +400,7 @@ fn collect_ui_data(
|
||||
file_count: volume.file_count(),
|
||||
delete_count: volume.deleted_count(),
|
||||
deleted_byte_count: volume.deleted_size(),
|
||||
ttl: volume.super_block().ttl.to_string(),
|
||||
ttl: volume.super_block.ttl.to_string(),
|
||||
read_only: volume.is_read_only(),
|
||||
version: volume.version().0 as u32,
|
||||
remote_storage_name,
|
||||
|
||||
@@ -63,12 +63,6 @@ pub struct VolumeServerState {
|
||||
pub file_size_limit_bytes: i64,
|
||||
/// Default IO rate limit for maintenance copy/replication work.
|
||||
pub maintenance_byte_per_second: i64,
|
||||
/// Bounds concurrent vacuum commits; None means unlimited. Queued commits
|
||||
/// wait on the semaphore, not on the store lock, so reads keep flowing.
|
||||
/// Mirrors Go's `vacuumCommitSlots`.
|
||||
pub vacuum_commit_slots: Option<tokio::sync::Semaphore>,
|
||||
/// Vacuum commits currently waiting for a slot.
|
||||
pub vacuum_commits_waiting: AtomicU32,
|
||||
/// Whether the server is connected to master (heartbeat active).
|
||||
pub is_heartbeating: AtomicBool,
|
||||
/// Whether master addresses are configured.
|
||||
@@ -120,21 +114,6 @@ pub struct VolumeServerState {
|
||||
pub cli_white_list: Vec<String>,
|
||||
/// Path to state.pb file for persisting VolumeServerState across restarts.
|
||||
pub state_file_path: String,
|
||||
/// Volumes with an EC decode in flight. A dropped request leaves the
|
||||
/// blocking job running; this keeps a retry from racing it on the
|
||||
/// same volume files.
|
||||
pub ec_decodes_in_flight:
|
||||
std::sync::Mutex<std::collections::HashSet<crate::storage::types::VolumeId>>,
|
||||
/// Volumes whose EC decode is in its publishing tail (journal catch-up,
|
||||
/// .idx write, compaction). Local .ecj appenders wait on
|
||||
/// `ec_decode_tail_notify` while their vid is listed, so no committed
|
||||
/// delete falls between the last catch_up and the .cpd/.cpx swap —
|
||||
/// the per-volume slice of Go's EcVolume.ecjFileAccessLock.
|
||||
pub ec_decode_tail:
|
||||
std::sync::Mutex<std::collections::HashSet<crate::storage::types::VolumeId>>,
|
||||
/// Wakes .ecj appenders waiting on `ec_decode_tail` when a decode's
|
||||
/// publishing tail ends.
|
||||
pub ec_decode_tail_notify: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
impl VolumeServerState {
|
||||
@@ -150,40 +129,6 @@ impl VolumeServerState {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait for a vacuum commit slot without touching the store lock; the wait
|
||||
/// counts against the caller's `grpc-timeout`. An already-expired deadline
|
||||
/// is refused even when a slot is free, matching Go's post-acquire
|
||||
/// `ctx.Err()` check in `acquireVacuumCommitSlot`.
|
||||
pub async fn acquire_vacuum_commit_slot(
|
||||
&self,
|
||||
deadline: Option<tokio::time::Instant>,
|
||||
) -> Result<Option<tokio::sync::SemaphorePermit<'_>>, tonic::Status> {
|
||||
let Some(slots) = &self.vacuum_commit_slots else {
|
||||
return Ok(None);
|
||||
};
|
||||
self.vacuum_commits_waiting.fetch_add(1, Ordering::Relaxed);
|
||||
let acquired = match deadline {
|
||||
Some(d) => match tokio::time::timeout_at(d, slots.acquire()).await {
|
||||
Ok(permit) => permit.map_err(|e| tonic::Status::internal(e.to_string())),
|
||||
Err(_) => Err(tonic::Status::deadline_exceeded(
|
||||
"timed out waiting for a vacuum commit slot",
|
||||
)),
|
||||
},
|
||||
None => slots
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::internal(e.to_string())),
|
||||
};
|
||||
self.vacuum_commits_waiting.fetch_sub(1, Ordering::Relaxed);
|
||||
let permit = acquired?;
|
||||
if deadline.is_some_and(|d| tokio::time::Instant::now() >= d) {
|
||||
return Err(tonic::Status::deadline_exceeded(
|
||||
"timed out waiting for a vacuum commit slot",
|
||||
));
|
||||
}
|
||||
Ok(Some(permit))
|
||||
}
|
||||
|
||||
/// Build the seed master set from a list of raw `host:port[.grpcPort]`
|
||||
/// addresses, normalised the same way Go's `pb.ServerAddress.ToHttpAddress`
|
||||
/// does (drop the `.grpcPort` suffix, preserve everything else).
|
||||
|
||||
@@ -1,18 +1,14 @@
|
||||
//! Async batched write processing for the volume server.
|
||||
//!
|
||||
//! Instead of each upload handler directly calling `write_needle`, writes are
|
||||
//! submitted to a queue. Each volume gets its own FIFO worker (Go's
|
||||
//! volume_write.go keeps one batching worker per volume), which drains its
|
||||
//! channel in batches of up to 128 entries or 4 MiB and shares the .dat/.idx
|
||||
//! flushes across durable writes (see `Volume::write_needles_grouped`).
|
||||
//! Writes to different volumes proceed in parallel; a volume's ordering and
|
||||
//! exclusion still come from its data-file write lease.
|
||||
//! submitted to a queue. A background worker drains the queue in batches (up to
|
||||
//! 128 entries), groups them by volume ID, and processes them together under a
|
||||
//! single store lock. Durable writes to a volume share their .dat and .idx
|
||||
//! flushes (see `Volume::write_needles_grouped`).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot};
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tracing::debug;
|
||||
|
||||
use crate::storage::needle::needle::Needle;
|
||||
@@ -33,10 +29,6 @@ pub struct WriteRequest {
|
||||
pub needle: Needle,
|
||||
pub fsync: bool,
|
||||
pub response_tx: oneshot::Sender<WriteResult>,
|
||||
/// The global queued-slot permit, owned by the request itself: a caller
|
||||
/// that cancels `submit` after the send must not free the slot while the
|
||||
/// worker still holds the needle.
|
||||
pub queue_permit: Option<OwnedSemaphorePermit>,
|
||||
}
|
||||
|
||||
/// Maximum number of write requests to batch together.
|
||||
@@ -46,204 +38,101 @@ const MAX_BATCH_SIZE: usize = 128;
|
||||
/// This prevents large writes from accumulating unbounded latency.
|
||||
const MAX_BATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Cap on volumes doing grouped I/O at once, so many writers cannot crowd
|
||||
/// readers off a slow disk the way unlimited parallel fsyncs would.
|
||||
const MAX_CONCURRENT_VOLUME_WRITES: usize = 8;
|
||||
|
||||
/// How often an idle worker re-checks that its volume is still mounted.
|
||||
/// Workers live for their volume's queue lifetime (Go parity), so the only
|
||||
/// retirement is the volume leaving the store.
|
||||
const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60);
|
||||
|
||||
/// Handle for submitting write requests to the per-volume workers.
|
||||
/// Handle for submitting write requests to the background worker.
|
||||
#[derive(Clone)]
|
||||
pub struct WriteQueue {
|
||||
state: Arc<VolumeServerState>,
|
||||
/// Per-volume channel capacity; backpressure kicks in when full.
|
||||
capacity: usize,
|
||||
io_permits: Arc<Semaphore>,
|
||||
/// Bounds queued+in-flight writes across all volumes — per-volume channels
|
||||
/// alone would multiply the old single queue's capacity by volume count.
|
||||
queue_permits: Arc<Semaphore>,
|
||||
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
|
||||
tx: mpsc::Sender<WriteRequest>,
|
||||
}
|
||||
|
||||
impl WriteQueue {
|
||||
/// Create a new write queue. Per-volume workers are spawned lazily on the
|
||||
/// first submission for that volume.
|
||||
/// Create a new write queue and spawn the background worker.
|
||||
///
|
||||
/// `capacity` controls the channel buffer size (backpressure kicks in when full).
|
||||
/// The worker holds a reference to `state` for accessing the store.
|
||||
pub fn new(state: Arc<VolumeServerState>, capacity: usize) -> Self {
|
||||
WriteQueue {
|
||||
state,
|
||||
capacity,
|
||||
io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)),
|
||||
queue_permits: Arc::new(Semaphore::new(capacity)),
|
||||
workers: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sender for the volume's worker, spawning one on first use.
|
||||
fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender<WriteRequest> {
|
||||
let mut workers = self.workers.lock().unwrap();
|
||||
match workers.entry(volume_id) {
|
||||
Entry::Occupied(e) => e.get().clone(),
|
||||
Entry::Vacant(e) => {
|
||||
let (tx, rx) = mpsc::channel(self.capacity);
|
||||
tokio::spawn(
|
||||
VolumeWriteWorker {
|
||||
rx,
|
||||
vid: volume_id,
|
||||
state: self.state.clone(),
|
||||
io_permits: self.io_permits.clone(),
|
||||
workers: self.workers.clone(),
|
||||
}
|
||||
.run(),
|
||||
);
|
||||
e.insert(tx.clone());
|
||||
tx
|
||||
}
|
||||
}
|
||||
let (tx, rx) = mpsc::channel(capacity);
|
||||
let worker = WriteQueueWorker { rx, state };
|
||||
tokio::spawn(worker.run());
|
||||
WriteQueue { tx }
|
||||
}
|
||||
|
||||
/// Submit a write request and wait for the result.
|
||||
///
|
||||
/// Returns `Err` if the worker has shut down or the response channel was dropped.
|
||||
pub async fn submit(&self, volume_id: VolumeId, needle: Needle, fsync: bool) -> WriteResult {
|
||||
// Workers spawn only for mounted volumes — a bogus id never creates
|
||||
// a channel, so volume-id sprays can't grow the worker map.
|
||||
{
|
||||
let store = self.state.store.read().unwrap();
|
||||
if store.find_volume(volume_id).is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
}
|
||||
// The permit travels inside the request, so the global queued bound
|
||||
// covers the request until the worker drops it — including after a
|
||||
// canceled submit.
|
||||
let queue_permit = self
|
||||
.queue_permits
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| broken_pipe("write queue shut down"))?;
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
let mut request = WriteRequest {
|
||||
let request = WriteRequest {
|
||||
volume_id,
|
||||
needle,
|
||||
fsync,
|
||||
response_tx,
|
||||
queue_permit: Some(queue_permit),
|
||||
};
|
||||
|
||||
// A send fails only against a retiring worker's closed channel; the
|
||||
// request was never accepted, so retrying it behind a fresh worker
|
||||
// keeps same-volume order. The volume recheck bounds the spin.
|
||||
loop {
|
||||
let tx = self.worker_for(volume_id);
|
||||
match tx.send(request).await {
|
||||
Ok(()) => break,
|
||||
Err(mpsc::error::SendError(returned)) => {
|
||||
request = returned;
|
||||
{
|
||||
let store = self.state.store.read().unwrap();
|
||||
if store.find_volume(volume_id).is_none() {
|
||||
return Err(VolumeError::NotFound);
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
// Send to queue; this awaits if the channel is full (backpressure).
|
||||
if self.tx.send(request).await.is_err() {
|
||||
return Err(VolumeError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::BrokenPipe,
|
||||
"write queue worker has shut down",
|
||||
)));
|
||||
}
|
||||
|
||||
// Wait for the worker to process our request.
|
||||
match response_rx.await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(broken_pipe("write queue worker dropped response channel")),
|
||||
Err(_) => Err(VolumeError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::BrokenPipe,
|
||||
"write queue worker dropped response channel",
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn broken_pipe(msg: &'static str) -> VolumeError {
|
||||
VolumeError::Io(std::io::Error::new(std::io::ErrorKind::BrokenPipe, msg))
|
||||
}
|
||||
|
||||
/// FIFO worker draining one volume's write channel in batches. Like Go's
|
||||
/// per-volume async writer it runs for the volume's queue lifetime — exiting
|
||||
/// only when the channel closes or its volume leaves the store — so no
|
||||
/// handoff can ever split one volume's write order across two workers.
|
||||
struct VolumeWriteWorker {
|
||||
/// Background worker that drains write requests and processes them in batches.
|
||||
struct WriteQueueWorker {
|
||||
rx: mpsc::Receiver<WriteRequest>,
|
||||
vid: VolumeId,
|
||||
state: Arc<VolumeServerState>,
|
||||
io_permits: Arc<Semaphore>,
|
||||
workers: Arc<Mutex<HashMap<VolumeId, mpsc::Sender<WriteRequest>>>>,
|
||||
}
|
||||
|
||||
impl VolumeWriteWorker {
|
||||
impl WriteQueueWorker {
|
||||
async fn run(mut self) {
|
||||
debug!("write queue worker started");
|
||||
|
||||
loop {
|
||||
let first = match tokio::time::timeout(IDLE_WORKER_POLL, self.rx.recv()).await {
|
||||
Ok(Some(req)) => req,
|
||||
// recv() yields None only once every sender is gone — the map
|
||||
// holds a sender per volume, so that is exactly queue shutdown.
|
||||
Ok(None) => return,
|
||||
Err(_) if self.volume_gone() => {
|
||||
self.retire().await;
|
||||
// Wait for the first request (blocks until one arrives or channel closes).
|
||||
let first = match self.rx.recv().await {
|
||||
Some(req) => req,
|
||||
None => {
|
||||
debug!("write queue channel closed, worker exiting");
|
||||
return;
|
||||
}
|
||||
Err(_) => continue,
|
||||
};
|
||||
self.process_from(first).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// The volume's handle left the store, so the worker can go too; an idle
|
||||
/// worker for a mounted volume stays (Go keeps the writer for the mount).
|
||||
fn volume_gone(&self) -> bool {
|
||||
let store = self.state.store.read().unwrap();
|
||||
store.find_volume(self.vid).is_none()
|
||||
}
|
||||
// Drain as many additional requests as available, up to MAX_BATCH_SIZE
|
||||
// or MAX_BATCH_BYTES (matches Go: 128 requests or 4MB, whichever comes first).
|
||||
let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
|
||||
let mut batch_bytes: usize = first.needle.data.len();
|
||||
batch.push(first);
|
||||
|
||||
/// Retire in a fixed order that keeps same-volume FIFO: close the channel
|
||||
/// so racing senders fail fast (they retry and wait for a fresh worker),
|
||||
/// drain what was already accepted, and only then give up the map slot —
|
||||
/// a replacement worker can never start while this one still holds queued
|
||||
/// writes.
|
||||
async fn retire(&mut self) {
|
||||
self.rx.close();
|
||||
while let Some(first) = self.rx.recv().await {
|
||||
self.process_from(first).await;
|
||||
}
|
||||
self.workers.lock().unwrap().remove(&self.vid);
|
||||
}
|
||||
|
||||
/// Collect a batch around `first` and process it under the I/O bound.
|
||||
async fn process_from(&mut self, first: WriteRequest) {
|
||||
// Drain as many additional requests as available, up to MAX_BATCH_SIZE
|
||||
// or MAX_BATCH_BYTES (matches Go: 128 requests or 4MB, whichever comes first).
|
||||
let mut batch = Vec::with_capacity(MAX_BATCH_SIZE);
|
||||
let mut batch_bytes: usize = first.needle.data.len();
|
||||
batch.push(first);
|
||||
while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
|
||||
match self.rx.try_recv() {
|
||||
Ok(req) => {
|
||||
batch_bytes += req.needle.data.len();
|
||||
batch.push(req);
|
||||
while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES {
|
||||
match self.rx.try_recv() {
|
||||
Ok(req) => {
|
||||
batch_bytes += req.needle.data.len();
|
||||
batch.push(req);
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
debug!("processing write batch of {} requests", batch.len());
|
||||
|
||||
// Bound the number of volumes doing grouped I/O at once, then run
|
||||
// the batch in spawn_blocking since write_needle does file I/O.
|
||||
let Ok(_permit) = self.io_permits.clone().acquire_owned().await else {
|
||||
return;
|
||||
};
|
||||
let state = self.state.clone();
|
||||
let _ = tokio::task::spawn_blocking(move || {
|
||||
process_batch(state, batch);
|
||||
})
|
||||
.await;
|
||||
let batch_size = batch.len();
|
||||
debug!("processing write batch of {} requests", batch_size);
|
||||
|
||||
// Process the batch in spawn_blocking since write_needle does file I/O.
|
||||
let state = self.state.clone();
|
||||
let _ = tokio::task::spawn_blocking(move || {
|
||||
process_batch(state, batch);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,28 +155,15 @@ fn process_batch(state: Arc<VolumeServerState>, batch: Vec<WriteRequest>) {
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve each group's volume handle under a brief store read, then run
|
||||
// the group's I/O with the guard released — one volume's write must not
|
||||
// block admission on another volume's.
|
||||
// Process each volume group under a single store lock.
|
||||
let mut store = state.store.write().unwrap();
|
||||
|
||||
for (vid, entries) in groups {
|
||||
let (mut writes, senders): (Vec<_>, Vec<_>) = entries
|
||||
.into_iter()
|
||||
.map(|(needle, fsync, response_tx)| ((needle, fsync), response_tx))
|
||||
.unzip();
|
||||
let vol = {
|
||||
let store = state.store.read().unwrap();
|
||||
store.writable_volume(vid)
|
||||
};
|
||||
let results = match vol {
|
||||
Ok(vol) => vol.write_needles_grouped(&mut writes),
|
||||
Err(e) => writes
|
||||
.iter()
|
||||
.map(|_| match e {
|
||||
VolumeError::ReadOnly(vid) => Err(VolumeError::ReadOnly(vid)),
|
||||
_ => Err(VolumeError::NotFound),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let results = store.write_volume_needles(vid, &mut writes);
|
||||
for (response_tx, result) in senders.into_iter().zip(results) {
|
||||
// Send result back; ignore error if receiver dropped.
|
||||
let _ = response_tx.send(result);
|
||||
@@ -330,8 +206,6 @@ mod tests {
|
||||
rack: String::new(),
|
||||
file_size_limit_bytes: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
vacuum_commit_slots: None,
|
||||
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
|
||||
is_heartbeating: AtomicBool::new(false),
|
||||
has_master: false,
|
||||
pre_stop_seconds: 0,
|
||||
@@ -355,9 +229,6 @@ mod tests {
|
||||
security_file: String::new(),
|
||||
cli_white_list: vec![],
|
||||
state_file_path: String::new(),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail_notify: tokio::sync::Notify::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -489,7 +360,6 @@ mod tests {
|
||||
},
|
||||
fsync: true,
|
||||
response_tx,
|
||||
queue_permit: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
@@ -524,199 +394,4 @@ mod tests {
|
||||
let result = queue2.submit(VolumeId(1), needle, false).await;
|
||||
assert!(result.is_err()); // NotFound is fine -- the point is it doesn't panic
|
||||
}
|
||||
|
||||
/// Issue #791: a write parked inside its .dat append must not hold the
|
||||
/// store guard, so another volume still resolves and writes. The hook
|
||||
/// runs under the volume's data-file lease, so the assertion is that the
|
||||
/// store — not the per-volume serialization — stays open.
|
||||
#[test]
|
||||
fn test_process_batch_does_not_block_other_volumes() {
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::types::{DiskType, NeedleId};
|
||||
use crate::storage::volume::{VolumeSpec, needle_write_hook};
|
||||
use std::sync::Barrier;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let state = make_test_state();
|
||||
{
|
||||
let mut store = state.store.write().unwrap();
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
for vid in [VolumeId(1), VolumeId(2)] {
|
||||
store
|
||||
.add_volume(vid, DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
let in_append = Arc::new(Barrier::new(2));
|
||||
let release = Arc::new(Barrier::new(2));
|
||||
let _reg = needle_write_hook::register(NeedleId(101), {
|
||||
let in_append = in_append.clone();
|
||||
let release = release.clone();
|
||||
move |_| {
|
||||
in_append.wait();
|
||||
release.wait();
|
||||
}
|
||||
});
|
||||
|
||||
let (response_tx, mut response_rx) = oneshot::channel();
|
||||
let batch = vec![WriteRequest {
|
||||
volume_id: VolumeId(1),
|
||||
needle: Needle {
|
||||
id: NeedleId(101),
|
||||
cookie: 0x1111.into(),
|
||||
data: vec![7u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
},
|
||||
fsync: false,
|
||||
response_tx,
|
||||
queue_permit: None,
|
||||
}];
|
||||
let writer = {
|
||||
let state = state.clone();
|
||||
std::thread::spawn(move || process_batch(state, batch))
|
||||
};
|
||||
|
||||
in_append.wait(); // volume 1's write is inside its .dat append.
|
||||
|
||||
// The store still admits a lifecycle writer and a volume-2 write.
|
||||
drop(state.store.try_write().unwrap());
|
||||
let vol2 = state
|
||||
.store
|
||||
.read()
|
||||
.unwrap()
|
||||
.writable_volume(VolumeId(2))
|
||||
.unwrap();
|
||||
let mut n2 = Needle {
|
||||
id: NeedleId(9),
|
||||
cookie: 0x2222.into(),
|
||||
data: vec![9u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
};
|
||||
vol2.write_needle(&mut n2, true, false).unwrap();
|
||||
|
||||
// Same-volume exclusion is intact: volume 1's data-file lease is held.
|
||||
let (_, vol1) = state.store.read().unwrap().find_volume(VolumeId(1)).unwrap();
|
||||
assert!(vol1.instance().try_write_lock().is_none());
|
||||
|
||||
release.wait();
|
||||
writer.join().unwrap();
|
||||
assert!(response_rx.try_recv().unwrap().is_ok());
|
||||
assert!(vol1.instance().try_write_lock().is_some());
|
||||
}
|
||||
|
||||
/// A write parked inside volume 1's .dat append must not keep later
|
||||
/// batches for other volumes queued behind it: each volume drains through
|
||||
/// its own FIFO worker, while same-volume writes still serialize.
|
||||
/// NeedleId(102) is reserved for this test — hooks are keyed by needle id
|
||||
/// process-wide and parallel tests must not share one.
|
||||
#[tokio::test]
|
||||
async fn test_write_queue_does_not_block_other_volumes() {
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::types::{DiskType, NeedleId};
|
||||
use crate::storage::volume::{VolumeSpec, needle_write_hook};
|
||||
use std::sync::Barrier;
|
||||
|
||||
let tmp = tempfile::TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let state = make_test_state();
|
||||
{
|
||||
let mut store = state.store.write().unwrap();
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
for vid in [VolumeId(1), VolumeId(2)] {
|
||||
store
|
||||
.add_volume(vid, DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
let queue = WriteQueue::new(state, MAX_BATCH_SIZE);
|
||||
|
||||
let in_append = Arc::new(Barrier::new(2));
|
||||
let release = Arc::new(Barrier::new(2));
|
||||
let _reg = needle_write_hook::register(NeedleId(102), {
|
||||
let in_append = in_append.clone();
|
||||
let release = release.clone();
|
||||
move |_| {
|
||||
in_append.wait();
|
||||
release.wait();
|
||||
}
|
||||
});
|
||||
|
||||
let needle_a = Needle {
|
||||
id: NeedleId(102),
|
||||
cookie: 0x1111.into(),
|
||||
data: vec![7u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
};
|
||||
let a1 = tokio::spawn({
|
||||
let queue = queue.clone();
|
||||
async move { queue.submit(VolumeId(1), needle_a, false).await }
|
||||
});
|
||||
// The hook parks on a blocking thread, so wait on the barrier off the
|
||||
// runtime worker.
|
||||
let parked = {
|
||||
let in_append = in_append.clone();
|
||||
tokio::task::spawn_blocking(move || in_append.wait())
|
||||
};
|
||||
parked.await.unwrap(); // volume 1's write is inside its .dat append.
|
||||
|
||||
// Volume 2's worker is independent: its submit completes now.
|
||||
let needle_b = Needle {
|
||||
id: NeedleId(9),
|
||||
cookie: 0x2222.into(),
|
||||
data: vec![9u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
};
|
||||
queue.submit(VolumeId(2), needle_b, true).await.unwrap();
|
||||
|
||||
// Same-volume FIFO is preserved: a second volume-1 write stays queued.
|
||||
let needle_a2 = Needle {
|
||||
id: NeedleId(2),
|
||||
cookie: 0x3333.into(),
|
||||
data: vec![8u8; 8],
|
||||
data_size: 8,
|
||||
..Needle::default()
|
||||
};
|
||||
let mut a2 = tokio::spawn({
|
||||
let queue = queue.clone();
|
||||
async move { queue.submit(VolumeId(1), needle_a2, false).await }
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(200), &mut a2)
|
||||
.await
|
||||
.is_err(),
|
||||
"a same-volume write must queue behind the stalled one"
|
||||
);
|
||||
|
||||
let released = {
|
||||
let release = release.clone();
|
||||
tokio::task::spawn_blocking(move || release.wait())
|
||||
};
|
||||
released.await.unwrap();
|
||||
assert!(a1.await.unwrap().is_ok());
|
||||
assert!(a2.await.unwrap().is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,9 +18,7 @@ use crate::storage::erasure_coding::ec_shard::{
|
||||
DATA_SHARDS_COUNT, ERASURE_CODING_LARGE_BLOCK_SIZE, ERASURE_CODING_SMALL_BLOCK_SIZE,
|
||||
EcVolumeShard, ShardId,
|
||||
};
|
||||
use crate::storage::erasure_coding::ec_volume::{
|
||||
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
|
||||
};
|
||||
use crate::storage::erasure_coding::ec_volume::{EcVolume, is_usable_ecx_file};
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::super_block::SUPER_BLOCK_SIZE;
|
||||
use crate::storage::types::*;
|
||||
@@ -37,7 +35,7 @@ pub struct DiskLocation {
|
||||
pub tags: Vec<String>,
|
||||
pub max_volume_count: AtomicI32,
|
||||
pub original_max_volume_count: i32,
|
||||
volumes: HashMap<VolumeId, Arc<Volume>>,
|
||||
volumes: HashMap<VolumeId, Volume>,
|
||||
ec_volumes: HashMap<VolumeId, EcVolume>,
|
||||
pub is_disk_space_low: Arc<AtomicBool>,
|
||||
pub available_space: AtomicU64,
|
||||
@@ -237,7 +235,6 @@ impl DiskLocation {
|
||||
}
|
||||
|
||||
for (collection, vid, v) in self.open_volumes(to_load, needle_map_kind) {
|
||||
let v = Arc::new(v);
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&collection, "volume"])
|
||||
.inc();
|
||||
@@ -463,16 +460,13 @@ impl DiskLocation {
|
||||
let idx_base = volume_file_name(&self.idx_directory, collection, vid);
|
||||
const MAX_SHARD_COUNT: usize = 32;
|
||||
|
||||
// Remove index files from idx directory (.ecx, .ecj, and a compaction
|
||||
// tmp a crash may have left beside the .ecj)
|
||||
// Remove index files from idx directory (.ecx, .ecj)
|
||||
rm_if_present(format!("{}.ecx", idx_base))?;
|
||||
rm_if_present(format!("{}.ecj", idx_base))?;
|
||||
rm_if_present(format!("{}{}", idx_base, ECJ_COMPACT_TMP_EXT))?;
|
||||
// Also try data directory in case .ecx/.ecj were created before -dir.idx was configured
|
||||
if self.idx_directory != self.directory {
|
||||
rm_if_present(format!("{}.ecx", base))?;
|
||||
rm_if_present(format!("{}.ecj", base))?;
|
||||
rm_if_present(format!("{}{}", base, ECJ_COMPACT_TMP_EXT))?;
|
||||
}
|
||||
|
||||
// Remove all EC shard files (.ec00 ~ .ec31)
|
||||
@@ -550,23 +544,20 @@ impl DiskLocation {
|
||||
None
|
||||
}
|
||||
|
||||
/// Find a volume by ID. Returns a shared handle like Go's findVolume:
|
||||
/// the lookup borrows the map briefly and I/O runs on the handle after.
|
||||
pub fn find_volume(&self, vid: VolumeId) -> Option<Arc<Volume>> {
|
||||
self.volumes.get(&vid).cloned()
|
||||
/// Find a volume by ID.
|
||||
pub fn find_volume(&self, vid: VolumeId) -> Option<&Volume> {
|
||||
self.volumes.get(&vid)
|
||||
}
|
||||
|
||||
/// Find a volume by ID (mutable).
|
||||
pub fn find_volume_mut(&mut self, vid: VolumeId) -> Option<&mut Volume> {
|
||||
self.volumes.get_mut(&vid)
|
||||
}
|
||||
|
||||
/// Add a volume to this location.
|
||||
pub fn set_volume(&mut self, vid: VolumeId, volume: Volume) {
|
||||
let collection = volume.collection.clone();
|
||||
if let Some(old) = self.volumes.insert(vid, Arc::new(volume)) {
|
||||
// A held handle would keep writing the same files against a stale
|
||||
// in-memory index; close it so the stale handle fails closed.
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&old.collection, "volume"])
|
||||
.dec();
|
||||
old.close();
|
||||
}
|
||||
self.volumes.insert(vid, volume);
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&collection, "volume"])
|
||||
.inc();
|
||||
@@ -590,20 +581,13 @@ impl DiskLocation {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[spec.collection, "volume"])
|
||||
.inc();
|
||||
if let Some(old) = self.volumes.insert(vid, Arc::new(v)) {
|
||||
// Same fail-closed rule as set_volume: a stale handle must not
|
||||
// keep writing the files the replacement just reopened.
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&old.collection, "volume"])
|
||||
.dec();
|
||||
old.close();
|
||||
}
|
||||
self.volumes.insert(vid, v);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove and close a volume.
|
||||
pub fn unload_volume(&mut self, vid: VolumeId) -> Option<Arc<Volume>> {
|
||||
if let Some(v) = self.volumes.remove(&vid) {
|
||||
pub fn unload_volume(&mut self, vid: VolumeId) -> Option<Volume> {
|
||||
if let Some(mut v) = self.volumes.remove(&vid) {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
@@ -630,21 +614,12 @@ impl DiskLocation {
|
||||
{
|
||||
return Err(v.compacting_error());
|
||||
}
|
||||
if let Some(v) = self.volumes.remove(&vid) {
|
||||
match v.destroy(only_empty, only_garbage, keep_remote_data) {
|
||||
Ok(()) => {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
// A failed destroy left the local files in place, so the
|
||||
// replica stays registered rather than becoming a phantom.
|
||||
self.volumes.insert(vid, v);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
if let Some(mut v) = self.volumes.remove(&vid) {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
v.destroy(only_empty, only_garbage, keep_remote_data)?;
|
||||
Ok(())
|
||||
} else {
|
||||
Err(VolumeError::NotFound)
|
||||
}
|
||||
@@ -660,14 +635,12 @@ impl DiskLocation {
|
||||
.collect();
|
||||
|
||||
for vid in vids {
|
||||
if let Some(v) = self.volumes.remove(&vid) {
|
||||
if let Some(mut v) = self.volumes.remove(&vid) {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
if let Err(e) = v.destroy(false, false, false) {
|
||||
warn!(volume_id = vid.0, error = %e, "delete collection: failed to destroy volume");
|
||||
self.volumes.insert(vid, v);
|
||||
} else {
|
||||
crate::metrics::VOLUME_GAUGE
|
||||
.with_label_values(&[&v.collection, "volume"])
|
||||
.dec();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -708,7 +681,7 @@ impl DiskLocation {
|
||||
|
||||
/// Iterate over all volumes.
|
||||
pub fn iter_volumes(&self) -> impl Iterator<Item = (&VolumeId, &Volume)> {
|
||||
self.volumes.iter().map(|(k, v)| (k, v.as_ref()))
|
||||
self.volumes.iter()
|
||||
}
|
||||
|
||||
/// Number of free volume slots.
|
||||
@@ -729,7 +702,12 @@ impl DiskLocation {
|
||||
|
||||
/// Iterate over all volumes.
|
||||
pub fn volumes(&self) -> impl Iterator<Item = (&VolumeId, &Volume)> {
|
||||
self.volumes.iter().map(|(k, v)| (k, v.as_ref()))
|
||||
self.volumes.iter()
|
||||
}
|
||||
|
||||
/// Iterate over all volumes (mutable).
|
||||
pub fn volumes_mut(&mut self) -> impl Iterator<Item = (&VolumeId, &mut Volume)> {
|
||||
self.volumes.iter_mut()
|
||||
}
|
||||
|
||||
/// Sum of unused space in writable volumes (volumeSizeLimit - actual size per volume).
|
||||
|
||||
@@ -97,97 +97,21 @@ pub fn read_ecj_deletions(
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
) -> io::Result<HashSet<NeedleId>> {
|
||||
Ok(EcjDeletions::read(dirs, collection, volume_id)?.ids)
|
||||
}
|
||||
|
||||
/// The ids [`read_ecj_deletions`] returns, plus how far each journal was read
|
||||
/// so that ids journaled later can be added.
|
||||
pub struct EcjDeletions {
|
||||
pub ids: HashSet<NeedleId>,
|
||||
/// Each distinct journal path and the whole-record length read so far.
|
||||
journals: Vec<(String, u64)>,
|
||||
}
|
||||
|
||||
impl EcjDeletions {
|
||||
pub fn read(dirs: &[&str], collection: &str, volume_id: VolumeId) -> io::Result<Self> {
|
||||
let mut journals: Vec<(String, u64)> = Vec::new();
|
||||
for dir in dirs {
|
||||
let path = format!("{}.ecj", volume_file_name(dir, collection, volume_id));
|
||||
if !journals.iter().any(|(p, _)| *p == path) {
|
||||
journals.push((path, 0));
|
||||
}
|
||||
let mut ids = HashSet::new();
|
||||
for (i, dir) in dirs.iter().enumerate() {
|
||||
if dirs[..i].contains(dir) {
|
||||
continue;
|
||||
}
|
||||
let mut deletions = EcjDeletions {
|
||||
ids: HashSet::new(),
|
||||
journals,
|
||||
let path = format!("{}.ecj", volume_file_name(dir, collection, volume_id));
|
||||
let file = match File::open(&path) {
|
||||
Ok(file) => file,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
deletions.catch_up()?;
|
||||
Ok(deletions)
|
||||
}
|
||||
|
||||
/// Adds the ids appended to each journal since the last read. A journal
|
||||
/// that shrank is read again from the start — and the whole set rebuilt,
|
||||
/// since ids already folded in from the truncated tail may have been a
|
||||
/// rolled-back append. Non-regular journals (the FIFOs the tests stand
|
||||
/// in for a blocking disk) stat empty and cannot be rolled back, so they
|
||||
/// never count as shrunk.
|
||||
pub fn catch_up(&mut self) -> io::Result<()> {
|
||||
let mut shrank = false;
|
||||
for (path, read_to) in &self.journals {
|
||||
if *read_to == 0 {
|
||||
continue;
|
||||
}
|
||||
match std::fs::metadata(path) {
|
||||
Ok(m) => {
|
||||
if m.is_file() && m.len() < *read_to {
|
||||
shrank = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// A journal that was read before and is now gone shrank to
|
||||
// nothing (e.g. the volume was destroyed mid-scan) — the ids
|
||||
// read from it no longer reflect committed content.
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||
shrank = true;
|
||||
break;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
if shrank {
|
||||
self.ids.clear();
|
||||
for (_, read_to) in &mut self.journals {
|
||||
*read_to = 0;
|
||||
}
|
||||
}
|
||||
for (path, read_to) in &mut self.journals {
|
||||
let file = match File::open(&*path) {
|
||||
Ok(file) => file,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let len = file.metadata()?.len();
|
||||
if len < *read_to {
|
||||
*read_to = 0;
|
||||
}
|
||||
read_ecj_ids(&file, *read_to, len, &mut self.ids)?;
|
||||
*read_to = len - len % NEEDLE_ID_SIZE as u64;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clears the set and re-reads every journal from the start. Run under
|
||||
/// the caller's store read lock (no append in flight): the result is
|
||||
/// then exactly the committed content — an earlier unlocked read may
|
||||
/// have folded in bytes a rolled-back append later truncated, or missed
|
||||
/// a record re-appended to the very offset a rollback freed.
|
||||
pub fn rescan(&mut self) -> io::Result<()> {
|
||||
self.ids.clear();
|
||||
for (_, read_to) in &mut self.journals {
|
||||
*read_to = 0;
|
||||
}
|
||||
self.catch_up()
|
||||
let len = file.metadata()?.len();
|
||||
read_ecj_ids(&file, len, &mut ids)?;
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
/// What it takes to rebuild a volume's .dat from its EC data shards.
|
||||
@@ -388,30 +312,6 @@ pub fn write_dat_file_from_shards(spec: &DatRebuild<'_>) -> io::Result<()> {
|
||||
write_result
|
||||
}
|
||||
|
||||
/// Fails when the decoded `.dat` in `dat_dir` is shorter than the
|
||||
/// `dat_file_size` bytes its EC index references: the caller deletes the
|
||||
/// shards next, and they are the only other copy of the needles past the cut.
|
||||
/// A longer file passes.
|
||||
pub fn verify_decoded_dat_file(
|
||||
dat_dir: &str,
|
||||
collection: &str,
|
||||
volume_id: VolumeId,
|
||||
dat_file_size: i64,
|
||||
) -> io::Result<()> {
|
||||
let dat_path = format!("{}.dat", volume_file_name(dat_dir, collection, volume_id));
|
||||
let size = std::fs::metadata(&dat_path)?.len();
|
||||
if (size as i64) < dat_file_size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
format!(
|
||||
"decoded {} is {} bytes, short of the {} its ec index references",
|
||||
dat_path, size, dat_file_size
|
||||
),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write .idx file from .ecx index + .ecj deletion journal.
|
||||
///
|
||||
/// See [`write_idx_file_from_ec_index_with_dirs`]; everything lives in `dir`.
|
||||
@@ -502,7 +402,7 @@ mod tests {
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
// Create volume with data
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
@@ -890,77 +790,4 @@ mod tests {
|
||||
let expected: HashSet<NeedleId> = [1, 2, 3, 7, 9].into_iter().map(NeedleId).collect();
|
||||
assert_eq!(ids, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_decoded_dat_file_rejects_a_short_dat() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let dat_path = format!("{dir}/1.dat");
|
||||
|
||||
let err = verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap_err();
|
||||
assert_eq!(err.kind(), io::ErrorKind::NotFound);
|
||||
|
||||
std::fs::write(&dat_path, vec![0u8; 99]).unwrap();
|
||||
let err = verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap_err();
|
||||
assert!(err.to_string().contains("short of the 100"), "{err}");
|
||||
|
||||
std::fs::write(&dat_path, vec![0u8; 100]).unwrap();
|
||||
verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap();
|
||||
std::fs::write(&dat_path, vec![0u8; 101]).unwrap();
|
||||
verify_decoded_dat_file(dir, "", VolumeId(1), 100).unwrap();
|
||||
}
|
||||
|
||||
/// Ids appended after the first read, including the rest of a record torn
|
||||
/// at that point, are picked up; a journal that shrank is read again.
|
||||
#[test]
|
||||
fn test_ecj_deletions_catch_up_reads_appended_ids() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let ecj_path = format!("{dir}/1.ecj");
|
||||
let entry = |id: u64| {
|
||||
let mut buf = [0u8; NEEDLE_ID_SIZE];
|
||||
NeedleId(id).to_bytes(&mut buf);
|
||||
buf
|
||||
};
|
||||
let append = |bytes: &[u8]| {
|
||||
let mut f = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&ecj_path)
|
||||
.unwrap();
|
||||
f.write_all(bytes).unwrap();
|
||||
};
|
||||
let ids = |d: &EcjDeletions| {
|
||||
let mut ids: Vec<u64> = d.ids.iter().map(|id| id.0).collect();
|
||||
ids.sort();
|
||||
ids
|
||||
};
|
||||
|
||||
// No journal yet.
|
||||
let mut deletions = EcjDeletions::read(&[dir, dir], "", VolumeId(1)).unwrap();
|
||||
assert!(deletions.ids.is_empty());
|
||||
|
||||
append(&entry(1));
|
||||
append(&entry(2)[..3]);
|
||||
deletions.catch_up().unwrap();
|
||||
assert_eq!(ids(&deletions), [1]);
|
||||
|
||||
append(&entry(2)[3..]);
|
||||
append(&entry(3));
|
||||
deletions.catch_up().unwrap();
|
||||
assert_eq!(ids(&deletions), [1, 2, 3]);
|
||||
|
||||
// A shrunk journal is rebuilt from its surviving content: ids folded
|
||||
// in from the truncated tail may have been rolled back and must not
|
||||
// linger as phantom tombstones.
|
||||
std::fs::write(&ecj_path, entry(9)).unwrap();
|
||||
deletions.catch_up().unwrap();
|
||||
assert_eq!(ids(&deletions), [9]);
|
||||
|
||||
// A journal removed since it was read is the extreme shrink: its
|
||||
// earlier ids must go with it, not linger.
|
||||
std::fs::remove_file(&ecj_path).unwrap();
|
||||
deletions.catch_up().unwrap();
|
||||
assert!(deletions.ids.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -848,7 +848,7 @@ mod tests {
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
// Create a volume with some data
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
@@ -895,7 +895,7 @@ mod tests {
|
||||
fn make_volume_with_needles(n: u64) -> TempDir {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
@@ -970,7 +970,7 @@ mod tests {
|
||||
// the dir path so a test can drop/truncate shards and rebuild.
|
||||
fn encode_sample_volume(tmp: &TempDir) -> String {
|
||||
let dir = tmp.path().to_str().unwrap().to_string();
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
&dir,
|
||||
&dir,
|
||||
VolumeId(1),
|
||||
@@ -1167,7 +1167,7 @@ mod tests {
|
||||
use crate::storage::volume::{Volume, VolumeSpec};
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap().to_string();
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
&dir,
|
||||
&dir,
|
||||
VolumeId(2),
|
||||
@@ -1216,7 +1216,7 @@ mod tests {
|
||||
use crate::storage::volume::{Volume, VolumeSpec};
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap().to_string();
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
&dir,
|
||||
&dir,
|
||||
VolumeId(3),
|
||||
@@ -1316,7 +1316,7 @@ mod tests {
|
||||
let idx_dir = idx_tmp.path().to_str().unwrap();
|
||||
|
||||
// Create a volume with separate data and index directories
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dat_dir,
|
||||
idx_dir,
|
||||
VolumeId(1),
|
||||
@@ -1396,7 +1396,7 @@ mod tests {
|
||||
let idx_dir = idx_tmp.path().to_str().unwrap();
|
||||
let wrong_dir = wrong_tmp.path().to_str().unwrap();
|
||||
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dat_dir,
|
||||
idx_dir,
|
||||
VolumeId(1),
|
||||
@@ -1430,7 +1430,7 @@ mod tests {
|
||||
/// a caller-chosen directory, which is what a split-disk test needs: the
|
||||
/// shards have to be scattered out of the directory they were encoded into.
|
||||
fn seed_encoded_volume(dir: &str, vid: VolumeId) {
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
vid,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,290 +0,0 @@
|
||||
//! Set-union merge of `.ecj` deletion journals for EC shard copy / index
|
||||
//! recovery. Mirrors Go's `weed/storage/erasure_coding/ecj_merge.go`.
|
||||
//!
|
||||
//! An EC volume's deletion journal (`<vid>.ecj`) is a *set* of deleted needle
|
||||
//! ids stored as 8-byte big-endian records. Shard copy and index recovery fold
|
||||
//! a peer's journal into the local one; they must append only the ids the
|
||||
//! local journal lacks, or every `ec_balance` round trip doubles the file.
|
||||
//!
|
||||
//! The journal is only ever appended to, never replaced: a mounted `EcVolume`
|
||||
//! holds it open, and a rename would leave that handle writing to an unlinked
|
||||
//! inode, losing every later delete at the next mount. A mounted volume merges
|
||||
//! through [`EcVolume::merge_journal`](super::ec_volume::EcVolume::merge_journal);
|
||||
//! [`append_ecj_ids`] is for a journal no volume has open.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::storage::types::{NEEDLE_ID_SIZE, NeedleId};
|
||||
use crate::storage::volume::fsync_dir;
|
||||
use crate::storage::volume_open::open_volume_file;
|
||||
|
||||
/// Bytes per read when scanning a journal; a multiple of `NEEDLE_ID_SIZE`.
|
||||
const ECJ_READ_CHUNK_BYTES: usize = 1 << 20;
|
||||
|
||||
/// Decodes `.ecj` records from a byte stream split at arbitrary boundaries (a
|
||||
/// CopyFile stream, chunked reads), collecting the distinct ids. Memory
|
||||
/// follows the number of distinct ids, not the journal's length. A trailing
|
||||
/// partial record is never decoded.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct EcjIdDecoder {
|
||||
ids: HashSet<NeedleId>,
|
||||
partial: [u8; NEEDLE_ID_SIZE],
|
||||
pending: usize,
|
||||
}
|
||||
|
||||
impl EcjIdDecoder {
|
||||
pub(crate) fn push(&mut self, mut bytes: &[u8]) {
|
||||
if self.pending > 0 {
|
||||
let n = (NEEDLE_ID_SIZE - self.pending).min(bytes.len());
|
||||
self.partial[self.pending..self.pending + n].copy_from_slice(&bytes[..n]);
|
||||
self.pending += n;
|
||||
bytes = &bytes[n..];
|
||||
if self.pending < NEEDLE_ID_SIZE {
|
||||
return;
|
||||
}
|
||||
self.ids.insert(NeedleId::from_bytes(&self.partial));
|
||||
self.pending = 0;
|
||||
}
|
||||
let mut records = bytes.chunks_exact(NEEDLE_ID_SIZE);
|
||||
for record in &mut records {
|
||||
self.ids.insert(NeedleId::from_bytes(record));
|
||||
}
|
||||
let rest = records.remainder();
|
||||
self.partial[..rest.len()].copy_from_slice(rest);
|
||||
self.pending = rest.len();
|
||||
}
|
||||
|
||||
pub(crate) fn into_ids(self) -> HashSet<NeedleId> {
|
||||
self.ids
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the distinct ids of the journal at `path` in bounded chunks. A missing
|
||||
/// file reads as empty. Also returns the whole-record length read; a torn
|
||||
/// trailing partial record is excluded from it.
|
||||
pub(crate) fn read_ecj_ids(path: &str) -> io::Result<(HashSet<NeedleId>, u64)> {
|
||||
let mut file = match fs::File::open(path) {
|
||||
Ok(f) => f,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok((HashSet::new(), 0)),
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let len = file.metadata()?.len();
|
||||
let size = len - len % NEEDLE_ID_SIZE as u64;
|
||||
let mut decoder = EcjIdDecoder::default();
|
||||
let mut buf = vec![0u8; (ECJ_READ_CHUNK_BYTES as u64).min(size) as usize];
|
||||
let mut off = 0u64;
|
||||
while off < size {
|
||||
let want = (buf.len() as u64).min(size - off) as usize;
|
||||
file.read_exact(&mut buf[..want])?;
|
||||
decoder.push(&buf[..want]);
|
||||
off += want as u64;
|
||||
}
|
||||
Ok((decoder.into_ids(), size))
|
||||
}
|
||||
|
||||
/// The ids of `incoming` that `has` does not report, ascending, so a merge
|
||||
/// appends deterministic output.
|
||||
pub(crate) fn ecj_delta(
|
||||
incoming: &HashSet<NeedleId>,
|
||||
has: impl Fn(&NeedleId) -> bool,
|
||||
) -> Vec<NeedleId> {
|
||||
let mut delta: Vec<NeedleId> = incoming.iter().copied().filter(|id| !has(id)).collect();
|
||||
delta.sort_unstable();
|
||||
delta
|
||||
}
|
||||
|
||||
pub(crate) fn encode_ecj_ids(ids: &[NeedleId]) -> Vec<u8> {
|
||||
let mut buf = vec![0u8; ids.len() * NEEDLE_ID_SIZE];
|
||||
for (record, id) in buf.chunks_exact_mut(NEEDLE_ID_SIZE).zip(ids) {
|
||||
id.to_bytes(record);
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
/// Append to the journal at `path` the ids of `incoming` that `local` lacks,
|
||||
/// in one write and one fsync, returning how many were added. `local` and
|
||||
/// `size` come from [`read_ecj_ids`] on the same path: if the journal's
|
||||
/// whole-record length is no longer `size`, returns `Ok(None)` so the caller
|
||||
/// re-reads. A torn tail past `size` is truncated first so the new records
|
||||
/// stay aligned.
|
||||
pub(crate) fn append_ecj_ids(
|
||||
path: &str,
|
||||
local: &HashSet<NeedleId>,
|
||||
incoming: &HashSet<NeedleId>,
|
||||
size: u64,
|
||||
) -> io::Result<Option<usize>> {
|
||||
let delta = ecj_delta(incoming, |id| local.contains(id));
|
||||
if delta.is_empty() {
|
||||
return Ok(Some(0));
|
||||
}
|
||||
let created = !Path::new(path).exists();
|
||||
let mut file = open_volume_file(OpenOptions::new().read(true).write(true).create(true), path)?;
|
||||
let len = file.metadata()?.len();
|
||||
if len - len % NEEDLE_ID_SIZE as u64 != size {
|
||||
return Ok(None);
|
||||
}
|
||||
if len != size {
|
||||
file.set_len(size)?;
|
||||
}
|
||||
let appended = file
|
||||
.seek(SeekFrom::Start(size))
|
||||
.and_then(|_| file.write_all(&encode_ecj_ids(&delta)))
|
||||
.and_then(|_| file.sync_all());
|
||||
if let Err(e) = appended {
|
||||
let _ = file.set_len(size);
|
||||
return Err(e);
|
||||
}
|
||||
if created {
|
||||
fsync_dir(path)?;
|
||||
}
|
||||
Ok(Some(delta.len()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ids(v: &[u64]) -> HashSet<NeedleId> {
|
||||
v.iter().map(|&id| NeedleId(id)).collect()
|
||||
}
|
||||
|
||||
fn bytes(v: &[u64]) -> Vec<u8> {
|
||||
encode_ecj_ids(&v.iter().map(|&id| NeedleId(id)).collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
fn records(path: &str) -> Vec<u64> {
|
||||
let data = fs::read(path).expect("read ecj");
|
||||
assert_eq!(data.len() % NEEDLE_ID_SIZE, 0, "journal must stay aligned");
|
||||
data.chunks_exact(NEEDLE_ID_SIZE)
|
||||
.map(|c| NeedleId::from_bytes(c).0)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Run the unmounted merge the way the server does: read, then append.
|
||||
fn merge_file(path: &str, incoming: &HashSet<NeedleId>) -> usize {
|
||||
let (local, size) = read_ecj_ids(path).expect("read");
|
||||
append_ecj_ids(path, &local, incoming, size)
|
||||
.expect("append")
|
||||
.expect("journal unchanged")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_handles_records_split_across_chunks() {
|
||||
let mut stream = bytes(&[1, 2, 3, 2, 0x0102030405060708]);
|
||||
stream.extend_from_slice(&[9, 9, 9]);
|
||||
for chunk in 1..=stream.len() {
|
||||
let mut d = EcjIdDecoder::default();
|
||||
for piece in stream.chunks(chunk) {
|
||||
d.push(piece);
|
||||
}
|
||||
assert_eq!(
|
||||
d.into_ids(),
|
||||
ids(&[1, 2, 3, 0x0102030405060708]),
|
||||
"chunk {}",
|
||||
chunk
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_ecj_ids_dedups_and_ignores_torn_tail() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let missing = dir.path().join("missing.ecj");
|
||||
let (got, size) = read_ecj_ids(missing.to_str().unwrap()).expect("read");
|
||||
assert!(got.is_empty());
|
||||
assert_eq!(size, 0);
|
||||
|
||||
let torn = dir.path().join("torn.ecj");
|
||||
let mut data = bytes(&[1, 2, 1]);
|
||||
data.extend_from_slice(&[7, 7, 7]);
|
||||
fs::write(&torn, data).unwrap();
|
||||
let (got, size) = read_ecj_ids(torn.to_str().unwrap()).expect("read");
|
||||
assert_eq!(got, ids(&[1, 2]));
|
||||
assert_eq!(size, 3 * NEEDLE_ID_SIZE as u64);
|
||||
|
||||
// A bloated journal repeating a few ids across several read chunks
|
||||
// keeps only the distinct ids.
|
||||
let bloated = dir.path().join("bloated.ecj");
|
||||
let data: Vec<u8> = (0..300_000u64).flat_map(|i| bytes(&[i % 3])).collect();
|
||||
fs::write(&bloated, &data).unwrap();
|
||||
let (got, size) = read_ecj_ids(bloated.to_str().unwrap()).expect("read");
|
||||
assert_eq!(got, ids(&[0, 1, 2]));
|
||||
assert_eq!(size, data.len() as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn appends_only_missing_ids() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("vol.ecj");
|
||||
let path = path.to_str().unwrap();
|
||||
fs::write(path, bytes(&[1, 2, 3])).unwrap();
|
||||
assert_eq!(merge_file(path, &ids(&[3, 4])), 1);
|
||||
assert_eq!(records(path), vec![1, 2, 3, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_stays_constant() {
|
||||
// A->B->A->B 20 times: the old append path doubled the journal each trip.
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let a = dir.path().join("a.ecj");
|
||||
let b = dir.path().join("b.ecj");
|
||||
let (a, b) = (a.to_str().unwrap(), b.to_str().unwrap());
|
||||
fs::write(a, bytes(&[1, 2])).unwrap();
|
||||
fs::write(b, bytes(&[2, 3])).unwrap();
|
||||
for i in 0..20 {
|
||||
let (src, dst) = if i % 2 == 0 { (a, b) } else { (b, a) };
|
||||
let (incoming, _) = read_ecj_ids(src).expect("read");
|
||||
merge_file(dst, &incoming);
|
||||
}
|
||||
for path in [a, b] {
|
||||
let mut got = records(path);
|
||||
got.sort_unstable();
|
||||
assert_eq!(got, vec![1, 2, 3]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_new_leaves_journal_alone() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let missing = dir.path().join("missing.ecj");
|
||||
assert_eq!(merge_file(missing.to_str().unwrap(), &ids(&[])), 0);
|
||||
assert!(
|
||||
!missing.exists(),
|
||||
"an empty merge must not create a journal"
|
||||
);
|
||||
|
||||
let path = dir.path().join("vol.ecj");
|
||||
let path = path.to_str().unwrap();
|
||||
fs::write(path, bytes(&[1, 2])).unwrap();
|
||||
assert_eq!(merge_file(path, &ids(&[2, 1])), 0);
|
||||
assert_eq!(records(path), vec![1, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repairs_torn_tail() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("vol.ecj");
|
||||
let path = path.to_str().unwrap();
|
||||
let mut data = bytes(&[1, 2]);
|
||||
data.extend_from_slice(&[9, 9, 9]);
|
||||
fs::write(path, data).unwrap();
|
||||
assert_eq!(merge_file(path, &ids(&[3])), 1);
|
||||
assert_eq!(records(path), vec![1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_changed_journal() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = dir.path().join("vol.ecj");
|
||||
let path = path.to_str().unwrap();
|
||||
fs::write(path, bytes(&[1])).unwrap();
|
||||
let (local, size) = read_ecj_ids(path).expect("read");
|
||||
fs::write(path, bytes(&[1, 5])).unwrap();
|
||||
let outcome = append_ecj_ids(path, &local, &ids(&[2]), size).expect("append");
|
||||
assert_eq!(outcome, None);
|
||||
assert_eq!(records(path), vec![1, 5]);
|
||||
}
|
||||
}
|
||||
@@ -1,326 +0,0 @@
|
||||
//! Process-wide coordination of everything that touches one `.ecj` path.
|
||||
//!
|
||||
//! Mount-time compaction replaces a deletion journal with a new inode. That is
|
||||
//! only safe while nothing else in this process can write the old one:
|
||||
//!
|
||||
//! - **Holders** are mounted `EcVolume`s with an append handle on the path. A
|
||||
//! store holds one `EcVolume` per disk location, and a shard mount or a
|
||||
//! cross-disk reconcile can point one disk's volume at another disk's
|
||||
//! `.ecj`, so several holders of one path are normal. A holder that keeps
|
||||
//! appending to a replaced inode acknowledges deletes that are gone at the
|
||||
//! next mount.
|
||||
//! - **Writers** append to or replace the path by name without holding it
|
||||
//! open across calls: `ReceiveFile` of an EC `.ecj`, and the unmounted
|
||||
//! append in `merge_ec_journal`, which `VolumeEcShardsCopy` and EC index
|
||||
//! recovery funnel a peer's journal through. Bytes they write after the
|
||||
//! compactor sized the journal would be dropped by the rename.
|
||||
//!
|
||||
//! Compaction therefore runs only while its caller is the sole holder and no
|
||||
//! writer is active, and while it runs no holder may open the path and no
|
||||
//! writer may start. Both wait instead; a compaction rewrites only the distinct
|
||||
//! id set, so the wait is short.
|
||||
//!
|
||||
//! No writer active at the reservation is not enough: one that ran while the
|
||||
//! holder loaded the journal, or after, and has finished may have rewritten it
|
||||
//! in place to the same length (`ReceiveFile` truncates and refills), which
|
||||
//! the inode-and-size re-check cannot see. So each writer bumps the path's
|
||||
//! write generation as it starts, and a holder may compact only if no writer
|
||||
//! was active when it registered and the generation has not moved since.
|
||||
//!
|
||||
//! Paths are keyed by their canonical parent directory, so two disk locations
|
||||
//! that spell one directory differently still meet here.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Condvar, LazyLock, Mutex, MutexGuard};
|
||||
|
||||
#[derive(Default)]
|
||||
struct PathState {
|
||||
holders: usize,
|
||||
writers: usize,
|
||||
compacting: bool,
|
||||
/// Writers that have started on the path. Lives as long as the entry,
|
||||
/// which a registered holder keeps.
|
||||
write_gen: u64,
|
||||
}
|
||||
|
||||
impl PathState {
|
||||
fn idle(&self) -> bool {
|
||||
self.holders == 0 && self.writers == 0 && !self.compacting
|
||||
}
|
||||
}
|
||||
|
||||
struct Registry {
|
||||
paths: Mutex<HashMap<PathBuf, PathState>>,
|
||||
changed: Condvar,
|
||||
}
|
||||
|
||||
static REGISTRY: LazyLock<Registry> = LazyLock::new(|| Registry {
|
||||
paths: Mutex::new(HashMap::new()),
|
||||
changed: Condvar::new(),
|
||||
});
|
||||
|
||||
fn lock() -> MutexGuard<'static, HashMap<PathBuf, PathState>> {
|
||||
// The critical sections only adjust counters and cannot panic midway, so
|
||||
// a poisoned lock still guards consistent state.
|
||||
REGISTRY.paths.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// Canonical key for `path`: its resolved parent directory joined with the
|
||||
/// file name. The file itself may not exist yet (a copy creates it), so only
|
||||
/// the directory is resolved.
|
||||
fn key_for(path: &str) -> PathBuf {
|
||||
let p = Path::new(path);
|
||||
let (Some(parent), Some(name)) = (p.parent(), p.file_name()) else {
|
||||
return std::path::absolute(p).unwrap_or_else(|_| p.to_path_buf());
|
||||
};
|
||||
let parent = if parent.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
parent
|
||||
};
|
||||
let dir = std::fs::canonicalize(parent)
|
||||
.or_else(|_| std::path::absolute(parent))
|
||||
.unwrap_or_else(|_| parent.to_path_buf());
|
||||
dir.join(name)
|
||||
}
|
||||
|
||||
/// Block until no compaction is running on `key`, then apply `f` to its state.
|
||||
fn update_when_not_compacting<R>(key: &Path, f: impl FnOnce(&mut PathState) -> R) -> R {
|
||||
let mut paths = lock();
|
||||
while paths.get(key).is_some_and(|s| s.compacting) {
|
||||
paths = REGISTRY
|
||||
.changed
|
||||
.wait(paths)
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
}
|
||||
f(paths.entry(key.to_path_buf()).or_default())
|
||||
}
|
||||
|
||||
fn release(key: &Path, f: impl FnOnce(&mut PathState)) {
|
||||
let mut paths = lock();
|
||||
if let Some(state) = paths.get_mut(key) {
|
||||
f(state);
|
||||
if state.idle() {
|
||||
paths.remove(key);
|
||||
}
|
||||
}
|
||||
drop(paths);
|
||||
REGISTRY.changed.notify_all();
|
||||
}
|
||||
|
||||
/// A mounted `EcVolume`'s registration as a holder of its `.ecj`. Taken before
|
||||
/// the journal is opened and released when dropped.
|
||||
pub(crate) struct EcjHold {
|
||||
key: PathBuf,
|
||||
/// The path's write generation when the hold was taken, and whether a
|
||||
/// writer was active then. Taken before the journal is opened and loaded,
|
||||
/// so they cover every write the load might have missed.
|
||||
write_gen: u64,
|
||||
writer_at_start: bool,
|
||||
}
|
||||
|
||||
impl EcjHold {
|
||||
/// Register as a holder of `ecj_path`, first waiting out any compaction in
|
||||
/// progress so the handle opened afterwards is on the final inode.
|
||||
pub(crate) fn acquire(ecj_path: &str) -> Self {
|
||||
let key = key_for(ecj_path);
|
||||
let (write_gen, writer_at_start) = update_when_not_compacting(&key, |s| {
|
||||
s.holders += 1;
|
||||
(s.write_gen, s.writers > 0)
|
||||
});
|
||||
EcjHold {
|
||||
key,
|
||||
write_gen,
|
||||
writer_at_start,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserve the path for a compaction, or `None` when another holder or an
|
||||
/// active writer could still reach the current inode, or when a writer
|
||||
/// has run on the path since the hold was taken, so the journal may no
|
||||
/// longer be what the holder loaded.
|
||||
pub(crate) fn try_begin_compaction(&self) -> Option<EcjCompaction> {
|
||||
let mut paths = lock();
|
||||
let state = paths.get_mut(&self.key)?;
|
||||
if state.holders != 1 || state.writers != 0 || state.compacting {
|
||||
return None;
|
||||
}
|
||||
if self.writer_at_start || state.write_gen != self.write_gen {
|
||||
return None;
|
||||
}
|
||||
state.compacting = true;
|
||||
Some(EcjCompaction {
|
||||
key: self.key.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EcjHold {
|
||||
fn drop(&mut self) {
|
||||
release(&self.key, |s| s.holders = s.holders.saturating_sub(1));
|
||||
}
|
||||
}
|
||||
|
||||
/// An exclusive reservation of a `.ecj` path for compaction. Holders and
|
||||
/// writers wait until it is dropped.
|
||||
pub(crate) struct EcjCompaction {
|
||||
key: PathBuf,
|
||||
}
|
||||
|
||||
impl Drop for EcjCompaction {
|
||||
fn drop(&mut self) {
|
||||
release(&self.key, |s| s.compacting = false);
|
||||
}
|
||||
}
|
||||
|
||||
/// An out-of-band writer (shard copy, index recovery, `ReceiveFile`) on a
|
||||
/// `.ecj` path.
|
||||
/// Compaction does not start while one is alive.
|
||||
pub(crate) struct EcjWrite {
|
||||
key: PathBuf,
|
||||
}
|
||||
|
||||
impl Drop for EcjWrite {
|
||||
fn drop(&mut self) {
|
||||
release(&self.key, |s| s.writers = s.writers.saturating_sub(1));
|
||||
}
|
||||
}
|
||||
|
||||
/// Register as a writer of `ecj_path`, waiting out any compaction in progress.
|
||||
/// Blocks; async callers use [`begin_ecj_write_async`].
|
||||
pub(crate) fn begin_ecj_write(ecj_path: &str) -> EcjWrite {
|
||||
let key = key_for(ecj_path);
|
||||
update_when_not_compacting(&key, |s| {
|
||||
s.writers += 1;
|
||||
s.write_gen += 1;
|
||||
});
|
||||
EcjWrite { key }
|
||||
}
|
||||
|
||||
/// [`begin_ecj_write`] for async handlers: the wait runs on the blocking pool
|
||||
/// so a compaction in progress never stalls a runtime worker.
|
||||
pub(crate) async fn begin_ecj_write_async(ecj_path: &str) -> EcjWrite {
|
||||
let path = ecj_path.to_string();
|
||||
match tokio::task::spawn_blocking(move || begin_ecj_write(&path)).await {
|
||||
Ok(write) => write,
|
||||
// Only a panic inside the registry lands here, and it leaves no count
|
||||
// behind; registering inline is still correct, merely blocking.
|
||||
Err(_) => begin_ecj_write(ecj_path),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::mpsc;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn ecj(dir: &TempDir) -> String {
|
||||
dir.path().join("1.ecj").to_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sole_holder_may_compact() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let hold = EcjHold::acquire(&ecj(&dir));
|
||||
assert!(hold.try_begin_compaction().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn second_holder_blocks_compaction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let a = EcjHold::acquire(&ecj(&dir));
|
||||
let b = EcjHold::acquire(&ecj(&dir));
|
||||
assert!(a.try_begin_compaction().is_none());
|
||||
drop(b);
|
||||
assert!(a.try_begin_compaction().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn active_writer_blocks_compaction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let hold = EcjHold::acquire(&ecj(&dir));
|
||||
let w = begin_ecj_write(&ecj(&dir));
|
||||
assert!(hold.try_begin_compaction().is_none());
|
||||
drop(w);
|
||||
// This hold loaded before the write; a later one may compact.
|
||||
drop(hold);
|
||||
let hold = EcjHold::acquire(&ecj(&dir));
|
||||
assert!(hold.try_begin_compaction().is_some());
|
||||
}
|
||||
|
||||
/// A writer that ran after the hold was taken, or was already running
|
||||
/// then, may have changed the journal the holder loaded, even though it
|
||||
/// has finished by the time compaction asks.
|
||||
#[test]
|
||||
fn finished_writer_since_hold_blocks_compaction() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = ecj(&dir);
|
||||
|
||||
let hold = EcjHold::acquire(&path);
|
||||
drop(begin_ecj_write(&path));
|
||||
assert!(
|
||||
hold.try_begin_compaction().is_none(),
|
||||
"a writer that started after the hold",
|
||||
);
|
||||
drop(hold);
|
||||
|
||||
let w = begin_ecj_write(&path);
|
||||
let hold = EcjHold::acquire(&path);
|
||||
drop(w);
|
||||
assert!(
|
||||
hold.try_begin_compaction().is_none(),
|
||||
"a writer active when the hold was taken",
|
||||
);
|
||||
drop(hold);
|
||||
|
||||
let hold = EcjHold::acquire(&path);
|
||||
assert!(
|
||||
hold.try_begin_compaction().is_some(),
|
||||
"no writer since the hold",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn differently_spelled_paths_share_one_key() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::create_dir(dir.path().join("sub")).unwrap();
|
||||
let plain = ecj(&dir);
|
||||
let dotted = dir
|
||||
.path()
|
||||
.join("sub")
|
||||
.join("..")
|
||||
.join("1.ecj")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
let a = EcjHold::acquire(&plain);
|
||||
let _b = EcjHold::acquire(&dotted);
|
||||
assert!(a.try_begin_compaction().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writer_waits_for_compaction_to_finish() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let path = ecj(&dir);
|
||||
let hold = EcjHold::acquire(&path);
|
||||
let compaction = hold.try_begin_compaction().unwrap();
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let p = path.clone();
|
||||
let t = std::thread::spawn(move || {
|
||||
let _w = begin_ecj_write(&p);
|
||||
tx.send(()).unwrap();
|
||||
});
|
||||
assert!(
|
||||
rx.recv_timeout(Duration::from_millis(100)).is_err(),
|
||||
"a writer must not start while a compaction holds the path",
|
||||
);
|
||||
drop(compaction);
|
||||
rx.recv_timeout(Duration::from_secs(5))
|
||||
.expect("writer must proceed once the compaction ends");
|
||||
t.join().unwrap();
|
||||
}
|
||||
}
|
||||
@@ -9,8 +9,6 @@ pub mod ec_encoder;
|
||||
pub mod ec_locate;
|
||||
pub mod ec_shard;
|
||||
pub mod ec_volume;
|
||||
pub mod ecj_merge;
|
||||
pub(crate) mod ecj_registry;
|
||||
|
||||
pub use ec_shard::{
|
||||
DATA_SHARDS_COUNT, EcVolumeShard, MAX_SHARD_COUNT, MIN_TOTAL_DISKS, PARITY_SHARDS_COUNT,
|
||||
|
||||
@@ -6,12 +6,10 @@ pub(crate) mod io_error;
|
||||
pub mod needle;
|
||||
pub mod needle_map;
|
||||
pub mod store;
|
||||
pub mod store_ec_journal;
|
||||
pub mod store_ec_mirror;
|
||||
pub mod store_ec_reconcile;
|
||||
pub mod super_block;
|
||||
pub mod types;
|
||||
pub mod vid_stamp;
|
||||
pub mod volume;
|
||||
pub mod volume_idx_rebuild;
|
||||
pub mod volume_idx_repair;
|
||||
|
||||
@@ -195,12 +195,7 @@ impl NeedleMapKind {
|
||||
// ============================================================================
|
||||
|
||||
/// Trait for appending to an index file.
|
||||
///
|
||||
/// The file is opened without append mode and each row is written at the
|
||||
/// current `idx_file_offset` — the same positioned-write model the Go
|
||||
/// server uses — because an append-mode handle cannot truncate on Windows,
|
||||
/// where the std library keeps it strictly append-only.
|
||||
pub trait IdxFileWriter: Write + Seek + Send + Sync {
|
||||
pub trait IdxFileWriter: Write + Send + Sync {
|
||||
fn sync_all(&self) -> io::Result<()>;
|
||||
/// Truncate the file to `len` bytes. Used to remove an orphan .idx row
|
||||
/// left by a failed redb commit so `idx_file_offset` stays a contiguous
|
||||
@@ -229,9 +224,6 @@ pub struct CompactNeedleMap {
|
||||
metric: NeedleMapMetric,
|
||||
idx_file: Option<Box<dyn IdxFileWriter>>,
|
||||
idx_file_offset: u64,
|
||||
/// The file holds bytes past `idx_file_offset` that must be trimmed
|
||||
/// before another row can land aligned.
|
||||
idx_torn: bool,
|
||||
}
|
||||
|
||||
impl Default for CompactNeedleMap {
|
||||
@@ -248,7 +240,6 @@ impl CompactNeedleMap {
|
||||
metric: NeedleMapMetric::default(),
|
||||
idx_file: None,
|
||||
idx_file_offset: 0,
|
||||
idx_torn: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -273,7 +264,6 @@ impl CompactNeedleMap {
|
||||
pub fn set_idx_file(&mut self, file: Box<dyn IdxFileWriter>, offset: u64) {
|
||||
self.idx_file = Some(file);
|
||||
self.idx_file_offset = offset;
|
||||
self.idx_torn = false;
|
||||
}
|
||||
|
||||
/// True when an .idx file writer is attached. A read-only load leaves
|
||||
@@ -288,8 +278,8 @@ impl CompactNeedleMap {
|
||||
/// Insert or update an entry. Appends to .idx file if present.
|
||||
pub fn put(&mut self, key: NeedleId, offset: Offset, size: Size) -> io::Result<()> {
|
||||
// Persist to idx file BEFORE mutating in-memory state for crash consistency
|
||||
self.append_to_index_file(key, offset, size)?;
|
||||
if self.idx_file.is_some() {
|
||||
if let Some(ref mut idx_file) = self.idx_file {
|
||||
idx::write_index_entry(idx_file, key, offset, size)?;
|
||||
self.idx_file_offset += NEEDLE_MAP_ENTRY_SIZE as u64;
|
||||
}
|
||||
|
||||
@@ -299,41 +289,6 @@ impl CompactNeedleMap {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write one row to the .idx file at `idx_file_offset`. A row left
|
||||
/// half-written by a failed write is trimmed back to the offset so the
|
||||
/// next row still lands aligned; while the trim keeps failing no row is
|
||||
/// written at all, or it would sit off alignment and parse as garbage on
|
||||
/// load. The offset itself is advanced by the caller once the row counts.
|
||||
fn append_to_index_file(
|
||||
&mut self,
|
||||
key: NeedleId,
|
||||
offset: Offset,
|
||||
size: Size,
|
||||
) -> io::Result<()> {
|
||||
let Some(idx_file) = self.idx_file.as_mut() else {
|
||||
return Ok(());
|
||||
};
|
||||
if self.idx_torn {
|
||||
match idx_file.truncate_to(self.idx_file_offset) {
|
||||
Ok(()) => self.idx_torn = false,
|
||||
Err(e) => {
|
||||
return Err(io::Error::other(format!(
|
||||
"index file still holds a torn row: {e}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
idx_file.seek(io::SeekFrom::Start(self.idx_file_offset))?;
|
||||
if let Err(e) = idx::write_index_entry(idx_file, key, offset, size) {
|
||||
if let Err(te) = idx_file.truncate_to(self.idx_file_offset) {
|
||||
self.idx_torn = true;
|
||||
tracing::warn!("failed to trim torn .idx row: {}", te);
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Look up a needle.
|
||||
pub fn get(&self, key: NeedleId) -> Option<NeedleValue> {
|
||||
self.map.get(key)
|
||||
@@ -358,8 +313,8 @@ impl CompactNeedleMap {
|
||||
}
|
||||
|
||||
// Always write tombstone to idx file (matching Go)
|
||||
self.append_to_index_file(key, offset, TOMBSTONE_FILE_SIZE)?;
|
||||
if self.idx_file.is_some() {
|
||||
if let Some(ref mut idx_file) = self.idx_file {
|
||||
idx::write_index_entry(idx_file, key, offset, TOMBSTONE_FILE_SIZE)?;
|
||||
self.idx_file_offset += NEEDLE_MAP_ENTRY_SIZE as u64;
|
||||
}
|
||||
|
||||
@@ -509,9 +464,6 @@ pub struct RedbNeedleMap {
|
||||
metric: NeedleMapMetric,
|
||||
idx_file: Option<Box<dyn IdxFileWriter>>,
|
||||
idx_file_offset: u64,
|
||||
/// The file holds bytes past `idx_file_offset` that must be trimmed
|
||||
/// before another row can land aligned.
|
||||
idx_torn: bool,
|
||||
/// Puts/deletes since the last durable checkpoint.
|
||||
writes_since_checkpoint: u32,
|
||||
}
|
||||
@@ -618,7 +570,6 @@ impl RedbNeedleMap {
|
||||
metric: NeedleMapMetric::default(),
|
||||
idx_file: None,
|
||||
idx_file_offset: 0,
|
||||
idx_torn: false,
|
||||
writes_since_checkpoint: 0,
|
||||
})
|
||||
}
|
||||
@@ -716,7 +667,6 @@ impl RedbNeedleMap {
|
||||
metric: NeedleMapMetric::default(),
|
||||
idx_file: None,
|
||||
idx_file_offset: 0,
|
||||
idx_torn: false,
|
||||
writes_since_checkpoint: 0,
|
||||
};
|
||||
|
||||
@@ -908,7 +858,6 @@ impl RedbNeedleMap {
|
||||
pub fn set_idx_file(&mut self, file: Box<dyn IdxFileWriter>, offset: u64) {
|
||||
self.idx_file = Some(file);
|
||||
self.idx_file_offset = offset;
|
||||
self.idx_torn = false;
|
||||
}
|
||||
|
||||
/// True when an .idx file writer is attached. See CompactNeedleMap.
|
||||
@@ -925,7 +874,9 @@ impl RedbNeedleMap {
|
||||
// commit leaves an orphan row in .idx that redb doesn't reflect, and
|
||||
// advancing the offset here would let a later checkpoint record it as
|
||||
// reflected, making the reload skip it permanently.
|
||||
self.append_to_index_file(key, offset, size)?;
|
||||
if let Some(ref mut idx_file) = self.idx_file {
|
||||
idx::write_index_entry(idx_file, key, offset, size)?;
|
||||
}
|
||||
|
||||
let key_u64: u64 = key.into();
|
||||
let packed = pack_needle_value(&NeedleValue { offset, size });
|
||||
@@ -996,41 +947,6 @@ impl RedbNeedleMap {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write one row to the .idx file at `idx_file_offset`. A row left
|
||||
/// half-written by a failed write is trimmed back to the offset so the
|
||||
/// next row still lands aligned; while the trim keeps failing no row is
|
||||
/// written at all, or it would sit off alignment and parse as garbage on
|
||||
/// load. The offset itself is advanced by the caller once the row counts.
|
||||
fn append_to_index_file(
|
||||
&mut self,
|
||||
key: NeedleId,
|
||||
offset: Offset,
|
||||
size: Size,
|
||||
) -> io::Result<()> {
|
||||
let Some(idx_file) = self.idx_file.as_mut() else {
|
||||
return Ok(());
|
||||
};
|
||||
if self.idx_torn {
|
||||
match idx_file.truncate_to(self.idx_file_offset) {
|
||||
Ok(()) => self.idx_torn = false,
|
||||
Err(e) => {
|
||||
return Err(io::Error::other(format!(
|
||||
"index file still holds a torn row: {e}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
idx_file.seek(io::SeekFrom::Start(self.idx_file_offset))?;
|
||||
if let Err(e) = idx::write_index_entry(idx_file, key, offset, size) {
|
||||
if let Err(te) = idx_file.truncate_to(self.idx_file_offset) {
|
||||
self.idx_torn = true;
|
||||
tracing::warn!("failed to trim torn .idx row: {}", te);
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Look up a needle. A redb failure is an ERROR, not an absent needle:
|
||||
/// answering "not found" would turn a database problem into a read miss
|
||||
/// and let a delete report success without recording a tombstone.
|
||||
@@ -1077,7 +993,9 @@ impl RedbNeedleMap {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
self.append_to_index_file(key, offset, TOMBSTONE_FILE_SIZE)?;
|
||||
if let Some(ref mut idx_file) = self.idx_file {
|
||||
idx::write_index_entry(idx_file, key, offset, TOMBSTONE_FILE_SIZE)?;
|
||||
}
|
||||
|
||||
let deleted_nv = NeedleValue {
|
||||
offset: old.offset,
|
||||
@@ -1167,13 +1085,10 @@ impl RedbNeedleMap {
|
||||
/// a failed redb commit. Without this the next successful write appends
|
||||
/// after the orphan, `idx_file_offset` advances past it, and a later
|
||||
/// checkpoint records an offset that makes the reload skip the orphan.
|
||||
/// When the trim fails the file is latched torn so no later row lands
|
||||
/// after bytes the map does not reflect.
|
||||
fn truncate_idx_to_offset(&mut self) {
|
||||
if let Some(ref mut idx_file) = self.idx_file
|
||||
&& let Err(e) = idx_file.truncate_to(self.idx_file_offset)
|
||||
{
|
||||
self.idx_torn = true;
|
||||
tracing::warn!("failed to truncate orphan .idx row: {}", e);
|
||||
}
|
||||
}
|
||||
@@ -1224,7 +1139,6 @@ impl RedbNeedleMap {
|
||||
self.db = reopened.db;
|
||||
self.metric = reopened.metric;
|
||||
self.idx_file_offset = actual_idx_size;
|
||||
self.idx_torn = reopened.idx_torn;
|
||||
// The reopen replayed all rows since the last durable checkpoint
|
||||
// non-durably; start the counter fresh.
|
||||
self.writes_since_checkpoint = 0;
|
||||
@@ -1777,7 +1691,7 @@ mod tests {
|
||||
)
|
||||
.unwrap();
|
||||
let writer = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.append(true)
|
||||
.open(&idx_path)
|
||||
.unwrap();
|
||||
nm.set_idx_file(Box::new(writer), idx_size);
|
||||
|
||||
@@ -6,52 +6,20 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::pb::master_pb;
|
||||
use crate::storage::disk_location::DiskLocation;
|
||||
use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId};
|
||||
use crate::storage::erasure_coding::ec_volume::{
|
||||
ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
|
||||
};
|
||||
use crate::storage::erasure_coding::ec_volume::{EcVolume, is_usable_ecx_file};
|
||||
use crate::storage::needle::needle::{Needle, get_actual_size};
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::super_block::{ReplicaPlacement, SUPER_BLOCK_SIZE};
|
||||
use crate::storage::types::*;
|
||||
use crate::storage::volume::{CompactionJob, VifVolumeInfo, Volume, VolumeError, VolumeSpec};
|
||||
|
||||
/// Mirrors Go's ensureCompactVolumeSpace, per filesystem: the new .dat lands
|
||||
/// next to the old one and the new .idx next to the old index, so when the
|
||||
/// index directory is on another filesystem each disk answers for its own
|
||||
/// share, while two directories on one filesystem must cover the sum.
|
||||
fn ensure_compact_volume_space(v: &Volume, preallocate: u64) -> Result<(), VolumeError> {
|
||||
let (data_bytes, index_bytes) = compaction_space_needed(v, preallocate);
|
||||
let (dir, dir_idx) = (v.dir(), v.dir_idx());
|
||||
let check = |dir: &str, needed: u64| -> Result<(), VolumeError> {
|
||||
let (_, free) = crate::storage::disk_location::get_disk_stats(dir);
|
||||
if free < needed {
|
||||
return Err(VolumeError::InsufficientSpace {
|
||||
vid: v.id,
|
||||
required: needed,
|
||||
free,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
if dir_idx.is_empty() || dir_idx == dir {
|
||||
check(dir, data_bytes + index_bytes)
|
||||
} else if !same_filesystem(dir, &dir_idx) {
|
||||
check(dir, data_bytes)?;
|
||||
check(&dir_idx, index_bytes)
|
||||
} else {
|
||||
check(dir, data_bytes + index_bytes)?;
|
||||
check(&dir_idx, index_bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// Top-level storage manager containing all disk locations and their volumes.
|
||||
pub struct Store {
|
||||
pub locations: Vec<DiskLocation>,
|
||||
@@ -72,10 +40,6 @@ pub struct Store {
|
||||
/// reporting the heartbeat that saw it.
|
||||
pub reported_collections: Mutex<HashSet<String>>,
|
||||
pub reported_ec_collections: Mutex<HashSet<String>>,
|
||||
pub max_volume_id_seen: AtomicU32,
|
||||
pub last_seen_topology_id: Mutex<String>,
|
||||
pub vid_stamp_lock: Mutex<()>,
|
||||
pub vid_stamp_dirty: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
@@ -95,10 +59,6 @@ impl Store {
|
||||
rack: String::new(),
|
||||
reported_collections: Mutex::new(HashSet::new()),
|
||||
reported_ec_collections: Mutex::new(HashSet::new()),
|
||||
max_volume_id_seen: AtomicU32::new(0),
|
||||
last_seen_topology_id: Mutex::new(String::new()),
|
||||
vid_stamp_lock: Mutex::new(()),
|
||||
vid_stamp_dirty: std::sync::atomic::AtomicBool::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,12 +120,6 @@ impl Store {
|
||||
// (read-only target, partial copy).
|
||||
self.reconcile_ec_shards_across_disks();
|
||||
|
||||
self.fold_location_into_vid_stamp(self.locations.last().unwrap());
|
||||
// Write the combined bound back to every dir, including ones whose
|
||||
// own stamp or volumes sit below it — a lost disk must not restart
|
||||
// the server below the bound its remaining dirs once reached.
|
||||
self.save_vid_stamps();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -184,13 +138,8 @@ impl Store {
|
||||
|
||||
// ---- Volume lookup ----
|
||||
|
||||
/// Find which location contains a volume, returning a shared handle.
|
||||
/// Mirroring Go's findVolume: the lookup only borrows the volume map,
|
||||
/// so callers can run volume I/O after releasing the store guard.
|
||||
pub fn find_volume(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
) -> Option<(usize, Arc<crate::storage::volume::Volume>)> {
|
||||
/// Find which location contains a volume.
|
||||
pub fn find_volume(&self, vid: VolumeId) -> Option<(usize, &crate::storage::volume::Volume)> {
|
||||
for (i, loc) in self.locations.iter().enumerate() {
|
||||
if let Some(v) = loc.find_volume(vid) {
|
||||
return Some((i, v));
|
||||
@@ -199,6 +148,19 @@ impl Store {
|
||||
None
|
||||
}
|
||||
|
||||
/// Find which location contains a volume (mutable).
|
||||
pub fn find_volume_mut(
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
) -> Option<(usize, &mut crate::storage::volume::Volume)> {
|
||||
for (i, loc) in self.locations.iter_mut().enumerate() {
|
||||
if let Some(v) = loc.find_volume_mut(vid) {
|
||||
return Some((i, v));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Check if a volume exists.
|
||||
pub fn has_volume(&self, vid: VolumeId) -> bool {
|
||||
self.find_volume(vid).is_some()
|
||||
@@ -208,10 +170,10 @@ impl Store {
|
||||
/// the volume in place. A no-op when the location has no separate index
|
||||
/// directory. Mirrors Go's `Store::ConsolidateVolumeIndex`.
|
||||
pub fn consolidate_volume_index(&mut self, vid: VolumeId) -> Result<(), VolumeError> {
|
||||
for loc in self.locations.iter() {
|
||||
for loc in self.locations.iter_mut() {
|
||||
let idx_dir = loc.idx_directory.clone();
|
||||
let data_dir = loc.directory.clone();
|
||||
if let Some(v) = loc.find_volume(vid) {
|
||||
if let Some(v) = loc.find_volume_mut(vid) {
|
||||
if idx_dir == data_dir {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -423,7 +385,6 @@ impl Store {
|
||||
disk_type: DiskType,
|
||||
spec: &VolumeSpec<'_>,
|
||||
) -> Result<(), VolumeError> {
|
||||
self.stamp_allocated_vid(vid);
|
||||
if self.find_volume(vid).is_some() {
|
||||
return Err(VolumeError::AlreadyExists);
|
||||
}
|
||||
@@ -743,20 +704,6 @@ impl Store {
|
||||
vol.read_needle(n)
|
||||
}
|
||||
|
||||
/// Read a needle from a volume using the given read option.
|
||||
///
|
||||
/// On return, `read_option.is_meta_only` tells the caller whether the
|
||||
/// payload was skipped.
|
||||
pub fn read_volume_needle_with_option(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
n: &mut Needle,
|
||||
read_option: &mut ReadOption,
|
||||
) -> Result<i32, VolumeError> {
|
||||
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
|
||||
vol.read_needle_with_option(n, read_option)
|
||||
}
|
||||
|
||||
/// Read a needle from a volume, optionally reading deleted needles.
|
||||
pub fn read_volume_needle_opt(
|
||||
&self,
|
||||
@@ -779,43 +726,51 @@ impl Store {
|
||||
vol.needle_read_plan(id, read_deleted)
|
||||
}
|
||||
|
||||
/// Re-lookup a needle's data-file offset after compaction may have moved it.
|
||||
/// Returns `(new_data_file_offset, current_compaction_revision)`.
|
||||
pub fn re_lookup_needle_data_offset(
|
||||
&self,
|
||||
vid: VolumeId,
|
||||
needle_id: NeedleId,
|
||||
) -> Result<(u64, u16), VolumeError> {
|
||||
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
|
||||
vol.re_lookup_needle_data_offset(needle_id)
|
||||
}
|
||||
|
||||
/// Write a needle to a volume. With `fsync` the volume flushes its .dat
|
||||
/// before returning, so the caller can ack a durable write.
|
||||
pub fn write_volume_needle(
|
||||
&self,
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
n: &mut Needle,
|
||||
fsync: bool,
|
||||
) -> Result<(u64, Size, bool), VolumeError> {
|
||||
self.writable_volume(vid)?.write_needle(n, true, fsync)
|
||||
self.writable_volume_mut(vid)?.write_needle(n, true, fsync)
|
||||
}
|
||||
|
||||
/// Write a batch of needles to one volume, sharing the syncs of its
|
||||
/// durable writes. See `Volume::write_needles_grouped`.
|
||||
pub fn write_volume_needles(
|
||||
&self,
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
writes: &mut [(Needle, bool)],
|
||||
) -> Vec<Result<(u64, Size, bool), VolumeError>> {
|
||||
match self.writable_volume(vid) {
|
||||
match self.writable_volume_mut(vid) {
|
||||
Ok(vol) => vol.write_needles_grouped(writes),
|
||||
// The lookup fails only with NotFound or the disk-space ReadOnly.
|
||||
Err(e) => writes
|
||||
.iter()
|
||||
.map(|_| match e {
|
||||
VolumeError::ReadOnly(vid) => Err(VolumeError::ReadOnly(vid)),
|
||||
VolumeError::ReadOnly(_) => Err(VolumeError::ReadOnly(vid)),
|
||||
_ => Err(VolumeError::NotFound),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a mounted volume handle for writing, checking disk space like
|
||||
/// Go's writableVolume. The returned Arc stays valid for I/O after the
|
||||
/// store guard is released; if the volume is unmounted meanwhile the I/O
|
||||
/// fails on the volume's closed state, not on a stale pointer.
|
||||
pub(crate) fn writable_volume(&self, vid: VolumeId) -> Result<Arc<Volume>, VolumeError> {
|
||||
fn writable_volume_mut(&mut self, vid: VolumeId) -> Result<&mut Volume, VolumeError> {
|
||||
// Check disk space on the location containing this volume.
|
||||
// We do this before the mutable borrow to avoid borrow conflicts.
|
||||
let loc_idx = self
|
||||
.find_volume(vid)
|
||||
.map(|(i, _)| i)
|
||||
@@ -827,13 +782,13 @@ impl Store {
|
||||
return Err(VolumeError::ReadOnly(vid));
|
||||
}
|
||||
|
||||
let (_, vol) = self.find_volume(vid).ok_or(VolumeError::NotFound)?;
|
||||
let (_, vol) = self.find_volume_mut(vid).ok_or(VolumeError::NotFound)?;
|
||||
Ok(vol)
|
||||
}
|
||||
|
||||
/// Delete a needle from a volume.
|
||||
pub fn delete_volume_needle(
|
||||
&self,
|
||||
&mut self,
|
||||
vid: VolumeId,
|
||||
n: &mut Needle,
|
||||
) -> Result<Size, VolumeError> {
|
||||
@@ -842,6 +797,8 @@ impl Store {
|
||||
if vol.is_no_write_or_delete() {
|
||||
return Err(VolumeError::ReadOnly(vid));
|
||||
}
|
||||
|
||||
let (_, vol) = self.find_volume_mut(vid).ok_or(VolumeError::NotFound)?;
|
||||
vol.delete_needle(n)
|
||||
}
|
||||
|
||||
@@ -967,7 +924,6 @@ impl Store {
|
||||
collection: &str,
|
||||
shard_ids: &[ShardId],
|
||||
) -> Result<(), VolumeError> {
|
||||
self.stamp_allocated_vid(vid);
|
||||
// Find the location where the EC files live
|
||||
let loc_idx = self.find_ec_location(vid, collection).ok_or_else(|| {
|
||||
VolumeError::Io(io::Error::new(
|
||||
@@ -990,7 +946,6 @@ impl Store {
|
||||
shard_id: ShardId,
|
||||
source_disk_type: &str,
|
||||
) -> Result<(), VolumeError> {
|
||||
self.stamp_allocated_vid(vid);
|
||||
// The .ecx may live on a different disk than the shard being mounted
|
||||
// (ec.balance / ec.rebuild can spread shards across sibling disks), so
|
||||
// look up its owner once and point EcVolume::new at the directory that
|
||||
@@ -1490,12 +1445,10 @@ impl Store {
|
||||
crate::storage::volume::volume_file_name(&loc.directory, collection, vid);
|
||||
let _ = std::fs::remove_file(format!("{}.ecx", idx_base));
|
||||
let _ = std::fs::remove_file(format!("{}.ecj", idx_base));
|
||||
let _ = std::fs::remove_file(format!("{}{}", idx_base, ECJ_COMPACT_TMP_EXT));
|
||||
// Also try data directory in case .ecx/.ecj were created before -dir.idx
|
||||
if loc.idx_directory != loc.directory {
|
||||
let _ = std::fs::remove_file(format!("{}.ecx", data_base));
|
||||
let _ = std::fs::remove_file(format!("{}.ecj", data_base));
|
||||
let _ = std::fs::remove_file(format!("{}{}", data_base, ECJ_COMPACT_TMP_EXT));
|
||||
}
|
||||
// A shard-only disk also drops its stale .vif (Go
|
||||
// removeEcSharedIndexFiles): a live .idx means this disk still
|
||||
@@ -1611,52 +1564,44 @@ impl Store {
|
||||
vid: VolumeId,
|
||||
preallocate: u64,
|
||||
) -> Result<Option<CompactionJob>, VolumeError> {
|
||||
let (_, v) = self
|
||||
.find_volume(vid)
|
||||
.ok_or(VolumeError::VolumeNotFound(vid))?;
|
||||
ensure_compact_volume_space(&v, preallocate)?;
|
||||
// Required space matches Go's CompactVolume check: the larger of the
|
||||
// requested preallocation and the estimated compacted size — the live
|
||||
// needles, not the .dat the garbage already occupies, so a full disk
|
||||
// can still be reclaimed.
|
||||
let (loc_idx, space_needed) = {
|
||||
let (loc_idx, v) = self
|
||||
.find_volume(vid)
|
||||
.ok_or(VolumeError::VolumeNotFound(vid))?;
|
||||
let live_count = (v.file_count() - v.deleted_count()).max(0) as u64;
|
||||
let live_bytes = v.content_size().saturating_sub(v.deleted_size());
|
||||
let per_needle = (get_actual_size(Size(0), v.version())
|
||||
+ NEEDLE_PADDING_SIZE as i64
|
||||
+ NEEDLE_MAP_ENTRY_SIZE as i64) as u64;
|
||||
let estimated = SUPER_BLOCK_SIZE as u64 + live_count * per_needle + live_bytes;
|
||||
let space_needed = std::cmp::max(preallocate, estimated);
|
||||
(loc_idx, space_needed + space_needed / 10)
|
||||
};
|
||||
|
||||
let dir = self.locations[loc_idx].directory.clone();
|
||||
let (_, free) = crate::storage::disk_location::get_disk_stats(&dir);
|
||||
if free < space_needed {
|
||||
return Err(VolumeError::InsufficientSpace {
|
||||
vid,
|
||||
required: space_needed,
|
||||
free,
|
||||
});
|
||||
}
|
||||
|
||||
let (_, v) = self
|
||||
.find_volume(vid)
|
||||
.find_volume_mut(vid)
|
||||
.ok_or(VolumeError::VolumeNotFound(vid))?;
|
||||
v.begin_compact_by_index()
|
||||
}
|
||||
|
||||
/// Rewrite the volume in `dir`/`dir_idx`, which is not mounted, with its
|
||||
/// live needles only. Go's `Store.CompactVolumeFiles`.
|
||||
pub fn compact_volume_files(
|
||||
dir: &str,
|
||||
dir_idx: &str,
|
||||
collection: &str,
|
||||
vid: VolumeId,
|
||||
needle_map_kind: NeedleMapKind,
|
||||
) -> Result<(), VolumeError> {
|
||||
let spec = VolumeSpec {
|
||||
collection,
|
||||
..VolumeSpec::default()
|
||||
};
|
||||
let v = Volume::new(dir, dir_idx, vid, needle_map_kind, &spec)?;
|
||||
let compact = || -> Result<(), VolumeError> {
|
||||
ensure_compact_volume_space(&v, 0)?;
|
||||
v.compact_by_index(0, 0, |_| true)?;
|
||||
v.commit_compact()
|
||||
};
|
||||
let result = compact();
|
||||
if result.is_err() {
|
||||
// A failed commit may have swapped only one of .dat/.idx;
|
||||
// reconcile rolls a decided swap forward or removes orphan
|
||||
// temp files before this volume can mount a mismatched pair.
|
||||
let _ = v.reconcile_compact_state();
|
||||
let _ = v.cleanup_compact();
|
||||
}
|
||||
v.close();
|
||||
result
|
||||
}
|
||||
|
||||
/// Commit a completed compaction: swap files and reload.
|
||||
pub fn commit_compact_volume(&mut self, vid: VolumeId) -> Result<(bool, u64), VolumeError> {
|
||||
let (_, v) = self
|
||||
.find_volume(vid)
|
||||
.find_volume_mut(vid)
|
||||
.ok_or(VolumeError::VolumeNotFound(vid))?;
|
||||
let is_read_only = v.is_read_only();
|
||||
v.commit_compact()?;
|
||||
@@ -1667,7 +1612,7 @@ impl Store {
|
||||
/// Clean up leftover compaction files.
|
||||
pub fn cleanup_compact_volume(&mut self, vid: VolumeId) -> Result<(), VolumeError> {
|
||||
let (_, v) = self
|
||||
.find_volume(vid)
|
||||
.find_volume_mut(vid)
|
||||
.ok_or(VolumeError::VolumeNotFound(vid))?;
|
||||
v.cleanup_compact()
|
||||
}
|
||||
@@ -1817,65 +1762,6 @@ fn owned_ec_shard_count(loc: &DiskLocation, vid: VolumeId, shard_ids: &[ShardId]
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Mirrors Go's compactionSpaceNeeded: what compaction will write, split into
|
||||
/// the new .dat and rebuilt .idx shares, each capped at its current file.
|
||||
fn compaction_space_needed(v: &Volume, preallocate: u64) -> (u64, u64) {
|
||||
let mut data_bytes = v.current_dat_file_size().unwrap_or(0);
|
||||
let mut index_bytes = v.idx_file_size();
|
||||
|
||||
let live_count = v.file_count() - v.deleted_count();
|
||||
let live_content = v.content_size() as i64 - v.deleted_size() as i64;
|
||||
// Unknown or inconsistent deleted sizes: the whole volume stays the
|
||||
// estimate.
|
||||
let deleted_size_known = v.deleted_count() == 0 || v.deleted_size() > 0;
|
||||
if deleted_size_known && live_count >= 0 && live_content >= 0 {
|
||||
// Empty-needle framing plus a padding unit covers the worst case.
|
||||
let per_needle =
|
||||
(get_actual_size(Size(0), v.version()) + NEEDLE_PADDING_SIZE as i64) as u64;
|
||||
let estimate = with_headroom(
|
||||
SUPER_BLOCK_SIZE as u64 + live_content as u64 + live_count as u64 * per_needle,
|
||||
);
|
||||
if estimate < data_bytes {
|
||||
data_bytes = estimate;
|
||||
}
|
||||
let estimate = with_headroom(live_count as u64 * NEEDLE_MAP_ENTRY_SIZE as u64);
|
||||
if estimate < index_bytes {
|
||||
index_bytes = estimate;
|
||||
}
|
||||
}
|
||||
if preallocate > data_bytes {
|
||||
data_bytes = preallocate;
|
||||
}
|
||||
(data_bytes, index_bytes)
|
||||
}
|
||||
|
||||
/// Headroom for Bloom-filter false positives in the live/deleted counters.
|
||||
fn with_headroom(estimate: u64) -> u64 {
|
||||
estimate + estimate / 16
|
||||
}
|
||||
|
||||
/// Whether two directories draw on the same free-space pool; in doubt, yes.
|
||||
fn same_filesystem(a: &str, b: &str) -> bool {
|
||||
if a == b {
|
||||
return true;
|
||||
}
|
||||
same_filesystem_impl(a, b)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn same_filesystem_impl(a: &str, b: &str) -> bool {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
match (std::fs::metadata(a), std::fs::metadata(b)) {
|
||||
(Ok(ma), Ok(mb)) => ma.dev() == mb.dev(),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn same_filesystem_impl(_a: &str, _b: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
@@ -2597,52 +2483,6 @@ mod tests {
|
||||
assert_eq!(selected, Some(0));
|
||||
}
|
||||
|
||||
// VolumeCopy picks a disk before deleting the replica it replaces, so only
|
||||
// the location actually holding that replica may count its slot as free.
|
||||
#[test]
|
||||
fn test_find_free_location_predicate_credits_only_the_holding_location() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let dir1 = tmp1.path().to_str().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
let dir2 = tmp2.path().to_str().unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
for dir in [dir1, dir2] {
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
1,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
for vid in [81, 82] {
|
||||
store
|
||||
.add_volume(VolumeId(vid), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
}
|
||||
let loc_of = |vid| store.find_volume(VolumeId(vid)).unwrap().0;
|
||||
assert_ne!(loc_of(81), loc_of(82), "fixture must fill both locations");
|
||||
|
||||
let hdd = |loc: &DiskLocation| loc.disk_type == DiskType::HardDrive;
|
||||
assert_eq!(store.find_free_location_predicate(hdd, None), None);
|
||||
assert_eq!(
|
||||
store.find_free_location_predicate(hdd, Some(VolumeId(99))),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
store.find_free_location_predicate(hdd, Some(VolumeId(81))),
|
||||
Some(loc_of(81))
|
||||
);
|
||||
assert_eq!(
|
||||
store.find_free_location_predicate(hdd, Some(VolumeId(82))),
|
||||
Some(loc_of(82))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_expired_ec_volumes_removes_expired_entries() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
@@ -2800,7 +2640,7 @@ mod tests {
|
||||
}
|
||||
let revision = {
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
v.super_block().compaction_revision
|
||||
v.super_block.compaction_revision
|
||||
};
|
||||
|
||||
let job = store
|
||||
@@ -2813,20 +2653,20 @@ mod tests {
|
||||
assert!(store.delete_volume(vid, false, false, false).is_err());
|
||||
store.delete_collection("").unwrap();
|
||||
assert!(store.cleanup_compact_volume(vid).is_err());
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
let (_, v) = store.find_volume_mut(vid).unwrap();
|
||||
assert!(v.relocate_index_to(other.path().to_str().unwrap()).is_err());
|
||||
// Go parity: a commit that finds the volume compacting is a no-op.
|
||||
store.commit_compact_volume(vid).unwrap();
|
||||
|
||||
let (_, v) = store.find_volume(vid).expect("still mounted");
|
||||
assert!(v.is_compacting());
|
||||
assert_eq!(v.super_block().compaction_revision, revision);
|
||||
assert_eq!(v.super_block.compaction_revision, revision);
|
||||
|
||||
job.run(|_| true).unwrap();
|
||||
assert!(!store.find_volume(vid).unwrap().1.is_compacting());
|
||||
store.commit_compact_volume(vid).unwrap();
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
assert_eq!(v.super_block().compaction_revision, revision + 1);
|
||||
assert_eq!(v.super_block.compaction_revision, revision + 1);
|
||||
assert!(store.unmount_volume(vid).unwrap());
|
||||
}
|
||||
|
||||
@@ -2866,7 +2706,7 @@ mod tests {
|
||||
store.delete_volume_needle(vid, &mut del).unwrap();
|
||||
let revision = {
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
v.super_block().compaction_revision
|
||||
v.super_block.compaction_revision
|
||||
};
|
||||
|
||||
let job = store.begin_compact_volume(vid, 0).unwrap().unwrap();
|
||||
@@ -2895,7 +2735,7 @@ mod tests {
|
||||
store.commit_compact_volume(vid).unwrap();
|
||||
|
||||
let (_, v) = store.find_volume(vid).unwrap();
|
||||
assert_eq!(v.super_block().compaction_revision, revision + 1);
|
||||
assert_eq!(v.super_block.compaction_revision, revision + 1);
|
||||
assert_eq!(read_test_needle(&store, vid, 99).unwrap(), b"late-write");
|
||||
assert_eq!(read_test_needle(&store, vid, 3).unwrap(), b"overwritten");
|
||||
assert!(read_test_needle(&store, vid, 4).is_err());
|
||||
|
||||
@@ -1,374 +0,0 @@
|
||||
//! Merging a peer's `.ecj` deletion ids into a local EC journal. Mirrors Go's
|
||||
//! `Store.MergeEcJournal` (`weed/storage/store_ec_journal.go`).
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::sync::RwLock;
|
||||
|
||||
use crate::storage::erasure_coding::ecj_merge::{append_ecj_ids, read_ecj_ids};
|
||||
use crate::storage::store::Store;
|
||||
use crate::storage::types::{NeedleId, VolumeId};
|
||||
|
||||
/// How often an unmounted merge re-reads a journal that changed under it. Only
|
||||
/// a mount-delete-unmount or a concurrent merge between the read and the
|
||||
/// append changes it, so one retry is nearly always enough.
|
||||
const ECJ_MERGE_ATTEMPTS: usize = 5;
|
||||
|
||||
/// Fold a peer's deletion `ids` into the local journal of EC volume `vid` on
|
||||
/// the receiving disk, the one whose data directory is `data_dir`; `ecj_path`
|
||||
/// is that journal's path in the disk's index directory. Appends only the ids
|
||||
/// the journal lacks and returns how many it added. Blocking: call it from
|
||||
/// `spawn_blocking`.
|
||||
///
|
||||
/// A mounted volume owns its journal: the merge goes through its open handle
|
||||
/// and in-memory set. That is the receiving disk's own runtime for `vid`,
|
||||
/// wherever its journal lives (it may sit in the data dir rather than
|
||||
/// `ecj_path`'s index dir), else a sibling runtime journaling into `ecj_path`
|
||||
/// itself: disks sharing one index directory, or reconciliation mounting `vid`
|
||||
/// on a disk that journals into another's (#9212). Otherwise `ecj_path` is
|
||||
/// appended to under the store write lock, which mounts take, so no mount can
|
||||
/// open it mid-append. The read that computes the delta runs outside the lock,
|
||||
/// and a journal that changed in between — including by a concurrent merge —
|
||||
/// is re-read.
|
||||
pub fn merge_ec_journal(
|
||||
store: &RwLock<Store>,
|
||||
vid: VolumeId,
|
||||
data_dir: &str,
|
||||
ecj_path: &str,
|
||||
ids: &HashSet<NeedleId>,
|
||||
) -> io::Result<usize> {
|
||||
merge_ec_journal_with(store, vid, data_dir, ecj_path, ids, read_ecj_ids)
|
||||
}
|
||||
|
||||
/// `merge_ec_journal` with the unlocked journal read injected, so a test can
|
||||
/// mount the volume between that read and the append.
|
||||
fn merge_ec_journal_with(
|
||||
store: &RwLock<Store>,
|
||||
vid: VolumeId,
|
||||
data_dir: &str,
|
||||
ecj_path: &str,
|
||||
ids: &HashSet<NeedleId>,
|
||||
mut read: impl FnMut(&str) -> io::Result<(HashSet<NeedleId>, u64)>,
|
||||
) -> io::Result<usize> {
|
||||
for _ in 0..ECJ_MERGE_ATTEMPTS {
|
||||
{
|
||||
let mut store = store
|
||||
.write()
|
||||
.map_err(|_| io::Error::other("store lock poisoned"))?;
|
||||
if let Some(primary) = mounted_ec_journal(&store, vid, data_dir, ecj_path)? {
|
||||
let ecv = store.locations[primary]
|
||||
.find_ec_volume_mut(vid)
|
||||
.expect("mounted journal runtime");
|
||||
let journal_path = ecv.ecj_file_name();
|
||||
let added = ecv.merge_journal(ids)?;
|
||||
// Publish to the holders of the file the merge wrote to — the
|
||||
// picked runtime's journal may live outside ecj_path, and a
|
||||
// holder of a different file must not claim ids it lacks.
|
||||
publish_to_journal_siblings(&store, primary, vid, &journal_path, ids);
|
||||
return Ok(added);
|
||||
}
|
||||
}
|
||||
let (local, size) = read(ecj_path)?;
|
||||
let mut store = store
|
||||
.write()
|
||||
.map_err(|_| io::Error::other("store lock poisoned"))?;
|
||||
if let Some(primary) = mounted_ec_journal(&store, vid, data_dir, ecj_path)? {
|
||||
// Mounted since the read: its handle owns the journal now.
|
||||
let ecv = store.locations[primary]
|
||||
.find_ec_volume_mut(vid)
|
||||
.expect("mounted journal runtime");
|
||||
let journal_path = ecv.ecj_file_name();
|
||||
let added = ecv.merge_journal(ids)?;
|
||||
publish_to_journal_siblings(&store, primary, vid, &journal_path, ids);
|
||||
return Ok(added);
|
||||
}
|
||||
// The path append registers as a writer so a mount compacting this
|
||||
// journal cannot swap its inode underneath it (the write itself is
|
||||
// already serialized with mounts by the store lock).
|
||||
let _ecj_write =
|
||||
crate::storage::erasure_coding::ecj_registry::begin_ecj_write(ecj_path);
|
||||
if let Some(added) = append_ecj_ids(ecj_path, &local, ids, size)? {
|
||||
return Ok(added);
|
||||
}
|
||||
}
|
||||
Err(io::Error::other(format!(
|
||||
"ec volume {}: journal {} kept changing during merge",
|
||||
vid.0, ecj_path
|
||||
)))
|
||||
}
|
||||
|
||||
/// The disk index of the runtime holding `ecj_path` open, if any: the disk at
|
||||
/// `data_dir`'s own, else the first sibling journaling into it.
|
||||
fn mounted_ec_journal(
|
||||
store: &Store,
|
||||
vid: VolumeId,
|
||||
data_dir: &str,
|
||||
ecj_path: &str,
|
||||
) -> io::Result<Option<usize>> {
|
||||
let owner = store
|
||||
.locations
|
||||
.iter()
|
||||
.position(|loc| Path::new(&loc.directory) == Path::new(data_dir))
|
||||
.ok_or_else(|| {
|
||||
io::Error::other(format!(
|
||||
"ec volume {}: no disk at {} owns journal {}",
|
||||
vid.0, data_dir, ecj_path
|
||||
))
|
||||
})?;
|
||||
let runtime = if store.locations[owner].has_ec_volume(vid) {
|
||||
Some(owner)
|
||||
} else {
|
||||
store.locations.iter().position(|loc| {
|
||||
loc.find_ec_volume(vid)
|
||||
.is_some_and(|ecv| Path::new(&ecv.ecj_file_name()) == Path::new(ecj_path))
|
||||
})
|
||||
};
|
||||
Ok(runtime)
|
||||
}
|
||||
|
||||
/// Publishes merged ids into every other runtime journaling into `ecj_path`.
|
||||
fn publish_to_journal_siblings(
|
||||
store: &Store,
|
||||
primary: usize,
|
||||
vid: VolumeId,
|
||||
ecj_path: &str,
|
||||
ids: &HashSet<NeedleId>,
|
||||
) {
|
||||
for (i, loc) in store.locations.iter().enumerate() {
|
||||
if i == primary {
|
||||
continue;
|
||||
}
|
||||
if let Some(ecv) = loc.find_ec_volume(vid) {
|
||||
if Path::new(&ecv.ecj_file_name()) == Path::new(ecj_path) {
|
||||
ecv.publish_merged_ids(ids);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::types::DiskType;
|
||||
use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo};
|
||||
use tempfile::TempDir;
|
||||
|
||||
const COLLECTION: &str = "c";
|
||||
const VID: VolumeId = VolumeId(9);
|
||||
|
||||
/// A store with one disk per entry of `data`, all sharing `idx` when given,
|
||||
/// else each indexing into its own data dir.
|
||||
fn make_store(tmp: &TempDir, data: &[&str], idx: Option<&str>) -> RwLock<Store> {
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
for d in data {
|
||||
let dir = tmp.path().join(d).to_string_lossy().into_owned();
|
||||
let idx_dir = idx
|
||||
.map(|i| tmp.path().join(i).to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| dir.clone());
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::create_dir_all(&idx_dir).unwrap();
|
||||
store
|
||||
.add_location(
|
||||
&dir,
|
||||
&idx_dir,
|
||||
100,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(0.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
RwLock::new(store)
|
||||
}
|
||||
|
||||
fn dir(tmp: &TempDir, d: &str) -> String {
|
||||
tmp.path().join(d).to_string_lossy().into_owned()
|
||||
}
|
||||
|
||||
fn records(ids: &[u64]) -> Vec<u8> {
|
||||
let ids: Vec<NeedleId> = ids.iter().copied().map(NeedleId).collect();
|
||||
crate::storage::erasure_coding::ecj_merge::encode_ecj_ids(&ids)
|
||||
}
|
||||
|
||||
fn id_set(ids: &[u64]) -> HashSet<NeedleId> {
|
||||
ids.iter().copied().map(NeedleId).collect()
|
||||
}
|
||||
|
||||
/// Shard 0 of `VID` and its `.vif` in `data_dir`.
|
||||
fn write_shard0(data_dir: &str) {
|
||||
let base = format!("{}/{}_{}", data_dir, COLLECTION, VID.0);
|
||||
std::fs::write(format!("{}.ec00", base), b"shard data nonempty").unwrap();
|
||||
let vif = VifVolumeInfo {
|
||||
version: 3,
|
||||
ec_shard_config: Some(VifEcShardConfig {
|
||||
data_shards: 10,
|
||||
parity_shards: 4,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
std::fs::write(
|
||||
format!("{}.vif", base),
|
||||
serde_json::to_string(&vif).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// `VID`'s `.ecx` and a `.ecj` holding `deleted` in `dir`; returns the
|
||||
/// journal path.
|
||||
fn write_index(dir: &str, deleted: &[u64]) -> String {
|
||||
let base = format!("{}/{}_{}", dir, COLLECTION, VID.0);
|
||||
std::fs::write(format!("{}.ecx", base), vec![0u8; 16]).unwrap();
|
||||
let ecj = format!("{}.ecj", base);
|
||||
std::fs::write(&ecj, records(deleted)).unwrap();
|
||||
ecj
|
||||
}
|
||||
|
||||
fn deleted_on(store: &RwLock<Store>, disk: usize, id: u64) -> bool {
|
||||
store.read().unwrap().locations[disk]
|
||||
.find_ec_volume(VID)
|
||||
.expect("mounted")
|
||||
.is_needle_deleted(NeedleId(id))
|
||||
}
|
||||
|
||||
/// Disks sharing one index directory all hold the same journal path.
|
||||
/// Copying shards onto a disk that has not mounted `vid` must still reach
|
||||
/// the sibling runtime holding that journal open.
|
||||
#[test]
|
||||
fn shared_index_dir_reaches_sibling_mount() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let store = make_store(&tmp, &["d0", "d1"], Some("idx"));
|
||||
write_shard0(&dir(&tmp, "d0"));
|
||||
let ecj = write_index(&dir(&tmp, "idx"), &[1]);
|
||||
store.write().unwrap().locations[0]
|
||||
.mount_ec_shards(VID, COLLECTION, &[0], "")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
store.read().unwrap().locations[0]
|
||||
.find_ec_volume(VID)
|
||||
.unwrap()
|
||||
.ecj_file_name(),
|
||||
ecj
|
||||
);
|
||||
|
||||
let added =
|
||||
merge_ec_journal(&store, VID, &dir(&tmp, "d1"), &ecj, &id_set(&[1, 2])).unwrap();
|
||||
assert_eq!(added, 1);
|
||||
assert!(
|
||||
deleted_on(&store, 0, 2),
|
||||
"the mounted sibling must see id 2"
|
||||
);
|
||||
assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2]));
|
||||
}
|
||||
|
||||
/// Every runtime holding the journal open must see merged ids in memory.
|
||||
#[test]
|
||||
fn shared_journal_reaches_every_holder() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let store = make_store(&tmp, &["d0", "d1"], Some("idx"));
|
||||
write_shard0(&dir(&tmp, "d0"));
|
||||
write_shard0(&dir(&tmp, "d1"));
|
||||
let ecj = write_index(&dir(&tmp, "idx"), &[1]);
|
||||
for i in 0..2 {
|
||||
store.write().unwrap().locations[i]
|
||||
.mount_ec_shards(VID, COLLECTION, &[0], "")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let added =
|
||||
merge_ec_journal(&store, VID, &dir(&tmp, "d1"), &ecj, &id_set(&[1, 2])).unwrap();
|
||||
assert_eq!(added, 1);
|
||||
assert!(
|
||||
deleted_on(&store, 0, 2) && deleted_on(&store, 1, 2),
|
||||
"every journal holder must see the merged id"
|
||||
);
|
||||
assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2]));
|
||||
}
|
||||
|
||||
/// The picked runtime may journal to a different file than the copied
|
||||
/// one — its index lives in its data directory while a sibling's lives
|
||||
/// in the index directory. The ids must be published only to holders of
|
||||
/// the file they were written to.
|
||||
#[test]
|
||||
fn publishes_to_actual_journal_holders() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let store = make_store(&tmp, &["d0", "d1"], Some("idx"));
|
||||
write_shard0(&dir(&tmp, "d0"));
|
||||
write_shard0(&dir(&tmp, "d1"));
|
||||
let data_ecj = write_index(&dir(&tmp, "d0"), &[1]);
|
||||
let idx_ecj = write_index(&dir(&tmp, "idx"), &[1]);
|
||||
for i in 0..2 {
|
||||
store.write().unwrap().locations[i]
|
||||
.mount_ec_shards(VID, COLLECTION, &[0], "")
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
store.read().unwrap().locations[0]
|
||||
.find_ec_volume(VID)
|
||||
.unwrap()
|
||||
.ecj_file_name(),
|
||||
data_ecj
|
||||
);
|
||||
|
||||
let added =
|
||||
merge_ec_journal(&store, VID, &dir(&tmp, "d0"), &idx_ecj, &id_set(&[1, 2])).unwrap();
|
||||
assert_eq!(added, 1);
|
||||
assert_eq!(std::fs::read(&data_ecj).unwrap(), records(&[1, 2]));
|
||||
assert_eq!(std::fs::read(&idx_ecj).unwrap(), records(&[1]));
|
||||
assert!(deleted_on(&store, 0, 2));
|
||||
assert!(
|
||||
!deleted_on(&store, 1, 2),
|
||||
"a different journal's holder must not claim the merged id"
|
||||
);
|
||||
}
|
||||
|
||||
/// A sibling disk can mount `vid` from the receiving disk's index (#9212)
|
||||
/// while the merge reads the journal unlocked. The merge must go through
|
||||
/// that mount and report what it added.
|
||||
#[test]
|
||||
fn mount_during_read_is_merged_through_and_counted() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let store = make_store(&tmp, &["d0", "d1"], None);
|
||||
let owner = dir(&tmp, "d0");
|
||||
let ecj = write_index(&owner, &[1]);
|
||||
write_shard0(&dir(&tmp, "d1"));
|
||||
|
||||
let mut mounted = false;
|
||||
let read = |path: &str| {
|
||||
let read = read_ecj_ids(path);
|
||||
if !mounted {
|
||||
store.write().unwrap().locations[1]
|
||||
.mount_ec_shards_with_idx_dir(VID, COLLECTION, &[0], &owner, "")
|
||||
.unwrap();
|
||||
mounted = true;
|
||||
}
|
||||
read
|
||||
};
|
||||
let added =
|
||||
merge_ec_journal_with(&store, VID, &owner, &ecj, &id_set(&[1, 2, 3]), read).unwrap();
|
||||
assert_eq!(added, 2, "the ids merged through the new mount are counted");
|
||||
assert!(deleted_on(&store, 1, 2) && deleted_on(&store, 1, 3));
|
||||
assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2, 3]));
|
||||
}
|
||||
|
||||
/// An unmounted journal is merged on disk and a repeat adds nothing; a
|
||||
/// data dir that is no disk is refused.
|
||||
#[test]
|
||||
fn unmounted_journal_is_idempotent() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let store = make_store(&tmp, &["d0"], Some("idx"));
|
||||
let ecj = write_index(&dir(&tmp, "idx"), &[1, 2]);
|
||||
let d0 = dir(&tmp, "d0");
|
||||
for want in [2, 0, 0] {
|
||||
let added = merge_ec_journal(&store, VID, &d0, &ecj, &id_set(&[2, 3, 4])).unwrap();
|
||||
assert_eq!(added, want);
|
||||
}
|
||||
assert_eq!(std::fs::read(&ecj).unwrap(), records(&[1, 2, 3, 4]));
|
||||
assert!(
|
||||
merge_ec_journal(&store, VID, &dir(&tmp, "elsewhere"), &ecj, &id_set(&[1])).is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,232 +0,0 @@
|
||||
//! Persisted volume-id high-water mark ("vid stamp"), mirroring Go's
|
||||
//! `weed/storage/vid_stamp.go`.
|
||||
//!
|
||||
//! Each volume dir carries a `.vid_stamp` JSON file with the highest volume
|
||||
//! id the server has ever observed plus the topology id it last heartbeated
|
||||
//! to. The bound is fed by heartbeat echoes, AllocateVolume, and local
|
||||
//! volume/EC creation, is reported back in `Heartbeat.max_volume_id_seen`,
|
||||
//! and survives master state loss so rebuilt masters cannot reissue ids an
|
||||
//! offline server still holds.
|
||||
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::storage::types::VolumeId;
|
||||
|
||||
use super::store::Store;
|
||||
|
||||
const VID_STAMP_FILE_NAME: &str = ".vid_stamp";
|
||||
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
struct VidStamp {
|
||||
#[serde(rename = "maxVolumeId")]
|
||||
max_volume_id: u32,
|
||||
#[serde(rename = "topologyId", default, skip_serializing_if = "String::is_empty")]
|
||||
topology_id: String,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
pub fn max_volume_id_seen(&self) -> u32 {
|
||||
self.max_volume_id_seen.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn last_seen_topology_id(&self) -> String {
|
||||
self.last_seen_topology_id
|
||||
.lock()
|
||||
.unwrap()
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub fn update_max_volume_id_seen(&self, vid: u32) {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
let prev = self.max_volume_id_seen.fetch_max(vid, Relaxed);
|
||||
if vid > prev || self.vid_stamp_dirty.load(Relaxed) {
|
||||
self.save_vid_stamps();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_last_seen_topology_id(&self, id: &str) {
|
||||
if id.is_empty() {
|
||||
return;
|
||||
}
|
||||
{
|
||||
let mut guard = self.last_seen_topology_id.lock().unwrap();
|
||||
if *guard == id {
|
||||
return;
|
||||
}
|
||||
*guard = id.to_string();
|
||||
}
|
||||
self.save_vid_stamps();
|
||||
}
|
||||
|
||||
/// Seed the bound from a freshly added location: its persisted stamp,
|
||||
/// its volumes, and its EC shards. Called per location from
|
||||
/// `add_location` after the EC reconcile passes have run.
|
||||
pub(crate) fn fold_location_into_vid_stamp(&self, loc: &super::disk_location::DiskLocation) {
|
||||
if let Ok(stamp) = read_vid_stamp(Path::new(&loc.directory)) {
|
||||
self.update_max_volume_id_seen(stamp.max_volume_id);
|
||||
if !stamp.topology_id.is_empty() {
|
||||
self.update_last_seen_topology_id(&stamp.topology_id);
|
||||
}
|
||||
}
|
||||
let mut max_vid: u32 = 0;
|
||||
for vid in loc.volume_ids() {
|
||||
max_vid = max_vid.max(vid.0);
|
||||
}
|
||||
for (vid, _) in loc.ec_volumes() {
|
||||
max_vid = max_vid.max(vid.0);
|
||||
}
|
||||
if max_vid > 0 {
|
||||
self.update_max_volume_id_seen(max_vid);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn save_vid_stamps(&self) {
|
||||
let _guard = self.vid_stamp_lock.lock().unwrap();
|
||||
let stamp = match serde_json::to_vec(&VidStamp {
|
||||
max_volume_id: self.max_volume_id_seen(),
|
||||
topology_id: self.last_seen_topology_id(),
|
||||
}) {
|
||||
Ok(s) => s,
|
||||
Err(_) => return,
|
||||
};
|
||||
let mut failed = false;
|
||||
for loc in &self.locations {
|
||||
if let Err(e) = write_vid_stamp(Path::new(&loc.directory), &stamp) {
|
||||
tracing::warn!("write vid stamp in {}: {}", loc.directory, e);
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
self.vid_stamp_dirty
|
||||
.store(failed, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn read_vid_stamp(dir: &Path) -> io::Result<VidStamp> {
|
||||
let data = fs::read(dir.join(VID_STAMP_FILE_NAME))?;
|
||||
serde_json::from_slice(&data).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
|
||||
}
|
||||
|
||||
fn write_vid_stamp(dir: &Path, stamp: &[u8]) -> io::Result<()> {
|
||||
let tmp = dir.join(format!("{}.tmp", VID_STAMP_FILE_NAME));
|
||||
{
|
||||
use std::io::Write;
|
||||
let mut f = fs::File::create(&tmp)?;
|
||||
f.write_all(stamp)?;
|
||||
f.sync_all()?;
|
||||
}
|
||||
fs::rename(&tmp, dir.join(VID_STAMP_FILE_NAME))?;
|
||||
if cfg!(unix) {
|
||||
fs::File::open(dir)?.sync_all()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stamps the volume id of a newly created volume or EC shard set. Called
|
||||
/// from `Store::add_volume` / `Store::mount_ec_shards*` so the bound covers
|
||||
/// ids before their files exist.
|
||||
impl Store {
|
||||
pub(crate) fn stamp_allocated_vid(&self, vid: VolumeId) {
|
||||
self.update_max_volume_id_seen(vid.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::MinFreeSpace;
|
||||
use crate::storage::needle_map::NeedleMapKind;
|
||||
use crate::storage::types::DiskType;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn make_store(dir: &str) -> Store {
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(dir, dir, 10, DiskType::HardDrive, MinFreeSpace::Percent(1.0), Vec::new())
|
||||
.unwrap();
|
||||
store
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stamp_written_on_bound_increase() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let store = make_store(dir);
|
||||
|
||||
store.update_max_volume_id_seen(42);
|
||||
|
||||
let stamp = read_vid_stamp(tmp.path()).unwrap();
|
||||
assert_eq!(stamp.max_volume_id, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stamp_survives_restart() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
|
||||
let store = make_store(dir);
|
||||
store.update_max_volume_id_seen(42);
|
||||
store.update_last_seen_topology_id("cluster-a");
|
||||
drop(store);
|
||||
|
||||
let restarted = make_store(dir);
|
||||
assert_eq!(restarted.max_volume_id_seen(), 42);
|
||||
assert_eq!(restarted.last_seen_topology_id(), "cluster-a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn later_disk_gets_combined_stamp() {
|
||||
let tmp1 = TempDir::new().unwrap();
|
||||
let tmp2 = TempDir::new().unwrap();
|
||||
// dir2 carries a stale stamp from an older run
|
||||
write_vid_stamp(
|
||||
tmp2.path(),
|
||||
&serde_json::to_vec(&VidStamp {
|
||||
max_volume_id: 10,
|
||||
topology_id: String::new(),
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
tmp1.path().to_str().unwrap(),
|
||||
tmp1.path().to_str().unwrap(),
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
store.update_max_volume_id_seen(100);
|
||||
store
|
||||
.add_location(
|
||||
tmp2.path().to_str().unwrap(),
|
||||
tmp2.path().to_str().unwrap(),
|
||||
10,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let stamp2 = read_vid_stamp(tmp2.path()).unwrap();
|
||||
assert_eq!(stamp2.max_volume_id, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bound_is_monotonic() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let store = make_store(dir);
|
||||
|
||||
store.update_max_volume_id_seen(10);
|
||||
store.update_max_volume_id_seen(5);
|
||||
assert_eq!(store.max_volume_id_seen(), 10);
|
||||
}
|
||||
}
|
||||
+955
-2077
File diff suppressed because it is too large
Load Diff
@@ -142,7 +142,7 @@ mod tests {
|
||||
let old_idx = old_idx_dir.to_str().unwrap();
|
||||
let new_idx = new_idx_dir.to_str().unwrap();
|
||||
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
data,
|
||||
old_idx,
|
||||
VolumeId(1),
|
||||
@@ -196,7 +196,7 @@ mod tests {
|
||||
let root = TempDir::new().unwrap();
|
||||
let dir = root.path().to_str().unwrap();
|
||||
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
@@ -242,7 +242,7 @@ mod tests {
|
||||
let root = TempDir::new().unwrap();
|
||||
let dir = root.path().to_str().unwrap();
|
||||
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
@@ -291,7 +291,7 @@ mod tests {
|
||||
let root = TempDir::new().unwrap();
|
||||
let dir = root.path().to_str().unwrap();
|
||||
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(1),
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::storage::idx;
|
||||
use crate::storage::needle::Needle;
|
||||
use crate::storage::needle::needle::needle_body_length;
|
||||
use crate::storage::types::*;
|
||||
use crate::storage::volume::{Volume, VolumeError, VolumeIo, fsync_dir};
|
||||
use crate::storage::volume::{Volume, VolumeError, fsync_dir};
|
||||
|
||||
/// Needles found in the head of .dat, keyed by id, plus the ids in .dat order.
|
||||
type DatHeadNeedles = (HashMap<NeedleId, (Offset, Size)>, Vec<NeedleId>);
|
||||
@@ -45,15 +45,15 @@ impl Volume {
|
||||
/// restores .idx to .dat append order, which several readers lean on: the
|
||||
/// fingerprint is gone so a later load stops after one row, and the last row
|
||||
/// is the .dat-tail needle again.
|
||||
pub(crate) fn repair_idx_head_tombstones(&self, io: &VolumeIo) -> Result<usize, VolumeError> {
|
||||
if !self.has_data_backend_io(io) {
|
||||
pub(crate) fn repair_idx_head_tombstones(&self) -> Result<usize, VolumeError> {
|
||||
if !self.has_data_backend() {
|
||||
return Ok(0);
|
||||
}
|
||||
let version = self.version();
|
||||
if !version.is_supported() {
|
||||
return Ok(0);
|
||||
}
|
||||
let first_needle_offset = self.super_block().block_size() as i64;
|
||||
let first_needle_offset = self.super_block.block_size() as i64;
|
||||
let idx_path = self.file_name(".idx");
|
||||
|
||||
let clobbered = idx_head_tombstone_count(&idx_path)?;
|
||||
@@ -68,7 +68,7 @@ impl Volume {
|
||||
"idx starts with offset-0 tombstones, recovering the rows they overwrote from .dat"
|
||||
);
|
||||
|
||||
let (mut lost, order) = self.scan_dat_head(io, version, first_needle_offset, clobbered)?;
|
||||
let (mut lost, order) = self.scan_dat_head(version, first_needle_offset, clobbered)?;
|
||||
drop_indexed_keys(&idx_path, &mut lost)?;
|
||||
if lost.is_empty() {
|
||||
return Ok(0);
|
||||
@@ -93,7 +93,6 @@ impl Volume {
|
||||
/// being resurrected.
|
||||
fn scan_dat_head(
|
||||
&self,
|
||||
io: &VolumeIo,
|
||||
version: Version,
|
||||
first_needle_offset: i64,
|
||||
limit: usize,
|
||||
@@ -104,7 +103,7 @@ impl Volume {
|
||||
|
||||
for _ in 0..limit {
|
||||
let mut header = [0u8; NEEDLE_HEADER_SIZE];
|
||||
match self.read_exact_at_backend(io, &mut header, offset as u64) {
|
||||
match self.read_exact_at_backend(&mut header, offset as u64) {
|
||||
Ok(()) => {}
|
||||
Err(VolumeError::Io(ref e)) if e.kind() == io::ErrorKind::UnexpectedEof => break,
|
||||
Err(e) => return Err(e),
|
||||
@@ -246,7 +245,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn write_test_volume(dir: &str, needle_count: u64) -> Vec<Vec<u8>> {
|
||||
let v = open_volume(dir);
|
||||
let mut v = open_volume(dir);
|
||||
let mut written = Vec::new();
|
||||
for i in 1..=needle_count {
|
||||
let data = format!("needle-{}-payload", i).into_bytes();
|
||||
@@ -356,7 +355,7 @@ mod tests {
|
||||
let rows = idx_rows(&idx_path);
|
||||
assert_eq!(
|
||||
rows[0].1,
|
||||
v.super_block().block_size() as i64,
|
||||
v.super_block.block_size() as i64,
|
||||
"first row does not index the first needle in .dat"
|
||||
);
|
||||
for i in 1..4 {
|
||||
@@ -381,7 +380,7 @@ mod tests {
|
||||
let idx_path = format!("{}/1.idx", dir);
|
||||
|
||||
write_test_volume(dir, 6);
|
||||
let v = open_volume(dir);
|
||||
let mut v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(3),
|
||||
..Needle::default()
|
||||
@@ -416,7 +415,7 @@ mod tests {
|
||||
let idx_path = format!("{}/1.idx", dir);
|
||||
|
||||
write_test_volume(dir, 8);
|
||||
let v = open_volume(dir);
|
||||
let mut v = open_volume(dir);
|
||||
v.delete_needle(&mut Needle {
|
||||
id: NeedleId(2),
|
||||
..Needle::default()
|
||||
@@ -457,10 +456,8 @@ mod tests {
|
||||
drop(dat);
|
||||
|
||||
let v = open_volume(dir);
|
||||
let first = v.super_block().block_size() as i64;
|
||||
let err = v
|
||||
.scan_dat_head(&v.io_read_for_test(), v.version(), first, 10)
|
||||
.unwrap_err();
|
||||
let first = v.super_block.block_size() as i64;
|
||||
let err = v.scan_dat_head(v.version(), first, 10).unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, VolumeError::Io(e) if e.kind() == io::ErrorKind::InvalidData),
|
||||
"expected a corrupt-data error, got {err:?}"
|
||||
|
||||
@@ -107,12 +107,7 @@ fn build_test_state(
|
||||
rack: String::new(),
|
||||
file_size_limit_bytes: 0,
|
||||
maintenance_byte_per_second: 0,
|
||||
vacuum_commit_slots: None,
|
||||
vacuum_commits_waiting: std::sync::atomic::AtomicU32::new(0),
|
||||
is_heartbeating: std::sync::atomic::AtomicBool::new(true),
|
||||
ec_decodes_in_flight: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail: std::sync::Mutex::new(std::collections::HashSet::new()),
|
||||
ec_decode_tail_notify: tokio::sync::Notify::new(),
|
||||
has_master: false,
|
||||
pre_stop_seconds: 0,
|
||||
volume_state_notify: tokio::sync::Notify::new(),
|
||||
@@ -511,7 +506,7 @@ async fn write_and_delete_on_read_only_volume_say_is_read_only() {
|
||||
.store
|
||||
.write()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.find_volume_mut(VolumeId(1))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_no_write_or_delete(true);
|
||||
@@ -539,7 +534,7 @@ async fn write_and_delete_on_read_only_volume_say_is_read_only() {
|
||||
.store
|
||||
.write()
|
||||
.unwrap()
|
||||
.find_volume(VolumeId(1))
|
||||
.find_volume_mut(VolumeId(1))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_no_write_or_delete(false);
|
||||
@@ -1044,7 +1039,7 @@ async fn chunk_manifest_expands_chunk_stored_on_ec_volume() {
|
||||
// Build regular volume 2 holding the chunk, then EC-encode it and mount all
|
||||
// 14 shards locally so reconstruct-on-read is a pure local read.
|
||||
{
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(2),
|
||||
@@ -1084,7 +1079,7 @@ async fn chunk_manifest_expands_chunk_stored_on_ec_volume() {
|
||||
let manifest_nid = NeedleId(0x7);
|
||||
let manifest_cookie = Cookie(0x55667788);
|
||||
{
|
||||
let store = state.store.read().unwrap();
|
||||
let mut store = state.store.write().unwrap();
|
||||
let mut n = Needle {
|
||||
id: manifest_nid,
|
||||
cookie: manifest_cookie,
|
||||
@@ -1148,7 +1143,7 @@ async fn delete_on_ec_volume_succeeds_when_the_needles_shard_is_not_mounted() {
|
||||
// Build regular volume 4, then EC-encode it. The volume is standalone and
|
||||
// never registered in the store, so afterwards it exists only as shards.
|
||||
{
|
||||
let v = Volume::new(
|
||||
let mut v = Volume::new(
|
||||
dir,
|
||||
dir,
|
||||
VolumeId(4),
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
//! Issue #791: the shared `RwLock<Store>` guards the volume map only.
|
||||
//! `find_volume` hands out an `Arc<Volume>` so needle I/O runs without the
|
||||
//! guard; a handle that outlives its mount must fail closed rather than I/O
|
||||
//! a torn-down volume — the same end Go reaches through nil needle-map/dat
|
||||
//! checks under the per-volume dataFileAccessLock.
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use seaweed_volume::config::MinFreeSpace;
|
||||
use seaweed_volume::storage::needle::needle::Needle;
|
||||
use seaweed_volume::storage::needle_map::NeedleMapKind;
|
||||
use seaweed_volume::storage::store::Store;
|
||||
use seaweed_volume::storage::types::{DiskType, NeedleId, VolumeId};
|
||||
use seaweed_volume::storage::volume::VolumeSpec;
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn store_with_two_volumes() -> (Arc<RwLock<Store>>, TempDir) {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let dir = tmp.path().to_str().unwrap();
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(dir, dir, 10, DiskType::HardDrive, MinFreeSpace::Percent(1.0), Vec::new())
|
||||
.unwrap();
|
||||
store
|
||||
.add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
store
|
||||
.add_volume(VolumeId(2), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
(Arc::new(RwLock::new(store)), tmp)
|
||||
}
|
||||
|
||||
fn needle(id: u64) -> Needle {
|
||||
Needle {
|
||||
id: NeedleId(id),
|
||||
cookie: 0x1234.into(),
|
||||
data: vec![7u8; 16],
|
||||
data_size: 16,
|
||||
..Needle::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_read_guard_admits_cross_volume_io() {
|
||||
let (store, _tmp) = store_with_two_volumes();
|
||||
|
||||
// Read-guarded callers can write on different volumes concurrently —
|
||||
// what used to require store.write() now only resolves a handle.
|
||||
let t = {
|
||||
let store = store.clone();
|
||||
std::thread::spawn(move || {
|
||||
let store = store.read().unwrap();
|
||||
store
|
||||
.write_volume_needle(VolumeId(1), &mut needle(1), false)
|
||||
.unwrap();
|
||||
})
|
||||
};
|
||||
{
|
||||
let store = store.read().unwrap();
|
||||
store
|
||||
.write_volume_needle(VolumeId(2), &mut needle(9), false)
|
||||
.unwrap();
|
||||
}
|
||||
t.join().unwrap();
|
||||
|
||||
let store = store.read().unwrap();
|
||||
let mut n = needle(9);
|
||||
n.data.clear();
|
||||
assert_eq!(store.read_volume_needle(VolumeId(2), &mut n).unwrap(), 16);
|
||||
let mut n = needle(1);
|
||||
n.data.clear();
|
||||
assert_eq!(store.read_volume_needle(VolumeId(1), &mut n).unwrap(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_volume_handle_fails_after_unmount() {
|
||||
let (store, _tmp) = store_with_two_volumes();
|
||||
|
||||
// A handle resolved before unmount keeps the Volume alive but must not
|
||||
// be able to write into its closed files.
|
||||
let (_, vol1) = store.read().unwrap().find_volume(VolumeId(1)).unwrap();
|
||||
store.write().unwrap().unmount_volume(VolumeId(1)).unwrap();
|
||||
|
||||
assert!(vol1.write_needle(&mut needle(1), true, false).is_err());
|
||||
let mut n = needle(1);
|
||||
n.data.clear();
|
||||
assert!(vol1.read_needle(&mut n).is_err());
|
||||
}
|
||||
Generated
+5
-14
@@ -485,7 +485,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"untrusted 0.7.1",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -5125,7 +5124,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"getrandom 0.2.17",
|
||||
"libc",
|
||||
"untrusted 0.9.0",
|
||||
"untrusted",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
@@ -5189,9 +5188,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
version = "0.23.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -5261,7 +5260,7 @@ dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted 0.9.0",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5343,9 +5342,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"prometheus",
|
||||
"prost",
|
||||
"prost-types",
|
||||
@@ -5784,7 +5781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -6216,12 +6213,6 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
`weed/pb/plugin.proto` is a language-agnostic contract: a maintenance worker
|
||||
connects out to admin, announces the job types it can detect and execute, and
|
||||
answers requests on that one stream. `weed worker -admin=host:12535` is the Go
|
||||
answers requests on that one stream. `weed worker -admin=host:23646` is the Go
|
||||
implementation of it from outside the admin process. This workspace is the Rust
|
||||
one.
|
||||
|
||||
@@ -35,7 +35,7 @@ binary for the runner's platform, resolved from the version in `Cargo.lock`.
|
||||
|
||||
## Running
|
||||
|
||||
cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535
|
||||
cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646
|
||||
|
||||
The admin's *HTTP* address is what an operator has; the gRPC port is derived
|
||||
from it the way the Go side does. Dialling the HTTP port fails as "frame with
|
||||
@@ -49,7 +49,7 @@ Released builds do not need a toolchain. The worker ships inside the SeaweedFS
|
||||
image, beside the Rust volume server, under the verb that mirrors
|
||||
`volume-rust`:
|
||||
|
||||
docker run chrislusf/seaweedfs worker-rust --admin admin:12535
|
||||
docker run chrislusf/seaweedfs worker-rust --admin admin:23646
|
||||
|
||||
and as `weed-worker_linux_{amd64,arm64}.tar.gz` on each GitHub release. Both are
|
||||
linux amd64/arm64 only — lance, arrow and datafusion make every extra target an
|
||||
@@ -59,7 +59,7 @@ entrypoint says so rather than failing as "not found".
|
||||
|
||||
## Metrics
|
||||
|
||||
cargo run -p weed-lance-worker -- --admin 127.0.0.1:12535 --metrics-port 9328
|
||||
cargo run -p weed-lance-worker -- --admin 127.0.0.1:23646 --metrics-port 9328
|
||||
|
||||
Serves `/health`, `/ready` and `/metrics` on that port, the same three the Go
|
||||
worker serves under `weed worker -metricsPort`, so one scrape config covers
|
||||
|
||||
@@ -12,10 +12,6 @@ name = "seaweed_worker_core"
|
||||
# Helpers the Rust volume server needs as well. A path dependency because
|
||||
# the two trees are separate cargo workspaces with no common root manifest.
|
||||
seaweed-common = { path = "../../../seaweed-common" }
|
||||
# Already in the graph through tonic's `tls-aws-lc` feature; named here so the
|
||||
# admin token can sign with the same HMAC the Go worker uses.
|
||||
aws-lc-rs = "1"
|
||||
base64 = "0.22"
|
||||
anyhow.workspace = true
|
||||
async-trait.workspace = true
|
||||
prost.workspace = true
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::time::Duration;
|
||||
/// How one worker process connects and how much work it will take on.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WorkerOptions {
|
||||
/// Admin gRPC address, e.g. "localhost:12535".
|
||||
/// Admin gRPC address, e.g. "localhost:23646".
|
||||
pub admin_address: String,
|
||||
pub worker_id: String,
|
||||
pub worker_version: String,
|
||||
@@ -17,15 +17,6 @@ pub struct WorkerOptions {
|
||||
/// section of security.toml. None means plaintext, which is the default the
|
||||
/// Go worker also takes when no certificates are configured.
|
||||
pub tls: Option<TlsOptions>,
|
||||
/// The shared secret the admin stream requires when the cluster configures
|
||||
/// `jwt.filer_signing.key`; mirrored from that config value. Empty means no
|
||||
/// token is attached, which only works against admins that either bound
|
||||
/// their worker listener to loopback or authenticate with mTLS instead.
|
||||
pub signing_key: Option<String>,
|
||||
/// Seconds the minted admin token lives, mirroring
|
||||
/// `jwt.filer_signing.expires_after_seconds`. Zero mints a token without an
|
||||
/// expiry, matching the Go default.
|
||||
pub signing_key_expires_after_seconds: u64,
|
||||
}
|
||||
|
||||
/// Certificates for the control stream. All three are required together: the
|
||||
@@ -44,7 +35,7 @@ pub struct TlsOptions {
|
||||
impl Default for WorkerOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
admin_address: "localhost:12535".to_string(),
|
||||
admin_address: "localhost:23646".to_string(),
|
||||
worker_id: String::new(),
|
||||
worker_version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
worker_address: String::new(),
|
||||
@@ -53,8 +44,6 @@ impl Default for WorkerOptions {
|
||||
max_detection_concurrency: 1,
|
||||
max_execution_concurrency: 1,
|
||||
tls: None,
|
||||
signing_key: None,
|
||||
signing_key_expires_after_seconds: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! `weed/pb/plugin.proto` is a language-agnostic gRPC stream: a worker connects
|
||||
//! out to admin, announces the job types it can detect and execute, and then
|
||||
//! answers requests on that one stream. `weed worker -admin=host:12535` is the
|
||||
//! answers requests on that one stream. `weed worker -admin=host:23646` is the
|
||||
//! Go implementation of the same contract from outside the admin process; this
|
||||
//! is the Rust one, and it knows nothing about any particular job.
|
||||
|
||||
@@ -13,7 +13,6 @@ pub mod metrics;
|
||||
pub mod registry;
|
||||
pub mod senders;
|
||||
pub mod stream;
|
||||
pub mod token;
|
||||
|
||||
/// Generated plugin.proto types.
|
||||
pub mod pb {
|
||||
|
||||
@@ -176,19 +176,10 @@ async fn serve_once(
|
||||
metadata: Default::default(),
|
||||
}))?;
|
||||
|
||||
let mut request = tonic::Request::new(UnboundedReceiverStream::new(rx));
|
||||
if let Some(token) = crate::token::admin_token(
|
||||
options.signing_key.as_deref().unwrap_or("").as_bytes(),
|
||||
options.signing_key_expires_after_seconds,
|
||||
) {
|
||||
request.metadata_mut().insert(
|
||||
"authorization",
|
||||
format!("Bearer {token}")
|
||||
.parse()
|
||||
.context("minted admin token is not valid metadata")?,
|
||||
);
|
||||
}
|
||||
let mut inbound = client.worker_stream(request).await?.into_inner();
|
||||
let mut inbound = client
|
||||
.worker_stream(UnboundedReceiverStream::new(rx))
|
||||
.await?
|
||||
.into_inner();
|
||||
|
||||
let heartbeat = spawn_heartbeat(
|
||||
sender.clone(),
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use aws_lc_rs::hmac;
|
||||
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
|
||||
|
||||
/// Mints the HS256 Bearer token the admin control stream requires when
|
||||
/// jwt.filer_signing.key is configured, matching the SeaweedFilerAdminClaims
|
||||
/// the Go worker sends: registered claims only, no session id.
|
||||
pub fn admin_token(signing_key: &[u8], expires_after_secs: u64) -> Option<String> {
|
||||
if signing_key.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let claims = if expires_after_secs > 0 {
|
||||
let exp = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or_default()
|
||||
+ expires_after_secs;
|
||||
format!("{{\"exp\":{exp}}}")
|
||||
} else {
|
||||
"{}".to_string()
|
||||
};
|
||||
let signing_input = format!(
|
||||
"{}.{}",
|
||||
URL_SAFE_NO_PAD.encode(b"{\"alg\":\"HS256\",\"typ\":\"JWT\"}"),
|
||||
URL_SAFE_NO_PAD.encode(claims.as_bytes())
|
||||
);
|
||||
let key = hmac::Key::new(hmac::HMAC_SHA256, signing_key);
|
||||
let signature = hmac::sign(&key, signing_input.as_bytes());
|
||||
Some(format!(
|
||||
"{signing_input}.{}",
|
||||
URL_SAFE_NO_PAD.encode(signature.as_ref())
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::admin_token;
|
||||
|
||||
#[test]
|
||||
fn no_key_no_token() {
|
||||
assert!(admin_token(b"", 0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minted_token_verifies_shape() {
|
||||
let token = admin_token(b"secret", 60).unwrap();
|
||||
let parts: Vec<&str> = token.split('.').collect();
|
||||
assert_eq!(parts.len(), 3);
|
||||
assert!(!parts.iter().any(|p| p.is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_expiry_has_empty_claims() {
|
||||
let token = admin_token(b"secret", 0).unwrap();
|
||||
let payload = token.split('.').nth(1).unwrap();
|
||||
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
|
||||
let claims = URL_SAFE_NO_PAD.decode(payload).unwrap();
|
||||
assert_eq!(claims, b"{}");
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ use weed_lance_worker::metrics::LanceMetrics;
|
||||
#[command(name = "weed-worker", about = "SeaweedFS maintenance worker")]
|
||||
struct Args {
|
||||
/// Admin server gRPC address.
|
||||
#[arg(long, default_value = "localhost:12535", env = "WEED_ADMIN")]
|
||||
#[arg(long, default_value = "localhost:23646", env = "WEED_ADMIN")]
|
||||
admin: String,
|
||||
|
||||
/// Worker identity reported to admin.
|
||||
@@ -61,18 +61,6 @@ struct Args {
|
||||
#[arg(long)]
|
||||
tls_server_name: Option<String>,
|
||||
|
||||
/// Shared secret for the admin control stream, the `jwt.filer_signing.key`
|
||||
/// value the Go worker reads from its config file. Required by admins that
|
||||
/// authenticate the worker stream with a signing key.
|
||||
#[arg(long, env = "WEED_FILER_SIGNING_KEY")]
|
||||
filer_signing_key: Option<String>,
|
||||
|
||||
/// Lifetime of the token minted from the signing key, mirroring
|
||||
/// `jwt.filer_signing.expires_after_seconds`. Zero mints a token with no
|
||||
/// expiry.
|
||||
#[arg(long, default_value = "0")]
|
||||
filer_signing_key_expires_after_seconds: u64,
|
||||
|
||||
/// Serve /health, /ready and /metrics on this port, the way
|
||||
/// `weed worker -metricsPort` does. Zero, the default, serves nothing.
|
||||
#[arg(long, default_value = "0", env = "WEED_METRICS_PORT")]
|
||||
@@ -143,8 +131,6 @@ async fn main() -> Result<()> {
|
||||
max_detection_concurrency: args.max_concurrency,
|
||||
max_execution_concurrency: args.max_concurrency,
|
||||
tls,
|
||||
signing_key: args.filer_signing_key,
|
||||
signing_key_expires_after_seconds: args.filer_signing_key_expires_after_seconds,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Cluster security group + the SeaweedFS port matrix.
|
||||
# Intra-cluster: all TCP allowed within the SG (master 9333/19333, volume
|
||||
# 8080/18080, filer 8888/18888, admin 22535, ...). Client-facing: only S3 (8333)
|
||||
# 8080/18080, filer 8888/18888, admin 33646, ...). Client-facing: only S3 (8333)
|
||||
# and filer (8888). Metrics ports (9327) are never opened to clients.
|
||||
|
||||
resource "aws_security_group" "cluster" {
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
AdminUrl = "http://localhost:12535"
|
||||
AdminUrl = "http://localhost:23646"
|
||||
MasterUrl = "http://localhost:9333"
|
||||
FilerUrl = "http://localhost:8888"
|
||||
)
|
||||
@@ -147,13 +147,13 @@ func ensureEnvironment(t *testing.T) {
|
||||
debugPort := 6060 + i
|
||||
dir, _ := filepath.Abs(filepath.Join("tmp", workerName))
|
||||
os.MkdirAll(dir, 0755)
|
||||
startWeed(t, workerName, "worker", "-admin=localhost:12535", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort))
|
||||
startWeed(t, workerName, "worker", "-admin=localhost:23646", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort))
|
||||
}
|
||||
|
||||
// 6. Start Admin
|
||||
os.RemoveAll(filepath.Join("tmp", "admin"))
|
||||
os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
|
||||
startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=12535", "-dataDir=./tmp/admin")
|
||||
startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
|
||||
// Admin is started after master, 14 volume servers, filer and 2 workers,
|
||||
// so under cold CI conditions the wait here has to absorb the tail of
|
||||
// every earlier subprocess coming up. 60s is too tight and has flaked;
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_dlm
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build linux || darwin
|
||||
|
||||
package fuse_p2p
|
||||
|
||||
import (
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<spark.version>3.5.8</spark.version>
|
||||
<spark.version>3.5.7</spark.version>
|
||||
<hadoop.version>3.3.6</hadoop.version>
|
||||
<scala.binary.version>2.12</scala.binary.version>
|
||||
<junit.version>4.13.2</junit.version>
|
||||
|
||||
+10
-7
@@ -7,20 +7,20 @@ require (
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
|
||||
github.com/segmentio/kafka-go v0.4.49
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/stretchr/testify v1.11.1
|
||||
google.golang.org/grpc v1.85.0-dev.0.20260915183914-4e49413dcab7
|
||||
)
|
||||
|
||||
replace github.com/seaweedfs/seaweedfs => ../../
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.2.3 // indirect
|
||||
github.com/andybalholm/brotli v1.2.2 // indirect
|
||||
github.com/aws/aws-sdk-go v1.55.8 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cognusion/imaging v1.0.4 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dustin/go-humanize v1.1.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/eapache/go-resiliency v1.7.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
@@ -61,15 +61,17 @@ require (
|
||||
github.com/petermattis/goid v0.0.0-20260113132338-7c7de50cc741 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.29 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/prometheus/client_golang v1.24.1 // indirect
|
||||
github.com/prometheus/client_model v0.6.3 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
github.com/prometheus/common v0.70.1 // indirect
|
||||
github.com/prometheus/procfs v0.22.0 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
|
||||
github.com/rdleal/intervalst v1.5.0 // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3 // indirect
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible // indirect
|
||||
github.com/shirou/gopsutil/v4 v4.26.7 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
@@ -89,8 +91,8 @@ require (
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.56.0 // indirect
|
||||
golang.org/x/image v0.46.0 // indirect
|
||||
golang.org/x/net v0.58.0 // indirect
|
||||
golang.org/x/sync v0.23.0 // indirect
|
||||
@@ -99,4 +101,5 @@ require (
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
+20
-16
@@ -10,8 +10,8 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ
|
||||
github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
|
||||
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ=
|
||||
github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
|
||||
github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
@@ -28,8 +28,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
|
||||
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA=
|
||||
github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
|
||||
@@ -174,8 +174,8 @@ github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59u
|
||||
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
|
||||
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
|
||||
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
|
||||
github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8=
|
||||
github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk=
|
||||
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
|
||||
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
|
||||
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
|
||||
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
|
||||
@@ -184,10 +184,12 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
|
||||
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
|
||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible h1:x8pckiT12QQhifwhDQpeISgDfsqmQ6VR4LFPQ64JRps=
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||
github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc=
|
||||
@@ -214,8 +216,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
|
||||
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
|
||||
@@ -261,15 +263,15 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
||||
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
||||
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
|
||||
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
@@ -286,8 +288,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
|
||||
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -349,6 +351,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user