diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index d92c1e355..554f85fbc 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -1296,6 +1296,180 @@ jobs: PYEOF echo "NetworkPolicy rendering tests passed" + echo "" + echo "=== Testing enterprise license mount + master persistence ===" + python3 - "$CHART_DIR" <<'PYEOF' + import subprocess, sys, yaml + chart = sys.argv[1] + + def render(values): + args = ["helm", "template", "test", chart] + for k, v in values.items(): + args += ["--set", f"{k}={v}"] + return subprocess.check_output(args, text=True) + + def workloads(manifest): + for d in yaml.safe_load_all(manifest): + if d and d.get("kind") in ("Deployment", "StatefulSet"): + yield d + + ALL_ON = { + "admin.enabled": "true", + "s3.enabled": "true", + "sftp.enabled": "true", + "worker.enabled": "true", + } + failed = [] + + # Case 1: unconfigured renders nothing, so existing installs are unaffected. + out = render(ALL_ON) + if "SEAWEED_LICENSE" in out or "seaweedfs-license" in out: + failed.append("defaults: license plumbing should not render") + else: + print("defaults: no license volume/env (unchanged)") + + # Case 2: only the workloads that run a master carry the license. + LICENSE_PATH = "/etc/seaweedfs/license/seaweed-license.json" + out = render(dict(ALL_ON, **{ + "global.seaweedfs.license.existingSecret": "lic", + })) + carrying = set() + for d in workloads(out): + name = d["metadata"]["name"] + pod = d["spec"]["template"]["spec"] + vols = [v for v in pod.get("volumes", []) if v.get("name") == "seaweedfs-license"] + if not vols: + continue + carrying.add(name) + secret = vols[0].get("secret", {}) + if secret.get("secretName") != "lic": + failed.append(f"{name}: license volume points at {secret.get('secretName')!r}, not the configured Secret") + items = [i.get("key") for i in secret.get("items", [])] + if items != ["seaweed-license.json"]: + failed.append(f"{name}: license volume projects {items}, expected only the license key") + for c in pod.get("containers", []): + mounts = [m for m in c.get("volumeMounts", []) if m["name"] == "seaweedfs-license"] + if not mounts: + failed.append(f"{name}/{c['name']}: license volume not mounted") + continue + # subPath would freeze the file at container start. + if any("subPath" in m for m in mounts): + failed.append(f"{name}/{c['name']}: license mount uses subPath (renewals would not propagate)") + if not all(m.get("readOnly") for m in mounts): + failed.append(f"{name}/{c['name']}: license mount is not readOnly") + env = {e["name"]: e.get("value") for e in c.get("env", [])} + if env.get("SEAWEED_LICENSE") != LICENSE_PATH: + failed.append(f"{name}/{c['name']}: SEAWEED_LICENSE not set to the mounted file") + if carrying != {"test-seaweedfs-master"}: + failed.append(f"license: expected only the master to carry it, got {sorted(carrying)}") + else: + print("license: master only, key-scoped, readOnly, no subPath, SEAWEED_LICENSE set") + + # all-in-one runs `weed server -master`, so it needs the license too. + out = render({ + "allInOne.enabled": "true", + "master.enabled": "false", + "volume.enabled": "false", + "filer.enabled": "false", + "global.seaweedfs.license.existingSecret": "lic", + }) + aio = [d for d in workloads(out) if d["metadata"]["name"].endswith("-all-in-one")] + if len(aio) != 1: + failed.append(f"all-in-one: expected 1 workload, got {len(aio)}") + else: + pod = aio[0]["spec"]["template"]["spec"] + if not any(v.get("name") == "seaweedfs-license" for v in pod.get("volumes", [])): + failed.append("all-in-one: missing the license volume") + else: + print("all-in-one: carries the license") + + # SEAWEED_LICENSE is reserved on both env paths. + out = render({ + "global.seaweedfs.license.existingSecret": "lic", + "global.seaweedfs.extraEnvironmentVars.SEAWEED_LICENSE": "/tmp/bogus.json", + }) + for d in workloads(out): + if not d["metadata"]["name"].endswith("-master"): + continue + for c in d["spec"]["template"]["spec"].get("containers", []): + names = [e["name"] for e in c.get("env", [])] + if names.count("SEAWEED_LICENSE") != 1: + failed.append(f"SEAWEED_LICENSE rendered {names.count('SEAWEED_LICENSE')} times") + else: + value = next(e.get("value") for e in c["env"] if e["name"] == "SEAWEED_LICENSE") + if value != LICENSE_PATH: + failed.append(f"SEAWEED_LICENSE overridden to {value!r} by extraEnvironmentVars") + else: + print("SEAWEED_LICENSE: reserved, rendered once, points at the mount") + + # secretExtraEnvironmentVars is rendered outside the merge helper. + out = render({ + "allInOne.enabled": "true", + "master.enabled": "false", + "volume.enabled": "false", + "filer.enabled": "false", + "global.seaweedfs.license.existingSecret": "lic", + "allInOne.secretExtraEnvironmentVars.SEAWEED_LICENSE.secretKeyRef.name": "x", + "allInOne.secretExtraEnvironmentVars.SEAWEED_LICENSE.secretKeyRef.key": "y", + }) + for d in workloads(out): + if not d["metadata"]["name"].endswith("-all-in-one"): + continue + for c in d["spec"]["template"]["spec"].get("containers", []): + names = [e["name"] for e in c.get("env", [])] + if names.count("SEAWEED_LICENSE") != 1: + failed.append(f"all-in-one: SEAWEED_LICENSE rendered {names.count('SEAWEED_LICENSE')} times via secretExtraEnvironmentVars") + elif next(e.get("value") for e in c["env"] if e["name"] == "SEAWEED_LICENSE") != LICENSE_PATH: + failed.append("all-in-one: secretExtraEnvironmentVars overrode SEAWEED_LICENSE") + else: + print("all-in-one: SEAWEED_LICENSE reserved on the secret env path too") + + # Case 3: the default stays hostPath. volumeClaimTemplates is immutable, + # so growing one here would break helm upgrade on existing releases. + masters = [d for d in workloads(render({})) if d["metadata"]["name"].endswith("-master")] + if len(masters) != 1: + failed.append(f"master: expected 1 StatefulSet by default, got {len(masters)}") + else: + d = masters[0] + claims = [c["metadata"]["name"] for c in d["spec"].get("volumeClaimTemplates", [])] + if any(c.startswith("data-") for c in claims): + failed.append(f"master: default grew a data volumeClaimTemplate ({claims}); " + "that breaks helm upgrade on existing releases") + data_host = [v["name"] for v in d["spec"]["template"]["spec"].get("volumes", []) + if v.get("hostPath") and v["name"].startswith("data-")] + if not data_host: + failed.append("master: default no longer renders a hostPath data volume") + else: + print("master: default still hostPath (upgrade-compatible)") + + # Case 4: opting into a claim renders one, with the requested size. + masters = [d for d in workloads(render({"master.data.type": "persistentVolumeClaim"})) + if d["metadata"]["name"].endswith("-master")] + if len(masters) != 1: + failed.append(f"master.data.type=persistentVolumeClaim: expected 1 StatefulSet, got {len(masters)}") + else: + d = masters[0] + claims = {c["metadata"]["name"]: c["spec"]["resources"]["requests"]["storage"] + for c in d["spec"].get("volumeClaimTemplates", [])} + data = {k: v for k, v in claims.items() if k.startswith("data-")} + if not data: + failed.append(f"master.data.type=persistentVolumeClaim: no data claim rendered ({claims})") + elif set(data.values()) != {"1Gi"}: + failed.append(f"master.data.type=persistentVolumeClaim: unexpected size {data}") + else: + print(f"master.data.type=persistentVolumeClaim: renders {sorted(data)} at 1Gi") + if any(v.get("hostPath") and v["name"].startswith("data-") + for v in d["spec"]["template"]["spec"].get("volumes", [])): + failed.append("master.data.type=persistentVolumeClaim: data still on a hostPath") + + if failed: + print("\nFAIL:", file=sys.stderr) + for f in failed: + print(f" - {f}", file=sys.stderr) + sys.exit(1) + PYEOF + echo "License + master persistence tests passed" + echo "All template rendering tests passed!" - name: Create kind cluster diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index cf6378bdf..7edde6754 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -107,6 +107,120 @@ https://github.com/rancher/local-path-provisioner you can use ANY storage class you like, just update the correct storage-class for your deployment. +### Master data: hostPath vs a claim + +The master's `-mdir` holds its Raft log and snapshots, and with them the +cluster's identity (its topology UUID). `master.data.type` defaults to +`hostPath`, which does not follow a pod to another node: a master that is +rescheduled comes back with an empty data directory and a brand new cluster +UUID. With the chart's default of a single master replica there is no peer to +recover the identity from either. + +Putting the master's data on a claim avoids that: + +```yaml +master: + data: + type: "persistentVolumeClaim" + size: "1Gi" + storageClass: "" # empty uses the cluster's default StorageClass +``` + +Raft state is small, so a modest claim is enough — sizing matters far more for +volume and filer. + +The default is left at `hostPath` for backward compatibility: +`volumeClaimTemplates` is immutable on a StatefulSet, so flipping the type on a +release that already exists fails, whether the chart changes the default or you +change it yourself: + +```text +StatefulSet.apps "-seaweedfs-master" is invalid: spec: Forbidden: +updates to statefulset spec for fields other than 'replicas', ... are forbidden +``` + +New installs can set the claim from the start. To move an **existing** release +onto a claim without losing the cluster UUID, use the migration below. The +claim has to be seeded while the master is stopped: a running master rewrites +its Raft state, so copying into a live pod is silently undone by the next +restart. + +The steps below are for the chart's default of a single master +(`master.replicas: 1`). With several master replicas, repeat steps 1, 3 and 4 +for every ordinal, or migrate one at a time and let the remaining quorum +re-replicate. + +Take the names from the cluster rather than assembling them — the release +name, `nameOverride` and `fullnameOverride` all feed the chart's fullname +helper, so `-seaweedfs` is not always right: + +```bash +NS=; REL= +# scope by instance as well as component: several releases can share a namespace +STS=$(kubectl -n $NS get sts \ + -l app.kubernetes.io/instance=$REL,app.kubernetes.io/component=master \ + -o jsonpath='{.items[0].metadata.name}') +POD=$STS-0 +# a StatefulSet names its claims