name: "helm: lint and test charts"

on:
  push:
    branches: [ master ]
    paths: ['k8s/**']
  pull_request:
    branches: [ master ]
    paths: ['k8s/**']

permissions:
  contents: read

jobs:
  lint-test:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
        with:
          fetch-depth: 0

      - name: Set up Helm
        uses: azure/setup-helm@v5
        with:
          version: v3.18.4

      - uses: actions/setup-python@v7
        with:
          python-version: '3.10'
          check-latest: true

      - name: Set up chart-testing
        uses: helm/chart-testing-action@v2.8.0

      - name: Run chart-testing (list-changed)
        id: list-changed
        run: |
          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts)
          if [[ -n "$changed" ]]; then
            echo "::set-output name=changed::true"
          fi

      - name: Run chart-testing (lint)
        run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts

      - name: Verify template rendering
        run: |
          set -e
          CHART_DIR="k8s/charts/seaweedfs"
          
          echo "=== Testing default configuration ==="
          helm template test $CHART_DIR > /tmp/default.yaml
          echo "Default configuration renders successfully"
          
          echo "=== Testing with S3 enabled ==="
          helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
          grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
          echo "S3 deployment renders correctly"
          
          echo "=== Testing with all-in-one mode ==="
          helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
          grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
          echo "All-in-one deployment renders correctly"
          
          echo "=== Testing with security enabled ==="
          helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true > /tmp/security.yaml
          grep -q "security-config" /tmp/security.yaml
          echo "Security configuration renders correctly"

          echo ""
          echo "=== Testing JWT expiration overrides ==="
          helm template test $CHART_DIR \
            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
            > /tmp/jwt-volume-write-expiration.yaml
          grep -q "security-config" /tmp/jwt-volume-write-expiration.yaml
          grep -q "expires_after_seconds = 11" /tmp/jwt-volume-write-expiration.yaml

          helm template test $CHART_DIR \
            --set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
            --set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
            --set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeRead=22 \
            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerWrite=33 \
            --set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerRead=44 \
            > /tmp/jwt-expiration.yaml

          assert_jwt_expiration() {
            local section="$1"
            local seconds="$2"
            awk -v section="[$section]" -v seconds="$seconds" '
              /^[[:space:]]*\[.*\][[:space:]]*$/ {
                in_section = index($0, section) > 0
              }
              in_section && $0 ~ "^[[:space:]]*expires_after_seconds = " seconds "$" {
                found = 1
              }
              END { exit !found }
            ' /tmp/jwt-expiration.yaml
          }

          assert_jwt_expiration jwt.signing 11
          assert_jwt_expiration jwt.signing.read 22
          assert_jwt_expiration jwt.filer_signing 33
          assert_jwt_expiration jwt.filer_signing.read 44

          helm template test $CHART_DIR \
            --set global.seaweedfs.enableSecurity=true \
            --set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
            --set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
            --set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
            > /tmp/jwt-default-expiration.yaml
          if grep -q "expires_after_seconds =" /tmp/jwt-default-expiration.yaml; then
            echo "FAIL: zero JWT expiration values should preserve runtime defaults"
            exit 1
          fi
          echo "JWT expiration overrides render correctly"

          echo ""
          echo "=== Testing IAM gRPC opt-in path ==="
          # Regression test: the filer registers the IAM gRPC service the
          # Admin UI Users tab calls only when jwt.filer_signing.key is in
          # security.toml. Operators must be able to enable that without
          # the cert-manager mTLS bundle.
          # Install PyYAML explicitly: this block runs before the later
          # security+S3 block that does the same install, and we don't
          # want to rely on the runner image shipping it.
          pip install pyyaml -q
          python3 - "$CHART_DIR" <<'PYEOF'
          import subprocess, sys, yaml
          chart = sys.argv[1]

          def render(values):
              args = ["helm", "template", "test", chart]
              for k, v in values.items():
                  args += ["--set", f"{k}={v}"]
              return subprocess.check_output(args, text=True)

          def docs(manifest):
              return [d for d in yaml.safe_load_all(manifest) if d]

          def configmap(manifest, name):
              for d in docs(manifest):
                  if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
                      return d
              return None

          def workload_mounts(manifest, name):
              for d in docs(manifest):
                  if d.get("kind") not in ("Deployment", "StatefulSet"):
                      continue
                  if d["metadata"]["name"] != name:
                      continue
                  pod = d["spec"]["template"]["spec"]
                  vols = {v["name"] for v in pod.get("volumes", [])}
                  mounts = set()
                  for c in pod.get("containers", []):
                      for vm in c.get("volumeMounts", []):
                          mounts.add(vm["name"])
                  return vols, mounts
              return None, None

          failed = []

          # Case 1: defaults. The chart historically rendered nothing
          # security-related; preserve that so this PR is non-breaking on
          # existing installs.
          out = render({})
          if configmap(out, "test-seaweedfs-security-config") is not None:
              failed.append("defaults: security ConfigMap should not render")
          else:
              print("defaults: no security-config ConfigMap (unchanged)")

          # Case 2: filerWrite=true alone is the documented opt-in for
          # the Admin UI Users tab. Configmap must render with
          # [jwt.filer_signing] and NO [grpc.*] sections (cert paths
          # only exist with mTLS).
          out = render({
              "global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true",
              "admin.enabled": "true",
          })
          cm = configmap(out, "test-seaweedfs-security-config")
          if cm is None:
              failed.append("filerWrite=true: security ConfigMap missing")
          else:
              toml = cm["data"]["security.toml"]
              if "[jwt.filer_signing]" not in toml:
                  failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
              if "[grpc" in toml:
                  failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
              if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
                  print("filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")

          # Case 3: filer + admin pods must MOUNT the security ConfigMap
          # under filerWrite=true so the JWT key reaches both processes.
          # Cert volumes must NOT be present (no mTLS).
          for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
              vols, mounts = workload_mounts(out, wl)
              if vols is None:
                  failed.append(f"filerWrite=true: workload {wl} not found")
                  continue
              if "security-config" not in vols or "security-config" not in mounts:
                  failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
              else:
                  print(f"filerWrite=true: {wl} mounts security-config")
              cert_vols = {v for v in vols if v.endswith("-cert")}
              if cert_vols:
                  failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")

          # Case 4: enableSecurity=true must still render the full toml
          # with both [jwt.signing] and [grpc.*]. Guards against the
          # decoupling change accidentally regressing the mTLS path.
          out = render({"global.seaweedfs.enableSecurity": "true"})
          cm = configmap(out, "test-seaweedfs-security-config")
          if cm is None:
              failed.append("enableSecurity=true: security ConfigMap missing")
          else:
              toml = cm["data"]["security.toml"]
              missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
              if missing:
                  failed.append(f"enableSecurity=true: security.toml missing {missing}")
              else:
                  print("enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")

          # Case 5: helper must tolerate explicit nulls (gemini-code-assist
          # PR review). securityConfig=null was the parens-pattern crash
          # the helper review caught.
          for null_path in ("global.seaweedfs.securityConfig",
                            "global.seaweedfs.securityConfig.jwtSigning"):
              try:
                  out = render({null_path: "null"})
              except subprocess.CalledProcessError as e:
                  failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
                  continue
              if configmap(out, "test-seaweedfs-security-config") is not None:
                  failed.append(f"{null_path}=null: should not render configmap")
              else:
                  print(f"{null_path}=null: render tolerates explicit null")

          if failed:
              print("\nFAIL:", file=sys.stderr)
              for f in failed:
                  print(f"  - {f}", file=sys.stderr)
              sys.exit(1)
          PYEOF
          echo "IAM gRPC decoupling tests passed"

          echo "=== Testing with monitoring enabled ==="
          helm template test $CHART_DIR \
            --set global.seaweedfs.monitoring.enabled=true \
            --set global.seaweedfs.monitoring.gatewayHost=prometheus \
            --set global.seaweedfs.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
          echo "Monitoring configuration renders correctly"
          
          echo "=== Testing with PVC storage ==="
          helm template test $CHART_DIR \
            --set master.data.type=persistentVolumeClaim \
            --set master.data.size=10Gi \
            --set master.data.storageClass=standard > /tmp/pvc.yaml
          grep -q "PersistentVolumeClaim" /tmp/pvc.yaml
          echo "PVC configuration renders correctly"
          
          echo "=== Testing with custom replicas ==="
          helm template test $CHART_DIR \
            --set master.replicas=3 \
            --set filer.replicas=2 \
            --set volume.replicas=3 > /tmp/replicas.yaml
          echo "Custom replicas configuration renders correctly"
          
          echo "=== Testing filer with S3 gateway ==="
          helm template test $CHART_DIR \
            --set filer.s3.enabled=true \
            --set filer.s3.enableAuth=true > /tmp/filer-s3.yaml
          echo "Filer S3 gateway renders correctly"
          
          echo "=== Testing SFTP enabled ==="
          helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml
          grep -q "seaweedfs-sftp" /tmp/sftp.yaml
          echo "SFTP deployment renders correctly"

          echo ""
          echo "=== Testing SFTP host key generation ==="
          # The chart must not ship host key material: keys are generated
          # per install, land in the secret the deployments mount at
          # sftp.hostKeysFolder, and must be PKCS#8 ed25519 private keys,
          # the shape the server's host key loader parses; it fails to
          # start otherwise.
          pip install pyyaml -q
          python3 - "$CHART_DIR" <<'PYEOF'
          import base64, re, subprocess, sys, yaml

          chart = sys.argv[1]

          def render(values):
              args = ["helm", "template", "test", chart]
              for k, v in values.items():
                  args += ["--set", f"{k}={v}"]
              return subprocess.check_output(args, text=True)

          def docs(manifest):
              return [d for d in yaml.safe_load_all(manifest) if d]

          def secret(manifest, name):
              for d in docs(manifest):
                  if d.get("kind") == "Secret" and d["metadata"]["name"] == name:
                      return d
              return None

          def pod_spec(manifest, name):
              for d in docs(manifest):
                  if d.get("kind") in ("Deployment", "StatefulSet") and d["metadata"]["name"] == name:
                      return d["spec"]["template"]["spec"]
              return None

          def script_of(spec):
              for c in spec["containers"]:
                  cmd = c.get("command", [])
                  if len(cmd) >= 3 and cmd[0] == "/bin/sh":
                      return cmd[2]
              raise AssertionError("no shell command block found")

          def volume_secret(spec, volname):
              for v in spec.get("volumes", []):
                  if v["name"] == volname:
                      return v["secret"]["secretName"]
              return None

          def mount_path(spec, volname):
              for c in spec["containers"]:
                  for vm in c.get("volumeMounts", []):
                      if vm["name"] == volname:
                          return vm["mountPath"]
              return None

          def parse_ed25519(pem):
              m = re.match(r"-----BEGIN PRIVATE KEY-----\n(.+?)-----END PRIVATE KEY-----", pem.strip(), re.S)
              if not m:
                  raise AssertionError("not a PKCS#8 PEM private key")
              der = base64.b64decode(m.group(1))
              # RFC 8410: fixed PKCS#8 prefix, then the 32-byte seed
              prefix = bytes.fromhex("302e020100300506032b657004220420")
              if len(der) != 48 or not der.startswith(prefix):
                  raise AssertionError("not an ed25519 PKCS#8 key")

          failed = []
          # public-key material of the key the chart used to bundle
          BUNDLED = "H4McwcDphteXVullu6q7ephEN1N60z"

          out1 = render({"sftp.enabled": "true"})
          out2 = render({"sftp.enabled": "true"})

          for label, out in (("first", out1), ("second", out2)):
              if BUNDLED in out:
                  failed.append(f"{label} render still contains the formerly bundled host key")

          def folder_key(out):
              s = secret(out, "test-seaweedfs-sftp-ssh-secret")
              if s is None:
                  return None
              return base64.b64decode(s["data"]["ssh_host_ed25519_key"]).decode()

          k1, k2 = folder_key(out1), folder_key(out2)
          if k1 is None or k2 is None:
              failed.append("sftp-ssh-secret not rendered with sftp.enabled=true")
          else:
              try:
                  parse_ed25519(k1)
                  print("generated host key parses as ed25519")
              except Exception as e:
                  failed.append(f"generated host key does not parse: {e}")
              if k1 == k2:
                  failed.append("two renders produced the same host key (key is not generated per install)")
              else:
                  print("host key differs between installs")

          legacy1 = secret(out1, "test-seaweedfs-sftp-secret")["stringData"]["seaweedfs_sftp_ssh_private_key"]
          legacy2 = secret(out2, "test-seaweedfs-sftp-secret")["stringData"]["seaweedfs_sftp_ssh_private_key"]
          try:
              parse_ed25519(legacy1)
          except Exception as e:
              failed.append(f"sftp-secret ssh key does not parse: {e}")
          if legacy1 == legacy2:
              failed.append("sftp-secret ssh key identical across renders")
          else:
              print("sftp-secret ssh key is generated per install")

          spec = pod_spec(out1, "test-seaweedfs-sftp")
          script = script_of(spec)
          if "-sshPrivateKey" in script:
              failed.append("sftp deployment passes -sshPrivateKey by default; the file only exists "
                            "when enableAuth mounts /etc/sw and a missing key file is fatal")
          if "-hostKeysFolder=/etc/sw/ssh" not in script:
              failed.append("sftp deployment missing -hostKeysFolder=/etc/sw/ssh")
          if volume_secret(spec, "config-ssh") != "test-seaweedfs-sftp-ssh-secret":
              failed.append("sftp config-ssh volume does not reference the generated secret")
          else:
              print("sftp deployment mounts the generated secret at the host keys folder")

          out = render({"sftp.enabled": "true", "sftp.existingSshConfigSecret": "my-keys"})
          if secret(out, "test-seaweedfs-sftp-ssh-secret") is not None:
              failed.append("existingSshConfigSecret set but the default ssh secret still renders")
          if volume_secret(pod_spec(out, "test-seaweedfs-sftp"), "config-ssh") != "my-keys":
              failed.append("existingSshConfigSecret is not the config-ssh volume source")
          else:
              print("existingSshConfigSecret replaces the generated secret")

          out = render({"allInOne.enabled": "true", "allInOne.sftp.enabled": "true"})
          spec = pod_spec(out, "test-seaweedfs-all-in-one")
          if secret(out, "test-seaweedfs-sftp-ssh-secret") is None:
              failed.append("all-in-one: ssh secret not rendered")
          if "-sftp.hostKeysFolder=/etc/sw/ssh" not in script_of(spec):
              failed.append("all-in-one: missing -sftp.hostKeysFolder=/etc/sw/ssh")
          if volume_secret(spec, "config-ssh") != "test-seaweedfs-sftp-ssh-secret":
              failed.append("all-in-one: config-ssh volume does not reference the generated secret")
          else:
              print("all-in-one mounts the generated secret")

          out = render({"sftp.enabled": "true", "sftp.hostKeysFolder": "/keys"})
          spec = pod_spec(out, "test-seaweedfs-sftp")
          if "-hostKeysFolder=/keys" not in script_of(spec) or mount_path(spec, "config-ssh") != "/keys":
              failed.append("custom hostKeysFolder: flag and secret mount do not agree")
          else:
              print("custom hostKeysFolder keeps flag and mount aligned")

          out = render({"allInOne.enabled": "true", "allInOne.sftp.enabled": "true",
                        "allInOne.sftp.hostKeysFolder": "/keys"})
          spec = pod_spec(out, "test-seaweedfs-all-in-one")
          if "-sftp.hostKeysFolder=/keys" not in script_of(spec) or mount_path(spec, "config-ssh") != "/keys":
              failed.append("all-in-one custom hostKeysFolder: flag and secret mount do not agree")
          else:
              print("all-in-one custom hostKeysFolder keeps flag and mount aligned")

          if failed:
              print("\nFAIL:", file=sys.stderr)
              for f in failed:
                  print(f"  - {f}", file=sys.stderr)
              sys.exit(1)
          PYEOF
          echo "SFTP host key generation tests passed"
          
          echo "=== Testing ingress configurations ==="
          helm template test $CHART_DIR \
            --set master.ingress.enabled=true \
            --set filer.ingress.enabled=true \
            --set s3.enabled=true \
            --set s3.ingress.enabled=true > /tmp/ingress.yaml
          grep -q "kind: Ingress" /tmp/ingress.yaml
          echo "Ingress configurations render correctly"
          
          echo "=== Testing COSI driver ==="
          helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
          grep -q "seaweedfs-cosi" /tmp/cosi.yaml
          echo "COSI driver renders correctly"
          
          echo ""
          echo "=== Testing long release name: service names match DNS references ==="
          # Use a release name that, combined with chart name "seaweedfs", exceeds 63 chars.
          # fullname = "my-very-long-release-name-that-will-cause-truncation-seaweedfs" (65 chars before trunc)
          LONG_RELEASE="my-very-long-release-name-that-will-cause-truncation"

          # --- Normal mode: master + filer-client services vs helper-produced addresses ---
          helm template "$LONG_RELEASE" $CHART_DIR \
            --set s3.enabled=true \
            --set global.seaweedfs.createBuckets[0].name=test > /tmp/longname.yaml

          # Extract Service names from metadata
          MASTER_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
            | grep -- '-master$')
          FILER_CLIENT_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
            | grep -- '-filer-client$')

          # Extract the hostname from WEED_CLUSTER_SW_MASTER in post-install-bucket-hook
          MASTER_ADDR=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname.yaml \
            | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')
          FILER_ADDR=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname.yaml \
            | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')

          # Extract the hostname from S3 deployment -filer= argument
          S3_FILER_HOST=$(grep '\-filer=' /tmp/longname.yaml \
            | head -1 | sed 's/.*-filer=\([^:]*\).*/\1/')

          # The address helpers produce "<svc>.<namespace>:<port>"; extract just the svc name
          MASTER_ADDR_SVC=$(echo "$MASTER_ADDR" | cut -d. -f1)
          FILER_ADDR_SVC=$(echo "$FILER_ADDR" | cut -d. -f1)
          S3_FILER_SVC=$(echo "$S3_FILER_HOST" | cut -d. -f1)

          echo "  master Service.name:       $MASTER_SVC"
          echo "  cluster.masterAddress svc:  $MASTER_ADDR_SVC"
          echo "  filer-client Service.name:  $FILER_CLIENT_SVC"
          echo "  cluster.filerAddress svc:   $FILER_ADDR_SVC"
          echo "  S3 -filer= svc:             $S3_FILER_SVC"

          [ "$MASTER_SVC" = "$MASTER_ADDR_SVC" ] || { echo "FAIL: master service name mismatch"; exit 1; }
          [ "$FILER_CLIENT_SVC" = "$FILER_ADDR_SVC" ] || { echo "FAIL: filer-client service name mismatch"; exit 1; }
          [ "$FILER_CLIENT_SVC" = "$S3_FILER_SVC" ] || { echo "FAIL: S3 -filer= does not match filer-client service"; exit 1; }
          echo "Normal mode: service names match DNS references with long release name"

          # --- All-in-one mode: all-in-one service vs both helper addresses ---
          helm template "$LONG_RELEASE" $CHART_DIR \
            --set allInOne.enabled=true \
            --set global.seaweedfs.createBuckets[0].name=test > /tmp/longname-aio.yaml

          AIO_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname-aio.yaml \
            | grep -- '-all-in-one$')
          AIO_MASTER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname-aio.yaml \
            | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)
          AIO_FILER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname-aio.yaml \
            | grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)

          echo "  all-in-one Service.name:    $AIO_SVC"
          echo "  cluster.masterAddress svc:  $AIO_MASTER_ADDR_SVC"
          echo "  cluster.filerAddress svc:   $AIO_FILER_ADDR_SVC"

          [ "$AIO_SVC" = "$AIO_MASTER_ADDR_SVC" ] || { echo "FAIL: all-in-one master address mismatch"; exit 1; }
          [ "$AIO_SVC" = "$AIO_FILER_ADDR_SVC" ] || { echo "FAIL: all-in-one filer address mismatch"; exit 1; }
          echo "All-in-one mode: service names match DNS references with long release name"

          echo ""
          echo "=== Testing security+S3: no blank lines in shell command blocks ==="
          # Render the three manifests that include seaweedfs.s3.tlsArgs:
          #   filer-statefulset, s3-deployment, all-in-one-deployment
          helm template test $CHART_DIR \
            --set global.seaweedfs.enableSecurity=true \
            --set filer.s3.enabled=true \
            --set s3.enabled=true > /tmp/security-s3.yaml
          helm template test $CHART_DIR \
            --set global.seaweedfs.enableSecurity=true \
            --set allInOne.enabled=true \
            --set allInOne.s3.enabled=true > /tmp/security-aio.yaml

          pip install pyyaml -q
          python3 - /tmp/security-s3.yaml /tmp/security-aio.yaml <<'PYEOF'
          import yaml, sys
          errors = []
          for path in sys.argv[1:]:
              with open(path) as f:
                  docs = list(yaml.safe_load_all(f))
              for doc in docs:
                  if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
                      continue
                  name = doc["metadata"]["name"]
                  for c in doc["spec"]["template"]["spec"].get("containers", []):
                      cmd = c.get("command", [])
                      if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
                          script = cmd[2]
                          for i, line in enumerate(script.splitlines(), 1):
                              if line.strip() == "":
                                  errors.append(f"{path}: {name}/{c['name']} has blank line at script line {i}")
          if errors:
              for e in errors:
                  print(f"FAIL: {e}", file=sys.stderr)
              print("Rendered with: global.seaweedfs.enableSecurity=true, filer.s3.enabled=true, s3.enabled=true, allInOne.enabled=true", file=sys.stderr)
              sys.exit(1)
          print("No blank lines in security+S3 command blocks")
          PYEOF

          echo ""
          echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
          # Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
          # the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
          # them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
          # makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
          # HTTP request to an HTTPS server" into the pod log.
          #
          # When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
          # together so the opt-in HTTPS listener actually has credentials.
          python3 - "$CHART_DIR" <<'PYEOF'
          import subprocess, sys, yaml
          chart = sys.argv[1]

          def render(values):
              args = ["helm", "template", "test", chart]
              for k, v in values.items():
                  args += ["--set", f"{k}={v}"]
              return subprocess.check_output(args, text=True)

          def script_of(manifest, kind_name):
              for doc in yaml.safe_load_all(manifest):
                  if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
                      continue
                  if doc["metadata"]["name"] != kind_name:
                      continue
                  for c in doc["spec"]["template"]["spec"]["containers"]:
                      cmd = c.get("command", [])
                      if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
                          return cmd[2]
              raise AssertionError(f"no container script for {kind_name}")

          cases = [
              # (values, workload-name, httpsPort-set?, arg-prefix)
              ({"global.seaweedfs.enableSecurity": "true",
                "s3.enabled": "true"},
               "test-seaweedfs-s3", False, ""),
              ({"global.seaweedfs.enableSecurity": "true",
                "s3.enabled": "true",
                "s3.httpsPort": "8443"},
               "test-seaweedfs-s3", True, ""),
              ({"global.seaweedfs.enableSecurity": "true",
                "filer.s3.enabled": "true"},
               "test-seaweedfs-filer", False, "s3."),
              ({"global.seaweedfs.enableSecurity": "true",
                "filer.s3.enabled": "true",
                "filer.s3.httpsPort": "8444"},
               "test-seaweedfs-filer", True, "s3."),
              ({"global.seaweedfs.enableSecurity": "true",
                "allInOne.enabled": "true",
                "allInOne.s3.enabled": "true"},
               "test-seaweedfs-all-in-one", False, "s3."),
              ({"global.seaweedfs.enableSecurity": "true",
                "allInOne.enabled": "true",
                "allInOne.s3.enabled": "true",
                "allInOne.s3.httpsPort": "8445"},
               "test-seaweedfs-all-in-one", True, "s3."),
          ]

          failed = False
          for values, name, https_on, prefix in cases:
              script = script_of(render(values), name)
              cert_flag = f"-{prefix}cert.file="
              key_flag = f"-{prefix}key.file="
              https_flag = f"-{prefix}port.https="
              has_cert = cert_flag in script
              has_key = key_flag in script
              has_https = https_flag in script
              label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
              if https_on:
                  if not (has_cert and has_key and has_https):
                      print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
                            f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
                      failed = True
                  else:
                      print(f"{label}: cert/key/https args emitted together")
              else:
                  if has_cert or has_key or has_https:
                      print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
                            f"main S3 -port would silently become HTTPS and break HTTP probes "
                            f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
                      failed = True
                  else:
                      print(f"{label}: no TLS args emitted, main -port stays HTTP")

              # bash -n: pin down that the rendered script parses. Guards against
              # a future helper change that leaves a dangling `\` with nothing
              # after it (every current caller already exits cleanly because
              # bash treats trailing `\<newline><EOF>` as line-continuation to
              # an empty line — but keep the contract explicit).
              parse = subprocess.run(["bash", "-n"], input=script, text=True,
                                     capture_output=True)
              if parse.returncode != 0:
                  print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
                        file=sys.stderr)
                  failed = True

          sys.exit(1 if failed else 0)
          PYEOF

          echo ""
          echo "=== Testing all-in-one env: a key in both global and component renders once ==="
          # Regression: all-in-one looped global and component extraEnvironmentVars
          # in two separate ranges, emitting duplicate env entries for any key set
          # in both maps. Render a shared key and assert it appears exactly once in
          # the all-in-one container, with the component value winning (consistent
          # with the merge helper the other components already use). pyyaml is
          # installed by the earlier IAM gRPC block in this same step.
          helm template test $CHART_DIR \
            --set allInOne.enabled=true \
            --set global.seaweedfs.extraEnvironmentVars.WEED_SHARED=fromGlobal \
            --set allInOne.extraEnvironmentVars.WEED_SHARED=fromComponent > /tmp/aio-env.yaml
          python3 - /tmp/aio-env.yaml <<'PYEOF'
          import sys, yaml
          from collections import Counter
          docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if d]
          dep = next(d for d in docs if d.get("kind") == "Deployment"
                     and d["metadata"]["name"].endswith("all-in-one"))
          envs = [e["name"] for c in dep["spec"]["template"]["spec"]["containers"]
                  for e in c.get("env", [])]
          dups = {k: v for k, v in Counter(envs).items() if v > 1}
          if dups:
              print(f"FAIL: duplicate env entries in all-in-one container: {dups}", file=sys.stderr)
              sys.exit(1)
          val = next(e.get("value") for c in dep["spec"]["template"]["spec"]["containers"]
                     for e in c.get("env", []) if e["name"] == "WEED_SHARED")
          if val != "fromComponent":
              print(f"FAIL: WEED_SHARED should take the component value 'fromComponent', got '{val}'",
                    file=sys.stderr)
              sys.exit(1)
          print("all-in-one env: shared key renders once, component value wins")
          PYEOF

          echo "=== Testing bucket versioning: YAML bool false suspends like string \"false\" ==="
          # bool false used to be a silent no-op while string "false" suspended.
          BOOL_FALSE=$(helm template test $CHART_DIR \
            --set s3.enabled=true \
            --set s3.createBuckets[0].name=verbucket \
            --set s3.createBuckets[0].versioning=false | grep 's3.bucket.versioning -name verbucket' || true)
          echo "$BOOL_FALSE" | grep -q -- '-status Suspended' || { echo "FAIL: bool false versioning did not Suspend the bucket"; exit 1; }
          echo "Bucket versioning: YAML bool false suspends consistently with string \"false\""

          echo "All template rendering tests passed!"

      - name: Create kind cluster
        uses: helm/kind-action@v1.14.0

      - name: Run chart-testing (install)
        run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts

      - name: Verify SFTP host key secret lifecycle
        run: |
          set -e
          CHART_DIR="k8s/charts/seaweedfs"
          NS="sftp-hostkey"
          SECRET="hk-seaweedfs-sftp-ssh-secret"
          SFTP_ARGS="--set sftp.enabled=true --set master.enabled=false --set volume.enabled=false --set filer.enabled=false"
          kubectl create namespace "$NS"

          echo "=== install generates a host key, upgrade keeps it ==="
          helm install hk $CHART_DIR -n "$NS" $SFTP_ARGS
          KEY1=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}')
          [ -n "$KEY1" ] || { echo "FAIL: install did not create a host key"; exit 1; }
          echo "$KEY1" | base64 -d | grep -q "BEGIN PRIVATE KEY" || { echo "FAIL: host key is not a PEM private key"; exit 1; }
          helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS
          KEY2=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}')
          [ "$KEY1" = "$KEY2" ] || { echo "FAIL: host key changed across upgrade"; exit 1; }
          echo "host key survives upgrade"

          echo "=== the key the chart used to bundle is replaced ==="
          kubectl delete secret "$SECRET" -n "$NS"
          kubectl create secret generic "$SECRET" -n "$NS" \
            --from-literal=ssh_host_ed25519_key="stand-in H4McwcDphteXVullu6q7ephEN1N60z stand-in"
          helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS
          ROTATED=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}' | base64 -d)
          case "$ROTATED" in
            *H4McwcDphteXVullu6q7ephEN1N60z*) echo "FAIL: bundled key survived the upgrade"; exit 1;;
          esac
          echo "$ROTATED" | grep -q "BEGIN PRIVATE KEY" || { echo "FAIL: replacement is not a generated key"; exit 1; }
          echo "bundled key rotated to a generated one"

          echo "=== operator-managed keys are kept as-is ==="
          kubectl delete secret "$SECRET" -n "$NS"
          ssh-keygen -q -t ed25519 -N "" -C "" -f /tmp/operator_key
          kubectl create secret generic "$SECRET" -n "$NS" --from-file=my_key=/tmp/operator_key
          helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS
          kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.my_key}' | base64 -d | cmp -s - /tmp/operator_key \
            || { echo "FAIL: operator key was modified"; exit 1; }
          NKEYS=$(kubectl get secret "$SECRET" -n "$NS" -o json | jq '.data | length')
          [ "$NKEYS" = "1" ] || { echo "FAIL: expected only the operator key, found $NKEYS entries"; exit 1; }
          echo "operator key kept, no extra key generated"

          kubectl delete namespace "$NS"
          echo "SFTP host key lifecycle tests passed"