diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
index e4ad5c71..e2e4965c 100644
--- a/.github/workflows/release-please.yml
+++ b/.github/workflows/release-please.yml
@@ -2,6 +2,11 @@ on:
   push:
     branches:
       - main
+    paths:
+      - ".github/workflows/release-please.yml"
+      - "release-please-config.json"
+      - ".release-please-manifest.json"
+      - "supplemental/helm/**"
 
 permissions:
   contents: write
@@ -61,6 +66,7 @@ jobs:
     if: ${{ needs.release.outputs.items_to_publish != '' && toJson(fromJson(needs.release.outputs.items_to_publish)) != '[]' }}
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         path: ${{ fromJSON(needs.release.outputs.items_to_publish) }}
 
@@ -101,14 +107,10 @@ jobs:
           .tmp/cr package ${{ matrix.path }}
           ls -la .cr-release-packages/
 
-      - name: Upload binaries to release
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: .cr-release-packages/*.tgz
-          file_glob: true
-          overwrite: true
-          tag: ${{ env.TAG }}
+      - name: Upload chart package to draft release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload "$TAG" .cr-release-packages/*.tgz --clobber
 
       - name: Update index
         run: |
@@ -122,3 +124,15 @@ jobs:
           # Generate index.yaml for GitHub Pages
           args=(-o "$owner" -r "$repo" -c "https://henrygd.github.io/beszel" --push -t "${{ secrets.CR_TOKEN }}" --index-path docs/index.yaml)
           .tmp/cr index "${args[@]}"
+
+      - name: Publish chart release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CHART_PATH: ${{ matrix.path }}
+        run: |
+          chart_name="${CHART_PATH##*/}"
+          release_id=$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" --jq .id)
+          gh api --method PATCH "repos/${GITHUB_REPOSITORY}/releases/${release_id}" \
+            -f "name=Helm chart: ${chart_name} v${VERSION}" \
+            -F draft=false \
+            -f make_latest=false
diff --git a/release-please-config.json b/release-please-config.json
index c60bcbe1..069ad95e 100644
--- a/release-please-config.json
+++ b/release-please-config.json
@@ -8,7 +8,8 @@
       "changelog-path": "CHANGELOG.md",
       "bump-minor-pre-major": true,
       "bump-patch-for-minor-pre-major": true,
-      "draft": false,
+      "draft": true,
+      "force-tag-creation": true,
       "prerelease": false
     },
     "supplemental/helm/beszel-hub": {
@@ -17,7 +18,8 @@
       "changelog-path": "CHANGELOG.md",
       "bump-minor-pre-major": true,
       "bump-patch-for-minor-pre-major": true,
-      "draft": false,
+      "draft": true,
+      "force-tag-creation": true,
       "prerelease": false
     }
   }
diff --git a/supplemental/helm/beszel-agent/Chart.yaml b/supplemental/helm/beszel-agent/Chart.yaml
index ee501d3b..be94952b 100644
--- a/supplemental/helm/beszel-agent/Chart.yaml
+++ b/supplemental/helm/beszel-agent/Chart.yaml
@@ -3,7 +3,7 @@ description: Installs beszel-agent in kubernetes
 home: https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-agent
 name: beszel-agent
 appVersion: "0.17.0"
-# Do not touch will be updated during release
+# This version is managed automatically by Release Please.
 version: 0.1.1
 sources:
   - https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-agent
diff --git a/supplemental/helm/beszel-hub/Chart.yaml b/supplemental/helm/beszel-hub/Chart.yaml
index 7a5293c5..6e9b6ef5 100644
--- a/supplemental/helm/beszel-hub/Chart.yaml
+++ b/supplemental/helm/beszel-hub/Chart.yaml
@@ -3,7 +3,7 @@ description: Installs beszel-hub in kubernetes
 home: https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-hub
 name: beszel-hub
 appVersion: "0.17.0"
-# Do not touch will be updated during release
+# This version is managed automatically by Release Please.
 version: 0.1.1
 sources:
   - https://github.com/henrygd/beszel/tree/main/supplemental/helm/beszel-hub