add job to automatically find and remove orphaned AKS clusters

pipelines/concourse-workers/pipeline.yml

@@ -1,4 +1,4 @@
-# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
+# Copyright 2020-2025 the Pinniped contributors. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
 display:
@@ -28,6 +28,14 @@ meta:
     GCP_SERVICE_ACCOUNT: ((gke-test-pool-manager-username))
     GCP_JSON_KEY: ((gke-test-pool-manager-json-key))
 
+  # Azure account info and which resource group the clusters should be created in and deleted from.
+  azure_account_params: &azure_account_params
+    AZURE_SUBSCRIPTION_ID: ((azure-bot-subscription-id))
+    AZURE_TENANT: ((azure-bot-tenant-id))
+    AZURE_RESOURCE_GROUP: pinniped-ci
+    AZURE_USERNAME: ((azure-bot-app-id))
+    AZURE_PASSWORD: ((azure-bot-password))
+
 resources:
 
   - name: pinniped-ci
@@ -56,6 +64,13 @@ resources:
       repository: google/cloud-sdk
       tag: slim
 
+  - name: aks-deployer-image
+    type: registry-image
+    icon: docker
+    check_every: 5m
+    source:
+      repository: mcr.microsoft.com/azure-cli
+
   - name: hourly
     type: time
     icon: calendar-clock
@@ -146,3 +161,19 @@ jobs:
         image: gcloud-image
         params:
           <<: *gke_account_params
+
+  - name: remove-orphaned-aks-clusters
+    public: true # all logs are publicly visible
+    plan:
+      - in_parallel:
+          - get: pinniped-ci
+          - get: aks-deployer-image
+          - get: hourly
+            trigger: true
+      - task: remove-orphaned-aks-clusters
+        attempts: 2
+        timeout: 25m
+        file: pinniped-ci/pipelines/shared-tasks/remove-orphaned-aks-clusters/task.yml
+        image: gcloud-image
+        params:
+          <<: *azure_account_params

pipelines/main/pipeline.yml

@@ -103,6 +103,7 @@ meta:
     AWS_SECRET_ACCESS_KEY: ((aws-concourse-ci-iam-key-secret))
     AWS_ROLE_ARN: ((aws-concourse-ci-role-arn))
 
+  # Azure account info and which resource group the clusters should be created in and deleted from.
   azure_account_params: &azure_account_params
     AZURE_REGION: westus2
     AZURE_TENANT: ((azure-bot-tenant-id))

pipelines/shared-tasks/deploy-aks-cluster/task.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
+# Copyright 2020-2025 the Pinniped contributors. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
 set -euo pipefail
@@ -15,12 +15,6 @@ az login \
 echo
 echo "Trying to use Kubernetes version $KUBE_VERSION"
 
-# mcr.microsoft.com/azure-cli image doesn't include jq anymore.
-# https://github.com/Azure/azure-cli/issues/29827#issuecomment-2326125769
-# https://github.com/MicrosoftDocs/azure-docs-cli/blob/main/docs-ref-conceptual/release-notes-azure-cli.md
-# But it does seem to include openssl!
-tdnf install jq --assumeyes
-
 # Look up the latest AKS Kubernetes version corresponding to $KUBE_VERSION.
 AKS_VERSIONS="$(az aks get-versions --location "$AZURE_REGION" -o json \
   | jq -r '.values[].patchVersions|keys' \

pipelines/shared-tasks/remove-orphaned-aks-clusters/task.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 the Pinniped contributors. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Sometimes something goes wrong with a AKS test job's cleanup and a
+# AKS cluster gets orphaned, meaning that it is still running but no
+# CI job is aware to clean it up.
+#
+# Find and delete all orphaned AKS clusters by deleting those which:
+# 1. Are running in Azure with a name that indicates that it was auto-created for testing,
+# 2. And are older than some number of hours since their creation time.
+#
+# Params are AZURE_TENANT, AZURE_USERNAME, AZURE_PASSWORD, AZURE_SUBSCRIPTION_ID, AZURE_RESOURCE_GROUP.
+
+set -euo pipefail
+
+# Login.
+az login \
+  --service-principal \
+  --tenant "$AZURE_TENANT" \
+  --username "$AZURE_USERNAME" \
+  --password "$AZURE_PASSWORD"
+
+# List all resources in the subscription. Using this API because it reveals the creation timestamp of every resource,
+# including the AKS clusters. Querying one specific AKS cluster does not return the creation timestamp. :(
+az rest \
+  --method GET \
+  --url "https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION_ID}/resources" \
+  --url-parameters api-version=2024-08-01 \$expand=createdTime >all-resources.json
+
+# Filter resources by clusters in the expected resource group.
+# Write another file where each line is a cluster name, followed by a space, followed by its creation time.
+cat all-resources.json |
+  jq -r ".value.[] | select(.type == \"Microsoft.ContainerService/managedClusters\") | select(.id | contains(\"/resourceGroups/${AZURE_RESOURCE_GROUP}/\")) | \"\(.name) \(.createdTime)\"" >all-clusters.txt
+
+echo "Found all clusters in expected resource group:"
+cat all-clusters.txt
+echo
+
+# Remove clusters with unexpected name formats. They might have been created manually for testing.
+cat all-clusters.txt | grep -E '^aks-[a-f0-9]+ ' >ci-clusters.txt
+
+echo "Only those clusters with expected naming convention:"
+cat ci-clusters.txt
+echo
+
+now_in_seconds_since_epoch=$(date +"%s")
+hours_ago_to_delete=2
+clusters_to_remove=()
+
+# Loop over each line in the file. Decide which clusters are too old.
+while IFS="" read -r line || [ -n "$line" ]; do
+  cluster_name=$(echo "$line" | cut -d ' ' -f1)
+  creation_time=$(echo "$line" | cut -d ' ' -f2)
+  # UTC date format example: 2025-01-03T20:13:02.5855661Z
+  # Note that this date command may not work on MacOS.
+  creation_time_seconds_since_epoch=$(date -u -d "$creation_time" '+%s')
+  if (($((now_in_seconds_since_epoch - creation_time_seconds_since_epoch)) > $((hours_ago_to_delete * 60 * 60)))); then
+    clusters_to_remove+=("$cluster_name")
+    echo "$cluster_name $creation_time (older than $hours_ago_to_delete hours)"
+  else
+    echo "$cluster_name $creation_time (less than $hours_ago_to_delete hours old)"
+  fi
+done <ci-clusters.txt
+
+# Remove any clusters that we decided are too old above.
+echo
+if [[ ${#clusters_to_remove[@]} -eq 0 ]]; then
+  echo "No old orphaned AKS clusters found to remove."
+else
+  echo "Planned to remove ${#clusters_to_remove[@]} AKS clusters(s) which are older than $hours_ago_to_delete hours: ${clusters_to_remove[*]} ..."
+
+  for cluster_name in "${clusters_to_remove[@]}"; do
+    echo "Removing cluster $cluster_name ..."
+    az aks delete --name "$cluster_name" --resource-group "$AZURE_RESOURCE_GROUP" --yes
+  done
+fi
+
+echo
+echo "Done!"

pipelines/shared-tasks/remove-orphaned-aks-clusters/task.yml

@@ -0,0 +1,16 @@
+# Copyright 2025 the Pinniped contributors. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+platform: linux
+inputs:
+  - name: pinniped-ci
+outputs:
+params:
+  AZURE_TENANT:
+  AZURE_USERNAME:
+  AZURE_PASSWORD:
+  AZURE_SUBSCRIPTION_ID:
+  AZURE_RESOURCE_GROUP:
+run:
+  path: pinniped-ci/pipelines/shared-tasks/remove-orphaned-aks-clusters/task.sh