From cbe99119a256dbd5299abd60bf82daf54926aab9 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 3 Jan 2025 14:22:36 -0800 Subject: [PATCH] add job to automatically find and remove orphaned AKS clusters --- pipelines/concourse-workers/pipeline.yml | 33 +++++++- pipelines/main/pipeline.yml | 1 + .../shared-tasks/deploy-aks-cluster/task.sh | 8 +- .../remove-orphaned-aks-clusters/task.sh | 81 +++++++++++++++++++ .../remove-orphaned-aks-clusters/task.yml | 16 ++++ 5 files changed, 131 insertions(+), 8 deletions(-) create mode 100755 pipelines/shared-tasks/remove-orphaned-aks-clusters/task.sh create mode 100644 pipelines/shared-tasks/remove-orphaned-aks-clusters/task.yml diff --git a/pipelines/concourse-workers/pipeline.yml b/pipelines/concourse-workers/pipeline.yml index f0f18080f..e13c61164 100644 --- a/pipelines/concourse-workers/pipeline.yml +++ b/pipelines/concourse-workers/pipeline.yml @@ -1,4 +1,4 @@ -# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2025 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 display: @@ -28,6 +28,14 @@ meta: GCP_SERVICE_ACCOUNT: ((gke-test-pool-manager-username)) GCP_JSON_KEY: ((gke-test-pool-manager-json-key)) + # Azure account info and which resource group the clusters should be created in and deleted from. + azure_account_params: &azure_account_params + AZURE_SUBSCRIPTION_ID: ((azure-bot-subscription-id)) + AZURE_TENANT: ((azure-bot-tenant-id)) + AZURE_RESOURCE_GROUP: pinniped-ci + AZURE_USERNAME: ((azure-bot-app-id)) + AZURE_PASSWORD: ((azure-bot-password)) + resources: - name: pinniped-ci @@ -56,6 +64,13 @@ resources: repository: google/cloud-sdk tag: slim + - name: aks-deployer-image + type: registry-image + icon: docker + check_every: 5m + source: + repository: mcr.microsoft.com/azure-cli + - name: hourly type: time icon: calendar-clock @@ -146,3 +161,19 @@ jobs: image: gcloud-image params: <<: *gke_account_params + + - name: remove-orphaned-aks-clusters + public: true # all logs are publicly visible + plan: + - in_parallel: + - get: pinniped-ci + - get: aks-deployer-image + - get: hourly + trigger: true + - task: remove-orphaned-aks-clusters + attempts: 2 + timeout: 25m + file: pinniped-ci/pipelines/shared-tasks/remove-orphaned-aks-clusters/task.yml + image: gcloud-image + params: + <<: *azure_account_params diff --git a/pipelines/main/pipeline.yml b/pipelines/main/pipeline.yml index df2b3a377..786319f94 100644 --- a/pipelines/main/pipeline.yml +++ b/pipelines/main/pipeline.yml @@ -103,6 +103,7 @@ meta: AWS_SECRET_ACCESS_KEY: ((aws-concourse-ci-iam-key-secret)) AWS_ROLE_ARN: ((aws-concourse-ci-role-arn)) + # Azure account info and which resource group the clusters should be created in and deleted from. azure_account_params: &azure_account_params AZURE_REGION: westus2 AZURE_TENANT: ((azure-bot-tenant-id)) diff --git a/pipelines/shared-tasks/deploy-aks-cluster/task.sh b/pipelines/shared-tasks/deploy-aks-cluster/task.sh index d1ee433a6..f7e03c0b0 100755 --- a/pipelines/shared-tasks/deploy-aks-cluster/task.sh +++ b/pipelines/shared-tasks/deploy-aks-cluster/task.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2020-2024 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2025 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -15,12 +15,6 @@ az login \ echo echo "Trying to use Kubernetes version $KUBE_VERSION" -# mcr.microsoft.com/azure-cli image doesn't include jq anymore. -# https://github.com/Azure/azure-cli/issues/29827#issuecomment-2326125769 -# https://github.com/MicrosoftDocs/azure-docs-cli/blob/main/docs-ref-conceptual/release-notes-azure-cli.md -# But it does seem to include openssl! -tdnf install jq --assumeyes - # Look up the latest AKS Kubernetes version corresponding to $KUBE_VERSION. AKS_VERSIONS="$(az aks get-versions --location "$AZURE_REGION" -o json \ | jq -r '.values[].patchVersions|keys' \ diff --git a/pipelines/shared-tasks/remove-orphaned-aks-clusters/task.sh b/pipelines/shared-tasks/remove-orphaned-aks-clusters/task.sh new file mode 100755 index 000000000..e1828f4d0 --- /dev/null +++ b/pipelines/shared-tasks/remove-orphaned-aks-clusters/task.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# Copyright 2025 the Pinniped contributors. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sometimes something goes wrong with a AKS test job's cleanup and a +# AKS cluster gets orphaned, meaning that it is still running but no +# CI job is aware to clean it up. +# +# Find and delete all orphaned AKS clusters by deleting those which: +# 1. Are running in Azure with a name that indicates that it was auto-created for testing, +# 2. And are older than some number of hours since their creation time. +# +# Params are AZURE_TENANT, AZURE_USERNAME, AZURE_PASSWORD, AZURE_SUBSCRIPTION_ID, AZURE_RESOURCE_GROUP. + +set -euo pipefail + +# Login. +az login \ + --service-principal \ + --tenant "$AZURE_TENANT" \ + --username "$AZURE_USERNAME" \ + --password "$AZURE_PASSWORD" + +# List all resources in the subscription. Using this API because it reveals the creation timestamp of every resource, +# including the AKS clusters. Querying one specific AKS cluster does not return the creation timestamp. :( +az rest \ + --method GET \ + --url "https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION_ID}/resources" \ + --url-parameters api-version=2024-08-01 \$expand=createdTime >all-resources.json + +# Filter resources by clusters in the expected resource group. +# Write another file where each line is a cluster name, followed by a space, followed by its creation time. +cat all-resources.json | + jq -r ".value.[] | select(.type == \"Microsoft.ContainerService/managedClusters\") | select(.id | contains(\"/resourceGroups/${AZURE_RESOURCE_GROUP}/\")) | \"\(.name) \(.createdTime)\"" >all-clusters.txt + +echo "Found all clusters in expected resource group:" +cat all-clusters.txt +echo + +# Remove clusters with unexpected name formats. They might have been created manually for testing. +cat all-clusters.txt | grep -E '^aks-[a-f0-9]+ ' >ci-clusters.txt + +echo "Only those clusters with expected naming convention:" +cat ci-clusters.txt +echo + +now_in_seconds_since_epoch=$(date +"%s") +hours_ago_to_delete=2 +clusters_to_remove=() + +# Loop over each line in the file. Decide which clusters are too old. +while IFS="" read -r line || [ -n "$line" ]; do + cluster_name=$(echo "$line" | cut -d ' ' -f1) + creation_time=$(echo "$line" | cut -d ' ' -f2) + # UTC date format example: 2025-01-03T20:13:02.5855661Z + # Note that this date command may not work on MacOS. + creation_time_seconds_since_epoch=$(date -u -d "$creation_time" '+%s') + if (($((now_in_seconds_since_epoch - creation_time_seconds_since_epoch)) > $((hours_ago_to_delete * 60 * 60)))); then + clusters_to_remove+=("$cluster_name") + echo "$cluster_name $creation_time (older than $hours_ago_to_delete hours)" + else + echo "$cluster_name $creation_time (less than $hours_ago_to_delete hours old)" + fi +done