Add a GCB job to delete GAE canary versions (#2714)

We've seen this issue happen more often than not recently, where GAE
canary deployment is stuck for about 10 min and the failed. The reason
is not clear, but delete the canary version prior to a deployment always
fixes the issue.

release/cloudbuild-delete-canary.yaml

@@ -0,0 +1,46 @@
+# This will delete canary GAE versions named "nomulus".
+#
+# For reasons unknown, Spinnaker occasionally gets stuck when deploying to GAE
+# canary, and the fix is to manually delete the canary versions before the
+# deployment.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config=cloudbuild-delete-canary.yaml \
+# --substitutions=_ENV=[ENV] ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+steps:
+# Pull the credential for nomulus tool.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gcloud secrets versions access latest \
+      --secret nomulus-tool-cloudbuild-credential > tool-credential.json
+# Delete unused GAE versions.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    if [ ${_ENV} == production ]
+    then
+      project_id="domain-registry"
+    else
+      project_id="domain-registry-${_ENV}"
+    fi
+
+    gcloud auth activate-service-account --key-file=tool-credential.json
+
+    for service in default pubapi backend bsa tools console
+    do
+      gcloud app versions delete nomulus --service=$service \
+        --project=$project_id --quiet;
+    done
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'

release/cloudbuild-release.yaml

@@ -88,6 +88,7 @@ steps:
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete.yaml
+    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete-canary.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-restart-proxies.yaml
     sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml
     sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync-and-tag.yaml
@@ -100,6 +101,8 @@ steps:
         > release/cloudbuild-deploy-gke-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete.yaml \
         > release/cloudbuild-delete-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete-canary.yaml \
+        > release/cloudbuild-delete-canary-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml \
         > release/cloudbuild-restart-proxies-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml | \