diff --git a/release/cloudbuild-delete-canary.yaml b/release/cloudbuild-delete-canary.yaml new file mode 100644 index 000000000..3589328f3 --- /dev/null +++ b/release/cloudbuild-delete-canary.yaml @@ -0,0 +1,46 @@ +# This will delete canary GAE versions named "nomulus". +# +# For reasons unknown, Spinnaker occasionally gets stuck when deploying to GAE +# canary, and the fix is to manually delete the canary versions before the +# deployment. +# +# To manually trigger a build on GCB, run: +# gcloud builds submit --config=cloudbuild-delete-canary.yaml \ +# --substitutions=_ENV=[ENV] .. +# +# To trigger a build automatically, follow the instructions below and add a trigger: +# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds +# +steps: +# Pull the credential for nomulus tool. +- name: 'gcr.io/$PROJECT_ID/builder:latest' + entrypoint: /bin/bash + args: + - -c + - | + set -e + gcloud secrets versions access latest \ + --secret nomulus-tool-cloudbuild-credential > tool-credential.json +# Delete unused GAE versions. +- name: 'gcr.io/$PROJECT_ID/builder:latest' + entrypoint: /bin/bash + args: + - -c + - | + if [ ${_ENV} == production ] + then + project_id="domain-registry" + else + project_id="domain-registry-${_ENV}" + fi + + gcloud auth activate-service-account --key-file=tool-credential.json + + for service in default pubapi backend bsa tools console + do + gcloud app versions delete nomulus --service=$service \ + --project=$project_id --quiet; + done +timeout: 3600s +options: + machineType: 'N1_HIGHCPU_8' diff --git a/release/cloudbuild-release.yaml b/release/cloudbuild-release.yaml index 93975e112..3dc8e7071 100644 --- a/release/cloudbuild-release.yaml +++ b/release/cloudbuild-release.yaml @@ -88,6 +88,7 @@ steps: sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete.yaml + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete-canary.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-restart-proxies.yaml sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync-and-tag.yaml @@ -100,6 +101,8 @@ steps: > release/cloudbuild-deploy-gke-${environment}.yaml sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete.yaml \ > release/cloudbuild-delete-${environment}.yaml + sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete-canary.yaml \ + > release/cloudbuild-delete-canary-${environment}.yaml sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml \ > release/cloudbuild-restart-proxies-${environment}.yaml sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml | \