From 9a420a69b07d0e4b730abd2ec37823c1082ef8b1 Mon Sep 17 00:00:00 2001 From: Juan Celhay Date: Mon, 3 Aug 2026 14:34:47 -0400 Subject: [PATCH] Add pre and post deploy steps to Cloud Deploy delivery pipeline (#3187) * Fix image replacement in cd (#3186) * read sql jobs from ar * revert release change * flatten file path for sql jobs * no source to sql command * add automation to pipeline * fix automation * fix replica seize for backend and console in partial phases --- release/cloudbuild-clouddeploy.yaml | 1 + release/cloudbuild-nomulus.yaml | 2 +- release/cloudbuild-release.yaml | 8 +++- release/clouddeploy/README.md | 2 +- release/clouddeploy/delivery-pipeline.yaml | 53 ++++++++++++++-------- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/release/cloudbuild-clouddeploy.yaml b/release/cloudbuild-clouddeploy.yaml index 4d1fdb1ff..27b40f5b4 100644 --- a/release/cloudbuild-clouddeploy.yaml +++ b/release/cloudbuild-clouddeploy.yaml @@ -84,6 +84,7 @@ steps: sed -i "s|artifactStorage: artifactStorage|artifactStorage: $artifact_storage|" "$target_file" sed -i "s|serviceAccount: serviceAccount|serviceAccount: $service_account|" "$target_file" + sed -i "s|serviceAccount: serviceAccount|serviceAccount: $service_account|" release/clouddeploy/delivery-pipeline.yaml sed -i "s|cluster: cluster|cluster: $cluster_val|" "$target_file" sed -i "s|workerPool: workerPool|workerPool: $worker_pool|" "$target_file" fi diff --git a/release/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml index dc38dbc39..54e748030 100644 --- a/release/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -203,7 +203,7 @@ steps: --images="gcr.io/${PROJECT_ID}/nomulus=gcr.io/${PROJECT_ID}/nomulus@${nomulus_digest}" \ --source=. \ --skaffold-file=release/clouddeploy/skaffold.yaml \ - --deploy-parameters="deployed_image=gcr.io/${PROJECT_ID}/nomulus@${nomulus_digest},base_image=us-docker.pkg.dev/${PROJECT_ID}/gcr.io/nomulus" + --deploy-parameters="deployed_image=gcr.io/${PROJECT_ID}/nomulus@${nomulus_digest},base_image=us-docker.pkg.dev/${PROJECT_ID}/gcr.io/nomulus,tag_name=${TAG_NAME},project_id=${PROJECT_ID}" # The tarballs and jars to upload to GCS. artifacts: objects: diff --git a/release/cloudbuild-release.yaml b/release/cloudbuild-release.yaml index 7c8405825..2955c23eb 100644 --- a/release/cloudbuild-release.yaml +++ b/release/cloudbuild-release.yaml @@ -236,11 +236,17 @@ steps: # partial phase manifests for stage in 1 5 do + if [ ${service} == backend ] || [ ${service} == console ] + then + replicas=1 + else + replicas=${stage} + fi awk 'NR==1,/^---$/ {if ($0 != "---") print}' ./jetty/kubernetes/nomulus-${env}-${service}.yaml | \ sed s/name:\ ${service}/name:\ ${service}-partial-phase/g | \ sed s/service:\ ${service}/deployment:\ ${service}-partial-phase/g | \ sed s/value:\ ${service}/value:\ ${service}-partial-phase/g | \ - sed "/^spec:$/a\ replicas: ${stage}" \ + sed "/^spec:$/a\ replicas: ${replicas}" \ > ./jetty/kubernetes/nomulus-${env}-${service}-partial-phase-${stage}.yaml done # gateway diff --git a/release/clouddeploy/README.md b/release/clouddeploy/README.md index b3c7dca92..75d5aa780 100644 --- a/release/clouddeploy/README.md +++ b/release/clouddeploy/README.md @@ -5,7 +5,7 @@ This directory contains the Google Cloud Deploy configuration files for the Nomu ## Files ### `delivery-pipeline.yaml` -Defines the `DeliveryPipeline` resource named `deploy-nomulus`. It sets up the serial pipeline for rolling out changes to different targets. +Defines the `DeliveryPipeline` resource named `deploy-nomulus` and its associated `Automation` resource (`deploy-nomulus/auto-advance-canary`). It sets up the serial pipeline for rolling out changes to different targets and automatically advancing canary rollouts. ### Target Configurations (e.g., `crash-target.yaml`) Files matching this format define the `Target` resources for Cloud Deploy. They specify the GKE cluster and other environment-specific settings for deployment. diff --git a/release/clouddeploy/delivery-pipeline.yaml b/release/clouddeploy/delivery-pipeline.yaml index b899be95d..3c57c2f14 100644 --- a/release/clouddeploy/delivery-pipeline.yaml +++ b/release/clouddeploy/delivery-pipeline.yaml @@ -14,17 +14,6 @@ serialPipeline: - phaseId: "canary-1" profiles: ["crash-partial-phase-1"] percentage: 10 - predeploy: - tasks: - - type: container - image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable - env: - TARGET_ID: ${{ target.id }} - command: ["/bin/bash"] - args: - - "-c" - - | - gcloud builds submit --config=release/cloudbuild-schema-verify-${TARGET_ID}.yaml analysis: # 10 minutes. duration: 600s @@ -55,17 +44,20 @@ serialPipeline: args: - "-c" - | - gcloud artifacts docker tags add $DEPLOYED_IMAGE \ - ${BASE_IMAGE}:live-cd-${TARGET_ID} + gcloud container images add-tag $DEPLOYED_IMAGE \ + ${BASE_IMAGE}:live-cd-${TARGET_ID} --quiet - type: container image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable env: TARGET_ID: ${{ target.id }} + TAG_NAME: ${{ deploy_params['tag_name'] }} + PROJECT_ID: ${{ deploy_params['project_id'] }} command: ["/bin/bash"] args: - "-c" - | - gcloud builds submit --config=release/cloudbuild-schema-deploy-${TARGET_ID}.yaml + gcloud storage cp gs://${PROJECT_ID}-deploy/${TAG_NAME}/cloudbuild-schema-deploy-${TARGET_ID}.yaml . + gcloud builds submit --no-source --config=cloudbuild-schema-deploy-${TARGET_ID}.yaml analysis: # 10 minutes. duration: 600s @@ -86,11 +78,14 @@ serialPipeline: image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable env: TARGET_ID: ${{ target.id }} + TAG_NAME: ${{ deploy_params['tag_name'] }} + PROJECT_ID: ${{ deploy_params['project_id'] }} command: ["/bin/bash"] args: - "-c" - | - gcloud builds submit --config=release/cloudbuild-schema-verify-${TARGET_ID}.yaml + gcloud storage cp gs://${PROJECT_ID}-deploy/${TAG_NAME}/cloudbuild-schema-verify-${TARGET_ID}.yaml . + gcloud builds submit --no-source --config=cloudbuild-schema-verify-${TARGET_ID}.yaml analysis: # 10 minutes. duration: 600s @@ -121,20 +116,42 @@ serialPipeline: args: - "-c" - | - gcloud artifacts docker tags add $DEPLOYED_IMAGE \ - ${BASE_IMAGE}:live-cd-${TARGET_ID} + gcloud container images add-tag $DEPLOYED_IMAGE \ + ${BASE_IMAGE}:live-cd-${TARGET_ID} --quiet - type: container image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable env: TARGET_ID: ${{ target.id }} + TAG_NAME: ${{ deploy_params['tag_name'] }} + PROJECT_ID: ${{ deploy_params['project_id'] }} command: ["/bin/bash"] args: - "-c" - | - gcloud builds submit --config=release/cloudbuild-schema-deploy-${TARGET_ID}.yaml + gcloud storage cp gs://${PROJECT_ID}-deploy/${TAG_NAME}/cloudbuild-schema-deploy-${TARGET_ID}.yaml . + gcloud builds submit --no-source --config=cloudbuild-schema-deploy-${TARGET_ID}.yaml analysis: # 10 minutes. duration: 600s googleCloud: alertPolicyChecks: sandboxStableDeploymentAlertPolicyChecks +--- +apiVersion: deploy.cloud.google.com/v1 +kind: Automation +metadata: + name: deploy-nomulus/auto-advance-canary +description: Automatically advances rollouts through canary-1 phase after successful deployment and analysis. +# Placeholder: Replace with project service account. +serviceAccount: serviceAccount +selector: + targets: + - id: crash + - id: sandbox +rules: +- advanceRolloutRule: + id: advance-canary-phases + sourcePhases: + - "canary-1" + wait: 0m +