Integrate AR exit gate promotion step into nomulus release (#3229)

* ar exit gate promotion steps

* move promote step to script

* promote only script

* add push for latest tag

* explicit call to bash to execute promote script

* comments

* echo failures

* review

release/cloudbuild-nomulus.yaml

@@ -53,42 +53,14 @@ steps:
     ./gradlew :jetty:buildNomulusImage :proxy:buildProxyImage :core:buildToolImage\
       -PmavenUrl=gcs://domain-registry-maven-repository/maven \
       -PpluginsUrl=gcs://domain-registry-maven-repository/plugins
-    docker tag nomulus gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME}
+    docker tag nomulus us-docker.pkg.dev/${PROJECT_ID}/staging/nomulus:${TAG_NAME}
-    docker tag nomulus gcr.io/${PROJECT_ID}/nomulus:latest
+    docker tag nomulus us-docker.pkg.dev/${PROJECT_ID}/staging/nomulus:latest
-    docker push gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME}
+    docker tag proxy us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:${TAG_NAME}
-    docker push gcr.io/${PROJECT_ID}/nomulus:latest
+    docker tag proxy us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:latest
-    docker tag proxy gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}
-    docker tag proxy gcr.io/${PROJECT_ID}/proxy:latest
-    docker push gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}
-    docker push gcr.io/${PROJECT_ID}/proxy:latest
     docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME}
     docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:latest
     docker push gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME}
     docker push gcr.io/${PROJECT_ID}/nomulus-tool:latest
-# Sign nomulus and proxy images.
-- name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  entrypoint: /bin/bash
-  args:
-  - -c
-  - |
-    nomulus_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/nomulus \
-    --format="get(digest)" --filter="tags = ${TAG_NAME}")
-    echo "$nomulus_digest" > /workspace/nomulus_digest
-    proxy_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \
-    --format="get(digest)" --filter="tags = ${TAG_NAME}")
-    echo "$proxy_digest" > /workspace/proxy_digest
-    gcloud --project=${PROJECT_ID} beta container binauthz attestations \
-      sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/nomulus@$nomulus_digest \
-      --attestor=build-attestor --attestor-project=${PROJECT_ID} \
-      --keyversion-project=${PROJECT_ID} --keyversion-location=global \
-      --keyversion-keyring=attestor-keys --keyversion-key=signing \
-      --keyversion=1
-    gcloud --project=${PROJECT_ID} beta container binauthz attestations \
-      sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$proxy_digest \
-      --attestor=build-attestor --attestor-project=${PROJECT_ID} \
-      --keyversion-project=${PROJECT_ID} --keyversion-location=global \
-      --keyversion-keyring=attestor-keys --keyversion-key=signing \
-      --keyversion=1
 # Get the tool image digest and substitute in the digest in other GCB files.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
@@ -177,35 +149,6 @@ steps:
     cp db/build/libs/schema.jar output/
     cp core/build/libs/nomulus-public.jar output/
     cp core/build/libs/nomulus-tests-alldeps.jar output/
-# Create a release in Cloud Deploy to trigger the deployment pipeline
-- name: 'gcr.io/${PROJECT_ID}/builder:latest'
-  entrypoint: /bin/bash
-  args:
-  - -c
-  - |
-    set -e
-    echo "============================================="
-    echo "Triggering Google Cloud Deploy Release"
-    echo "============================================="
-    echo "Tag Name: ${TAG_NAME}"
-    echo "Project ID: ${PROJECT_ID}"
-    pipeline="deploy-nomulus"
-    region="us-central1"
-    # Release names must consist of lowercase letters, numbers, and hyphens.
-    release_name=$(echo "${TAG_NAME}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
-    echo "Release Name: $release_name"
-    echo "============================================="
-    # Read the pre-fetched image digest from the workspace file
-    nomulus_digest=$(cat /workspace/nomulus_digest)
-    proxy_digest=$(cat /workspace/proxy_digest)
-    gcloud deploy releases create "$release_name" \
-      --delivery-pipeline="$pipeline" \
-      --region="$region" \
-      --project=${PROJECT_ID} \
-      --images="gcr.io/${PROJECT_ID}/nomulus=gcr.io/${PROJECT_ID}/nomulus@${nomulus_digest},gcr.io/${PROJECT_ID}/proxy=gcr.io/${PROJECT_ID}/proxy@${proxy_digest}" \
-      --source=. \
-      --skaffold-file=release/clouddeploy/skaffold.yaml \
-      --deploy-parameters="deployed_image=gcr.io/${PROJECT_ID}/nomulus@${nomulus_digest},base_image=us-docker.pkg.dev/${PROJECT_ID}/gcr.io/nomulus,tag_name=${TAG_NAME},project_id=${PROJECT_ID}"
 # The tarballs and jars to upload to GCS.
 artifacts:
   objects:
@@ -228,11 +171,12 @@ artifacts:
     - 'release/cloudbuild-restart-proxies-*.yaml'
     - 'jetty/kubernetes/*.yaml'
     - 'jetty/kubernetes/gateway/*.yaml'
-# The images are already uploaded, but we still need to include them there so that
+# Images to upload to staging repository for build provenance generation.
-# the GCB pubsub message contains them (for Spinnaker to consume).
 images:
-  - 'gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME}'
+  - 'us-docker.pkg.dev/${PROJECT_ID}/staging/nomulus:${TAG_NAME}'
-  - 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}'
+  - 'us-docker.pkg.dev/${PROJECT_ID}/staging/nomulus:latest'
+  - 'us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:${TAG_NAME}'
+  - 'us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:latest'
 timeout: 7200s
 options:
   machineType: 'E2_HIGHCPU_32'

release/cloudbuild-proxy.yaml

@@ -18,37 +18,20 @@ steps:
   - :proxy:buildProxyImage
   - -PmavenUrl=gcs://domain-registry-maven-repository/maven
   - -PpluginsUrl=gcs://domain-registry-maven-repository/plugins
-# Tag and push the image. We can't let Cloud Build's default processing do that for us
+# Tag the image for staging repository.
-# because we need to push the image before we can sign it in the following step.
 - name: 'gcr.io/${PROJECT_ID}/builder:latest'
   entrypoint: /bin/bash
   args:
   - -c
   - |
     set -e
-    docker tag proxy gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}
+    docker tag proxy us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:${TAG_NAME}
-    docker tag proxy gcr.io/${PROJECT_ID}/proxy:latest
+    docker tag proxy us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:latest
-    docker push gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}
-    docker push gcr.io/${PROJECT_ID}/proxy:latest
   dir: 'proxy'
-# Get the image digest, sign it and substitute in the digest in the tagging yaml file.
+# Images to upload to staging repository for build provenance generation.
-- name: 'gcr.io/${PROJECT_ID}/builder:latest'
+images:
-  entrypoint: /bin/bash
+  - 'us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:${TAG_NAME}'
-  args:
+  - 'us-docker.pkg.dev/${PROJECT_ID}/staging/proxy:latest'
-  - -c
-  - |
-    set -e
-    digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \
-      --format="get(digest)" --filter="tags = ${TAG_NAME}")
-    gcloud --project=${PROJECT_ID} alpha container binauthz attestations \
-      sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$digest \
-      --attestor=build-attestor --attestor-project=${PROJECT_ID} \
-      --keyversion-project=${PROJECT_ID} --keyversion-location=global \
-      --keyversion-keyring=attestor-keys --keyversion-key=signing \
-      --keyversion=1
-# Images to upload to GCR. Even though the image has already been uploaded, we still include it
-# here so that the GCB pubsub message contains it (for Spinnaker to consume).
-images: ['gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}']
 # Config files to upload to GCS.
 artifacts:
   objects:

release/cloudbuild-release.yaml

@@ -319,11 +319,12 @@ steps:
     rm ${gradle_bin}
     sed -i s%services.gradle.org/distributions%storage.googleapis.com/${gcs_loc}% \
       gradle/wrapper/gradle-wrapper.properties
-# Conditionally trigger the appropriate build based on the tag format.
+# Conditionally trigger the appropriate build based on the tag format and promote artifacts.
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   env:
     - 'TAG_NAME=${TAG_NAME}'
+    - 'PROJECT_ID=${PROJECT_ID}'
   args:
   - -c
   - |
@@ -332,14 +333,55 @@ steps:
     if [[ "$TAG_NAME" =~ ^nomulus-20[0-9]{2}[0-1][0-9][0-3][0-9]-RC[0-9]{2}$ ]]; then
       echo "Tag format matches a nomulus release. Triggering nomulus build..."
       gcloud builds submit . --config=release/cloudbuild-nomulus.yaml --substitutions="TAG_NAME=$TAG_NAME"
+      echo "Promoting and signing nomulus release artifacts..."
+      bash ./release/promote_artifacts.sh "nomulus" "$TAG_NAME" "$PROJECT_ID"
     # Check for a proxy release tag (e.g., "proxy-v1.2.3")
     elif [[ "$TAG_NAME" =~ ^proxy-20[0-9]{2}[0-1][0-9][0-3][0-9]-RC[0-9]{2}$ ]]; then
       echo "Tag format matches a proxy release. Triggering proxy build..."
       gcloud builds submit . --config=release/cloudbuild-proxy.yaml --substitutions="TAG_NAME=$TAG_NAME"
+      echo "Promoting and signing proxy release artifacts..."
+      bash ./release/promote_artifacts.sh "proxy" "$TAG_NAME" "$PROJECT_ID"
     else
       echo "Tag format '$TAG_NAME' does not match a known release type. Exiting."
       exit 1
     fi
+# Create a release in Cloud Deploy to trigger the deployment pipeline
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  env:
+    - 'TAG_NAME=${TAG_NAME}'
+    - 'PROJECT_ID=${PROJECT_ID}'
+  args:
+  - -c
+  - |
+    set -e
+    if [[ "$TAG_NAME" =~ ^nomulus-20[0-9]{2}[0-1][0-9][0-3][0-9]-RC[0-9]{2}$ ]]; then
+      echo "============================================="
+      echo "Triggering Google Cloud Deploy Release"
+      echo "============================================="
+      echo "Tag Name: ${TAG_NAME}"
+      echo "Project ID: ${PROJECT_ID}"
+      pipeline="deploy-nomulus"
+      region="us-central1"
+      # Release names must consist of lowercase letters, numbers, and hyphens.
+      release_name=$(echo "${TAG_NAME}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
+      echo "Release Name: $release_name"
+      echo "============================================="
+      nomulus_digest=$(gcloud artifacts docker images describe \
+        "us-docker.pkg.dev/${PROJECT_ID}/gcr.io/nomulus:${TAG_NAME}" \
+        --format="value(image_summary.digest)")
+      proxy_digest=$(gcloud artifacts docker images describe \
+        "us-docker.pkg.dev/${PROJECT_ID}/gcr.io/proxy:${TAG_NAME}" \
+        --format="value(image_summary.digest)")
+      gcloud deploy releases create "$release_name" \
+        --delivery-pipeline="$pipeline" \
+        --region="$region" \
+        --project=${PROJECT_ID} \
+        --images="gcr.io/${PROJECT_ID}/nomulus=gcr.io/${PROJECT_ID}/nomulus@${nomulus_digest},gcr.io/${PROJECT_ID}/proxy=gcr.io/${PROJECT_ID}/proxy@${proxy_digest}" \
+        --source=. \
+        --skaffold-file=release/clouddeploy/skaffold.yaml \
+        --deploy-parameters="deployed_image=gcr.io/${PROJECT_ID}/nomulus@${nomulus_digest},base_image=us-docker.pkg.dev/${PROJECT_ID}/gcr.io/nomulus,tag_name=${TAG_NAME},project_id=${PROJECT_ID}"
+    fi
 # Run the BEAM smoke test, using the builder and pipeline image just created
 - name: 'gcr.io/$PROJECT_ID/builder:latest'
   entrypoint: /bin/bash

release/promote_artifacts.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+# Copyright 2026 The Nomulus Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script handles post-build promotion and BinAuthz signing:
+# 1. Obtains the image digests from the staging repository.
+# 2. Promotes the images from staging to gcr.io via the Artifact Registry
+#    promoteArtifact API (evaluating BCID exit gate policy and attaching VSA).
+# 3. Signs the promoted gcr.io images with Binary Authorization.
+#
+# Usage:
+#   release/promote_artifacts.sh <release_type> <tag_name> <project_id>
+# where <release_type> is "nomulus" or "proxy".
+
+set -e
+
+if [[ $# -ne 3 ]]; then
+  echo "Usage: $0 <nomulus|proxy> <tag_name> <project_id>"
+  exit 1
+fi
+
+RELEASE_TYPE="$1"
+TAG_NAME="$2"
+PROJECT_ID="$3"
+
+LOCATION="us"
+DEST_REPO="gcr.io"
+SRC_REPO="staging"
+ENDPOINT="https://artifactregistry.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/repositories/${DEST_REPO}:promoteArtifact"
+
+promote_artifact() {
+  local pkg="$1"
+  local digest="$2"
+
+  echo "================================================================================"
+  echo "Promoting ${pkg}:${TAG_NAME} (${digest}) to ${DEST_REPO} via Artifact Registry"
+  echo "================================================================================"
+
+  local token
+  token=$(gcloud auth print-access-token)
+
+  local payload
+  payload=$(printf '{"source_repository":"projects/%s/locations/%s/repositories/%s","source_version":"projects/%s/locations/%s/repositories/%s/packages/%s/versions/%s","include_all_tags":true,"overwrite_tags":true,"attachment_behavior":"PUBLIC_BCID_VSA_ONLY"}' \
+    "${PROJECT_ID}" "${LOCATION}" "${SRC_REPO}" \
+    "${PROJECT_ID}" "${LOCATION}" "${SRC_REPO}" "${pkg}" "${digest}")
+
+  local operation_json
+  operation_json=$(curl -s --no-progress-meter \
+    -H "Authorization: Bearer ${token}" \
+    -H "Content-Type: application/json" \
+    -X POST "${ENDPOINT}" \
+    -d "${payload}")
+
+  local operation_name
+  if ! operation_name=$(echo "${operation_json}" | python3 -c \
+      "import sys, json; d=json.load(sys.stdin); \
+       sys.exit(1) if 'error' in d or not d.get('name') else print(d['name'])" \
+      2>/dev/null); then
+    echo "Failed to initiate promotion for ${pkg}: ${operation_json}"
+    exit 1
+  fi
+
+  echo "Promotion operation started: ${operation_name}"
+  echo "Polling operation status until completion..."
+
+  local max_attempts=60
+  local status_output=""
+  local status_json=""
+  for ((attempt = 1; attempt <= max_attempts; attempt++)); do
+    if ! status_output=$(gcloud artifacts operations describe "${operation_name}" \
+        --project="${PROJECT_ID}" \
+        --location="${LOCATION}" \
+        --format="json" 2>&1) || [[ -z "${status_output}" ]]; then
+      echo "Warning: Failed to query operation status; retrying in 5s..."
+      if [[ -n "${status_output}" ]]; then
+        echo "${status_output}"
+      fi
+      sleep 5
+      continue
+    fi
+    status_json="${status_output}"
+
+    local result
+    result=$(echo "${status_json}" | python3 -c "import sys, json; d=json.load(sys.stdin); print('IN_PROGRESS' if not d.get('done') else ('ERROR: ' + json.dumps(d['error']) if 'error' in d else 'SUCCESS'))" 2>/dev/null || echo "RETRY")
+
+    if [[ "${result}" == "SUCCESS" ]]; then
+      echo "================================================================================"
+      echo "Artifact promotion succeeded! BCID VSA attached to ${DEST_REPO}/${pkg}."
+      echo "================================================================================"
+      return 0
+    elif [[ "${result}" =~ ^ERROR: ]]; then
+      echo "================================================================================"
+      echo "ERROR: Artifact promotion failed BCID policy evaluation or execution for ${pkg}:"
+      echo "${result}"
+      echo "================================================================================"
+      exit 1
+    elif [[ "${result}" == "IN_PROGRESS" || "${result}" == "RETRY" ]]; then
+      echo "Operation in progress... (attempt ${attempt}/${max_attempts}), retrying in 5s..."
+    else
+      echo "Warning: Unknown result '${result}'" \
+        "(attempt ${attempt}/${max_attempts}), retrying in 5s..."
+    fi
+    sleep 5
+  done
+
+  echo "ERROR: Timed out waiting for promotion operation on ${pkg} to complete."
+  if [[ -n "${status_output}" ]]; then
+    echo "Last operation status: ${status_output}"
+  fi
+  exit 1
+}
+
+sign_binauthz() {
+  local image_name="$1"
+  local digest="$2"
+
+  echo "================================================================================"
+  echo "Signing ${image_name}@${digest} with Binary Authorization"
+  echo "================================================================================"
+
+  gcloud --project="${PROJECT_ID}" beta container binauthz attestations \
+    sign-and-create --artifact-url="${DEST_REPO}/${PROJECT_ID}/${image_name}@${digest}" \
+    --attestor=build-attestor --attestor-project="${PROJECT_ID}" \
+    --keyversion-project="${PROJECT_ID}" --keyversion-location=global \
+    --keyversion-keyring=attestor-keys --keyversion-key=signing \
+    --keyversion=1
+}
+
+if [[ "${RELEASE_TYPE}" == "nomulus" ]]; then
+  echo "Retrieving digests from staging for nomulus release..."
+  nomulus_digest=$(gcloud artifacts docker images describe \
+    "${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${SRC_REPO}/nomulus:${TAG_NAME}" \
+    --format="value(image_summary.digest)")
+  proxy_digest=$(gcloud artifacts docker images describe \
+    "${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${SRC_REPO}/proxy:${TAG_NAME}" \
+    --format="value(image_summary.digest)")
+
+  echo "nomulus digest: ${nomulus_digest}"
+  echo "proxy digest:   ${proxy_digest}"
+
+  promote_artifact "nomulus" "${nomulus_digest}"
+  promote_artifact "proxy" "${proxy_digest}"
+
+  sign_binauthz "nomulus" "${nomulus_digest}"
+  sign_binauthz "proxy" "${proxy_digest}"
+
+elif [[ "${RELEASE_TYPE}" == "proxy" ]]; then
+  echo "Retrieving digest from staging for proxy release..."
+  proxy_digest=$(gcloud artifacts docker images describe \
+    "${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${SRC_REPO}/proxy:${TAG_NAME}" \
+    --format="value(image_summary.digest)")
+
+  echo "proxy digest: ${proxy_digest}"
+
+  promote_artifact "proxy" "${proxy_digest}"
+  sign_binauthz "proxy" "${proxy_digest}"
+else
+  echo "Unknown release type: ${RELEASE_TYPE}"
+  exit 1
+fi