mirror of
https://github.com/google/nomulus
synced 2026-02-05 12:31:15 +00:00
464f9aed1f
* Migrate Spec11 pipeline to flex template Unfortunately this PR has turned out to be much bigger than I initially conceived. However this is no good way to separate it out because the changes are intertwined. This PR includes 3 main changes: 1. Change the spec11 pipline to use Dataflow Flex Template. 2. Retire the use of the old JPA layer that relies on credential saved in KMS. 3. Some extensive refactoring to streamline the logic and improve test isolation. * Fix job name and remove projectId from options * Add parameter logs * Set RegistryEnvironment * Remove logging and modify safe browsing API key regex * Rename a test method and rebase * Remove unused Junit extension * Specify job region
108 lines
3.9 KiB
YAML
108 lines
3.9 KiB
YAML
# To run the build locally, install cloud-build-local first.
|
|
# Then run:
|
|
# cloud-build-local --config=cloudbuild-deploy.yaml --dryrun=false \
|
|
# --substitutions=TAG_NAME=[TAG],_ENV=[ENV] ..
|
|
#
|
|
# This will deploy Beam pipelines to GCS for the PROJECT_ID defined in gcloud
|
|
# tool.
|
|
#
|
|
# To manually trigger a build on GCB, run:
|
|
# gcloud builds submit --config=cloudbuild-deploy.yaml \
|
|
# --substitutions=TAG_NAME=[TAG],_ENV=[ENV] ..
|
|
#
|
|
# To trigger a build automatically, follow the instructions below and add a trigger:
|
|
# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
|
|
#
|
|
# Note: to work around issue in Spinnaker's 'Deployment Manifest' stage,
|
|
# variable references must avoid the ${var} format. Valid formats include
|
|
# $var or ${"${var}"}. This file use the former. Since TAG_NAME and _ENV are
|
|
# expanded in the copies sent to Spinnaker, we preserve the brackets around
|
|
# them for safe pattern matching during release.
|
|
# See https://github.com/spinnaker/spinnaker/issues/3028 for more information.
|
|
steps:
|
|
# Pull the credential for nomulus tool.
|
|
- name: 'gcr.io/$PROJECT_ID/builder:latest'
|
|
args:
|
|
- gsutil
|
|
- cp
|
|
- gs://$PROJECT_ID-deploy/secrets/tool-credential.json.enc
|
|
- .
|
|
# Decrypt the credential.
|
|
- name: 'gcr.io/$PROJECT_ID/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
set -e
|
|
cat tool-credential.json.enc | base64 -d | gcloud kms decrypt \
|
|
--ciphertext-file=- --plaintext-file=tool-credential.json \
|
|
--location=global --keyring=nomulus-tool-keyring --key=nomulus-tool-key
|
|
# Deploy the invoicing pipeline to GCS.
|
|
- name: 'gcr.io/$PROJECT_ID/nomulus-tool:latest'
|
|
args:
|
|
- -e
|
|
- ${_ENV}
|
|
- --credential
|
|
- tool-credential.json
|
|
- deploy_invoicing_pipeline
|
|
# Deploy the GAE config files.
|
|
# First authorize the gcloud tool to use the credential json file, then
|
|
# download and unzip the tarball that contains the relevant config files
|
|
- name: 'gcr.io/$PROJECT_ID/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
set -e
|
|
gcloud auth activate-service-account --key-file=tool-credential.json
|
|
if [ ${_ENV} == production ]; then
|
|
project_id="domain-registry"
|
|
else
|
|
project_id="domain-registry-${_ENV}"
|
|
fi
|
|
gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
|
|
tar -xvf ${_ENV}.tar
|
|
# Note that this currently does not work for google.com projects that
|
|
# we use due to b/137891685. External projects are likely to work.
|
|
for filename in cron dispatch dos index queue; do
|
|
gcloud -q --project $project_id app deploy \
|
|
default/WEB-INF/appengine-generated/$filename.yaml
|
|
done
|
|
# Save the deployed tag for the current environment on GCS, and update the
|
|
# mappings from Nomulus releases to Appengine versions.
|
|
- name: 'gcr.io/$PROJECT_ID/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
set -e
|
|
echo ${TAG_NAME} | \
|
|
gsutil cp - gs://$PROJECT_ID-deployed-tags/nomulus.${_ENV}.tag
|
|
# Update the release to AppEngine version mapping.
|
|
if [ ${_ENV} == production ]; then
|
|
project_id="domain-registry"
|
|
else
|
|
project_id="domain-registry-${_ENV}"
|
|
fi
|
|
local_map="nomulus.${_ENV}.tmp"
|
|
gcloud app versions list \
|
|
--project $project_id --hide-no-traffic \
|
|
--format="csv[no-heading](SERVICE,VERSION.ID)" | \
|
|
grep -e "^backend\|^default\|^pubapi\|^tools" |\
|
|
while read line; do echo "${TAG_NAME},$line"; done | tee "$local_map"
|
|
num_versions=$(cat "$local_map" | wc -l)
|
|
if [ "$num_versions" -ne 4 ]; then
|
|
echo "Expecting exactly four active services. Found $num_versions"
|
|
exit 1
|
|
fi
|
|
gsutil cp "$local_map" gs://$PROJECT_ID-deployed-tags/nomulus.${_ENV}.tmp
|
|
# Atomically append uploaded tmp file to nomulus.${_ENV}.versions
|
|
gsutil compose \
|
|
gs://$PROJECT_ID-deployed-tags/nomulus.${_ENV}.versions \
|
|
gs://$PROJECT_ID-deployed-tags/nomulus.${_ENV}.tmp \
|
|
gs://$PROJECT_ID-deployed-tags/nomulus.${_ENV}.versions
|
|
|
|
timeout: 3600s
|
|
options:
|
|
machineType: 'N1_HIGHCPU_8'
|