mirror of
https://github.com/google/nomulus
synced 2026-01-05 04:56:03 +00:00
d4a70c29a8
We need to stop using maven repo on GCS to store artifacts for the schema compatibility tests. After public access is removed from GCS buckets, Kokoro won't be able to access it: normal access will be denied, and the repo is too large to map (copy) to Kokoro VM as a resource. This PR uploads the relevant jars to each release's folder. See go/dr-gcs-public-access-prevention for details.
215 lines
9.6 KiB
YAML
215 lines
9.6 KiB
YAML
# To manually trigger a build on GCB, run:
|
|
# gcloud builds submit --config cloudbuild-nomulus.yaml --substitutions TAG_NAME=[TAG] ..
|
|
#
|
|
# To trigger a build automatically, follow the instructions below and add a trigger:
|
|
# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
|
|
steps:
|
|
# Create a directory to store the artifacts
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
args: ['mkdir', 'nomulus']
|
|
# Run tests
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
# Set home for Gradle caches. Must be consistent with last step below
|
|
# and ./build_nomulus_for_env.sh
|
|
env: [ 'GRADLE_USER_HOME=/workspace/cloudbuild-caches' ]
|
|
args: ['./gradlew',
|
|
'test',
|
|
'-PskipDockerIncompatibleTests=true',
|
|
'-PmavenUrl=gcs://domain-registry-maven-repository/maven',
|
|
'-PpluginsUrl=gcs://domain-registry-maven-repository/plugins'
|
|
]
|
|
# Build and package the deployment files for each environment, and the tool
|
|
# binary and image.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
# Set home for Gradle caches. Must be consistent with last step below
|
|
# and ./build_nomulus_for_env.sh
|
|
env: [ 'GRADLE_USER_HOME=/workspace/cloudbuild-caches' ]
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
for _env in tool alpha crash sandbox production
|
|
do
|
|
release/build_nomulus_for_env.sh $${_env} output
|
|
done
|
|
# Save TAG_NAME in ./output/tag_name, to be uploaded later. This file is purely
|
|
# informational. It makes it easier to tell the tag of the current 'live' release.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args: [ '-c', 'echo ${TAG_NAME} > output/tag_name' ]
|
|
# Build Nomulus, tool and proxy image, them upload them to GCR.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
# Set home for Gradle caches. Must be consistent with last step below
|
|
# and ./build_nomulus_for_env.sh
|
|
env: [ 'GRADLE_USER_HOME=/workspace/cloudbuild-caches' ]
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
./gradlew :jetty:buildNomulusImage :proxy:buildProxyImage :core:buildToolImage\
|
|
-PmavenUrl=gcs://domain-registry-maven-repository/maven \
|
|
-PpluginsUrl=gcs://domain-registry-maven-repository/plugins
|
|
docker tag nomulus gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME}
|
|
docker tag nomulus gcr.io/${PROJECT_ID}/nomulus:latest
|
|
docker push gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME}
|
|
docker push gcr.io/${PROJECT_ID}/nomulus:latest
|
|
docker tag proxy gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}
|
|
docker tag proxy gcr.io/${PROJECT_ID}/proxy:latest
|
|
docker push gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}
|
|
docker push gcr.io/${PROJECT_ID}/proxy:latest
|
|
docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME}
|
|
docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:latest
|
|
docker push gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME}
|
|
docker push gcr.io/${PROJECT_ID}/nomulus-tool:latest
|
|
# Sign nomulus and proxy images.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
nomulus_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/nomulus \
|
|
--format="get(digest)" --filter="tags = ${TAG_NAME}")
|
|
proxy_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \
|
|
--format="get(digest)" --filter="tags = ${TAG_NAME}")
|
|
gcloud --project=${PROJECT_ID} beta container binauthz attestations \
|
|
sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/nomulus@$nomulus_digest \
|
|
--attestor=build-attestor --attestor-project=${PROJECT_ID} \
|
|
--keyversion-project=${PROJECT_ID} --keyversion-location=global \
|
|
--keyversion-keyring=attestor-keys --keyversion-key=signing \
|
|
--keyversion=1
|
|
gcloud --project=${PROJECT_ID} beta container binauthz attestations \
|
|
sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$proxy_digest \
|
|
--attestor=build-attestor --attestor-project=${PROJECT_ID} \
|
|
--keyversion-project=${PROJECT_ID} --keyversion-location=global \
|
|
--keyversion-keyring=attestor-keys --keyversion-key=signing \
|
|
--keyversion=1
|
|
# Get the tool image digest and substitute in the digest in other GCB files.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
set -e
|
|
digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/nomulus-tool \
|
|
--format="get(digest)" --filter="tags = ${TAG_NAME}")
|
|
# schema-deploy and schema-verify scripts
|
|
sed -i s/nomulus-tool:latest/nomulus-tool@$digest/g release/cloudbuild-schema-*.yaml
|
|
# Build and upload the prober_cert_updater image. This image extends from the `builder` and the
|
|
# nomulus.jar built earlier.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
set -e
|
|
# The nomulus jar is not under the working dir. Must be copied over.
|
|
cp ../../output/nomulus.jar .
|
|
docker build -t gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME} \
|
|
--build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} .
|
|
docker tag gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME} \
|
|
gcr.io/${PROJECT_ID}/prober_cert_updater:latest
|
|
docker push gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME}
|
|
docker push gcr.io/${PROJECT_ID}/prober_cert_updater:latest
|
|
dir: 'release/prober-cert-updater/'
|
|
# Build and upload the db_object_updater image. This image extends from the `builder` and the
|
|
# nomulus.jar built earlier.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
entrypoint: /bin/bash
|
|
args:
|
|
- -c
|
|
- |
|
|
set -e
|
|
# The nomulus jar is not under the working dir. Must be copied over.
|
|
cp ../../output/nomulus.jar .
|
|
docker build -t gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} \
|
|
--build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} .
|
|
docker tag gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} \
|
|
gcr.io/${PROJECT_ID}/db_object_updater:latest
|
|
docker push gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME}
|
|
docker push gcr.io/${PROJECT_ID}/db_object_updater:latest
|
|
dir: 'release/db-object-updater/'
|
|
# Build and stage Dataflow Flex templates.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
entrypoint: /bin/bash
|
|
# Set home for Gradle caches. Must be consistent with the previous steps above
|
|
# and ./build_nomulus_for_env.sh
|
|
env: [ 'GRADLE_USER_HOME=/workspace/cloudbuild-caches' ]
|
|
args:
|
|
- -c
|
|
- |
|
|
./release/stage_beam_pipeline.sh \
|
|
beamPipelineCommon \
|
|
beam_pipeline_common \
|
|
${TAG_NAME} \
|
|
${PROJECT_ID} \
|
|
google.registry.beam.spec11.Spec11Pipeline \
|
|
google/registry/beam/spec11_pipeline_metadata.json \
|
|
google.registry.beam.billing.InvoicingPipeline \
|
|
google/registry/beam/invoicing_pipeline_metadata.json \
|
|
google.registry.beam.billing.ExpandBillingRecurrencesPipeline \
|
|
google/registry/beam/expand_billing_recurrences_pipeline_metadata.json \
|
|
google.registry.beam.rde.RdePipeline \
|
|
google/registry/beam/rde_pipeline_metadata.json \
|
|
google.registry.beam.resave.ResaveAllEppResourcesPipeline \
|
|
google/registry/beam/resave_all_epp_resources_pipeline_metadata.json \
|
|
google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline \
|
|
google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json
|
|
# Tentatively build and publish Cloud SQL schema jar here, before the schema release process is
|
|
# finalized. Also publish nomulus:core jars that are needed for server/schema compatibility tests.
|
|
- name: 'gcr.io/${PROJECT_ID}/builder:latest'
|
|
entrypoint: /bin/bash
|
|
# Set home for Gradle caches. Must be consistent with previous steps above
|
|
# and ./build_nomulus_for_env.sh
|
|
env: [ 'GRADLE_USER_HOME=/workspace/cloudbuild-caches' ]
|
|
args:
|
|
- -c
|
|
- |
|
|
set -e
|
|
./gradlew \
|
|
:db:publish \
|
|
-PmavenUrl=gcs://domain-registry-maven-repository/maven \
|
|
-PpluginsUrl=gcs://domain-registry-maven-repository/plugins \
|
|
-Ppublish_repo=gcs://${PROJECT_ID}-deployed-tags/maven \
|
|
-Pschema_version=${TAG_NAME}
|
|
./gradlew \
|
|
:core:publish \
|
|
-PmavenUrl=gcs://domain-registry-maven-repository/maven \
|
|
-PpluginsUrl=gcs://domain-registry-maven-repository/plugins \
|
|
-Ppublish_repo=gcs://${PROJECT_ID}-deployed-tags/maven \
|
|
-Pnomulus_version=${TAG_NAME}
|
|
# Upload jars for use by schema deployment and schema compatibility test.
|
|
# The jars published to GCS will be unaccessible to Kokoro once public access is removed.
|
|
cp db/build/libs/schema.jar output/
|
|
cp core/build/libs/nomulus-public.jar output/
|
|
cp core/build/libs/nomulus-tests-alldeps.jar output/
|
|
# The tarballs and jars to upload to GCS.
|
|
artifacts:
|
|
objects:
|
|
location: 'gs://${PROJECT_ID}-deploy/${TAG_NAME}'
|
|
paths:
|
|
- 'output/*.tar'
|
|
- 'output/tag_name'
|
|
- 'output/nomulus.jar'
|
|
- 'output/nomulus-public.jar'
|
|
- 'output/nomulus-tests-alldeps.jar'
|
|
- 'output/schema.jar'
|
|
- 'core/src/main/java/google/registry/config/files/nomulus-config-*.yaml'
|
|
- 'core/src/main/java/google/registry/config/files/cloud-tasks-queue.xml'
|
|
- 'core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-*.xml'
|
|
- 'release/cloudbuild-sync-and-tag.yaml'
|
|
- 'release/cloudbuild-deploy-*.yaml'
|
|
- 'release/cloudbuild-delete-*.yaml'
|
|
- 'release/cloudbuild-renew-prober-certs-*.yaml'
|
|
- 'release/cloudbuild-schema-deploy-*.yaml'
|
|
- 'release/cloudbuild-schema-verify-*.yaml'
|
|
- 'release/cloudbuild-restart-proxies-*.yaml'
|
|
- 'jetty/kubernetes/*.yaml'
|
|
- 'jetty/kubernetes/gateway/*.yaml'
|
|
# The images are already uploaded, but we still need to include them there so that
|
|
# the GCB pubsub message contains them (for Spinnaker to consume).
|
|
images:
|
|
- 'gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME}'
|
|
- 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}'
|
|
timeout: 7200s
|
|
options:
|
|
machineType: 'E2_HIGHCPU_32' |