diff --git a/release/cloudbuild-dev-resource.yaml b/release/cloudbuild-dev-resource.yaml index 5ac4ae2f2..847cc2769 100644 --- a/release/cloudbuild-dev-resource.yaml +++ b/release/cloudbuild-dev-resource.yaml @@ -10,21 +10,21 @@ # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds steps: # Compile javadoc -- name: 'gcr.io/${PROJECT_ID}/builder' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash args: ['./gradlew', ':javadoc'] # Upload the files to GCS # We don't use GCB's built-in artifacts uploader because we want to delete # the existing files in the bucket first, and we want to parallelize the # uploading process. -- name: 'gcr.io/${PROJECT_ID}/builder' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash args: ['gsutil', '-m', 'rsync', '-d', '-r', 'build/docs/javadoc', 'gs://${PROJECT_ID}-javadoc'] # Upload the files to GCS # We don't use GCB's built-in artifacts uploader because we want to delete # the existing files in the bucket first, and we want to parallelize the # uploading process. -- name: 'gcr.io/${PROJECT_ID}/builder' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash args: ['gsutil', '-m', 'rsync', '-d', '-r', 'db/src/main/resources/sql/er_diagram', 'gs://${PROJECT_ID}-er-diagram'] diff --git a/release/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml index 399450eb0..3598ff53a 100644 --- a/release/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -36,18 +36,53 @@ steps: # informational. It makes it easier to tell the tag of the current 'live' release. - name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash - args: [ '-c', 'echo ${TAG_NAME} > output/tag_name'] -# Tag the tool image and push it to the registry. + args: [ '-c', 'echo ${TAG_NAME} > output/tag_name' ] +# Build Nomulus, tool and proxy image, them upload them to GCR. +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + # Set home for Gradle caches. Must be consistent with last step below + # and ./build_nomulus_for_env.sh + env: [ 'GRADLE_USER_HOME=/workspace/cloudbuild-caches' ] + entrypoint: /bin/bash + args: + - -c + - | + ./gradlew :jetty:buildNomulusImage :proxy:buildProxyImage :core:buildToolImage\ + -PmavenUrl=gcs://domain-registry-maven-repository/maven \ + -PpluginsUrl=gcs://domain-registry-maven-repository/plugins + docker tag nomulus gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME} + docker tag nomulus gcr.io/${PROJECT_ID}/nomulus:latest + docker push gcr.io/${PROJECT_ID}/nomulus:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/nomulus:latest + docker tag proxy gcr.io/${PROJECT_ID}/proxy:${TAG_NAME} + docker tag proxy gcr.io/${PROJECT_ID}/proxy:latest + docker push gcr.io/${PROJECT_ID}/proxy:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/proxy:latest + docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME} + docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:latest + docker push gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/nomulus-tool:latest +# Sign nomulus and proxy images. - name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash args: - -c - | - set -e - docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME} - docker tag nomulus-tool gcr.io/${PROJECT_ID}/nomulus-tool:latest - docker push gcr.io/${PROJECT_ID}/nomulus-tool:${TAG_NAME} - docker push gcr.io/${PROJECT_ID}/nomulus-tool:latest + nomulus_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/nomulus \ + --format="get(digest)" --filter="tags = ${TAG_NAME}") + proxy_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \ + --format="get(digest)" --filter="tags = ${TAG_NAME}") + gcloud --project=${PROJECT_ID} beta container binauthz attestations \ + sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/nomulus@$nomulus_digest \ + --attestor=build-attestor --attestor-project=${PROJECT_ID} \ + --keyversion-project=${PROJECT_ID} --keyversion-location=global \ + --keyversion-keyring=attestor-keys --keyversion-key=signing \ + --keyversion=1 + gcloud --project=${PROJECT_ID} beta container binauthz attestations \ + sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$proxy_digest \ + --attestor=build-attestor --attestor-project=${PROJECT_ID} \ + --keyversion-project=${PROJECT_ID} --keyversion-location=global \ + --keyversion-keyring=attestor-keys --keyversion-key=signing \ + --keyversion=1 # Get the tool image digest and substitute in the digest in other GCB files. - name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash @@ -62,9 +97,9 @@ steps: sed -i s/'nomulus-tool:latest'/nomulus-tool@$digest/g release/cloudbuild-deploy-*.yaml # schema-deploy and schema-verify scripts sed -i s/'nomulus-tool:latest'/nomulus-tool@$digest/g release/cloudbuild-schema-*.yaml -# Build the prober_cert_updater image and upload it to GCR. This image extends -# from the `builder` and the nomulus.jar built earlier. -- name: 'gcr.io/cloud-builders/docker' +# Build and upload the prober_cert_updater image. This image extends from the `builder` and the +# nomulus.jar built earlier. +- name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash args: - -c @@ -76,8 +111,8 @@ steps: --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} . docker tag gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME} \ gcr.io/${PROJECT_ID}/prober_cert_updater:latest - docker push gcr.io/${PROJECT_ID}/prober_cert_updater:latest docker push gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/prober_cert_updater:latest dir: 'release/prober-cert-updater/' # Update the prober_updater image digest in relevant GCB files. - name: 'gcr.io/${PROJECT_ID}/builder:latest' @@ -86,14 +121,13 @@ steps: - -c - | set -e - digest=$(gcloud container images list-tags \ - gcr.io/${PROJECT_ID}/prober_cert_updater \ - --format="get(digest)" --filter="tags = ${TAG_NAME}") - sed -i s/'prober_cert_updater:latest'/prober_cert_updater@$digest/g \ + digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/prober_cert_updater \ + --format="get(digest)" --filter="tags = ${TAG_NAME}") + sed -i s/prober_cert_updater:latest/prober_cert_updater@$digest/g \ release/cloudbuild-renew-prober-certs-*.yaml -# Build the db_object_updater image and upload it to GCR. This image extends -# from the `builder` and the nomulus.jar built earlier. -- name: 'gcr.io/cloud-builders/docker' +# Build and upload the db_object_updater image. This image extends from the `builder` and the +# nomulus.jar built earlier. +- name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash args: - -c @@ -105,8 +139,8 @@ steps: --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} . docker tag gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} \ gcr.io/${PROJECT_ID}/db_object_updater:latest - docker push gcr.io/${PROJECT_ID}/db_object_updater:latest docker push gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/db_object_updater:latest dir: 'release/db-object-updater/' # Update the db_object_updater image digest in relevant GCB files. - name: 'gcr.io/${PROJECT_ID}/builder:latest' @@ -115,15 +149,14 @@ steps: - -c - | set -e - digest=$(gcloud container images list-tags \ - gcr.io/${PROJECT_ID}/db_object_updater \ + digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/db_object_updater \ --format="get(digest)" --filter="tags = ${TAG_NAME}") - sed -i s/'db_object_updater:latest'/db_object_updater@$digest/g \ + sed -i s/db_object_updater:latest/db_object_updater@$digest/g \ release/cloudbuild-sync-db-objects-*.yaml # Build and stage Dataflow Flex templates. - name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash - # Set home for Gradle caches. Must be consistent with previous steps above + # Set home for Gradle caches. Must be consistent with the previous steps above # and ./build_nomulus_for_env.sh env: [ 'GRADLE_USER_HOME=/workspace/cloudbuild-caches' ] args: @@ -146,9 +179,8 @@ steps: google/registry/beam/resave_all_epp_resources_pipeline_metadata.json \ google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline \ google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json -# Tentatively build and publish Cloud SQL schema jar here, before schema release -# process is finalized. Also publish nomulus:core jars that are needed for -# server/schema compatibility tests. +# Tentatively build and publish Cloud SQL schema jar here, before the schema release process is +# finalized. Also publish nomulus:core jars that are needed for server/schema compatibility tests. - name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash # Set home for Gradle caches. Must be consistent with previous steps above @@ -191,7 +223,6 @@ artifacts: - 'release/cloudbuild-schema-deploy-*.yaml' - 'release/cloudbuild-schema-verify-*.yaml' - 'release/cloudbuild-sync-db-objects-*.yaml' - timeout: 7200s options: machineType: 'E2_HIGHCPU_32' diff --git a/release/cloudbuild-release.yaml b/release/cloudbuild-release.yaml index e2c2f7b7c..f6004cd1a 100644 --- a/release/cloudbuild-release.yaml +++ b/release/cloudbuild-release.yaml @@ -7,7 +7,7 @@ # # This pipeline prepares a release. The pipeline should be run against the Nomulus public repo on # GitHub. It builds the builder and base images, and hard codes the sha256 hashes of the resulting -# images in the merged code base (internal + public) , which is tagged and pushed into the release +# images in the merged code base (internal + public), which is tagged and pushed into the release # repo. Actual release artifacts are built from the release repo, ensuring reproducibility. steps: # Check the out internal repo. @@ -42,7 +42,7 @@ steps: rm -rf .git && rm -rf nomulus-internal/.git cp -rf nomulus-internal/* . rm -rf nomulus-internal -# Build the builder image and tag the proxy base image, then upload them to GCR. +# Build the builder image and pull the base images, them upload them to GCR. - name: 'gcr.io/cloud-builders/docker' entrypoint: /bin/bash args: @@ -51,18 +51,18 @@ steps: set -e docker build -t gcr.io/${PROJECT_ID}/builder:${TAG_NAME} . docker tag gcr.io/${PROJECT_ID}/builder:${TAG_NAME} gcr.io/${PROJECT_ID}/builder:latest - docker pull gcr.io/distroless/java - docker tag gcr.io/distroless/java gcr.io/${PROJECT_ID}/base:${TAG_NAME} - docker tag gcr.io/distroless/java gcr.io/${PROJECT_ID}/base:latest - docker pull gcr.io/distroless/java:debug - docker tag gcr.io/distroless/java:debug gcr.io/${PROJECT_ID}/base-debug:${TAG_NAME} - docker tag gcr.io/distroless/java:debug gcr.io/${PROJECT_ID}/base-debug:latest - docker push gcr.io/${PROJECT_ID}/builder:latest docker push gcr.io/${PROJECT_ID}/builder:${TAG_NAME} - docker push gcr.io/${PROJECT_ID}/base:latest - docker push gcr.io/${PROJECT_ID}/base:${TAG_NAME} - docker push gcr.io/${PROJECT_ID}/base-debug:latest - docker push gcr.io/${PROJECT_ID}/base-debug:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/builder:latest + docker pull jetty:12-jdk21 + docker tag jetty:12-jdk21 gcr.io/${PROJECT_ID}/jetty:${TAG_NAME} + docker tag jetty:12-jdk21 gcr.io/${PROJECT_ID}/jetty:latest + docker push gcr.io/${PROJECT_ID}/jetty:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/jetty:latest + docker pull eclipse-temurin:21 + docker tag eclipse-temurin:21 gcr.io/${PROJECT_ID}/temurin:${TAG_NAME} + docker tag eclipse-temurin:21 gcr.io/${PROJECT_ID}/temurin:latest + docker push gcr.io/${PROJECT_ID}/temurin:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/temurin:latest dir: 'release/builder/' # Do text replacement in the merged repo, hardcoding image digests. - name: 'gcr.io/cloud-builders/gcloud' @@ -73,27 +73,31 @@ steps: set -e builder_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/builder \ --format='get(digest)' --filter='tags = ${TAG_NAME}') - base_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/base \ + jetty_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/jetty \ --format='get(digest)' --filter='tags = ${TAG_NAME}') - debug_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/base-debug \ + temurin_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/temurin \ --format='get(digest)' --filter='tags = ${TAG_NAME}') - sed -i s%distroless/java%${PROJECT_ID}/base@$base_digest% proxy/Dockerfile - sed -i s%distroless/java:debug%${PROJECT_ID}/base-debug@$debug_digest% core/Dockerfile + sed -i s%eclipse-temurin:21%gcr.io/${PROJECT_ID}/temurin@$temurin_digest%g proxy/Dockerfile + sed -i s%eclipse-temurin:21%gcr.io/${PROJECT_ID}/temurin@$temurin_digest%g core/Dockerfile + sed -i s%jetty:12-jdk21%gcr.io/${PROJECT_ID}/jetty@$jetty_digest%g jetty/Dockerfile sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-proxy.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-nomulus.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-deploy.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-sync.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-tag.yaml + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete.yaml sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync.yaml sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-deploy.yaml - for environment in alpha crash sandbox production; do + for environment in alpha crash qa sandbox production; do sed s/'$${_ENV}'/${environment}/g release/cloudbuild-deploy.yaml \ > release/cloudbuild-deploy-${environment}.yaml sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete.yaml \ > release/cloudbuild-delete-${environment}.yaml done -# Build the schema_deployer image and upload it to GCR. +# Build and upload the schema_deployer image. - name: 'gcr.io/cloud-builders/docker' entrypoint: /bin/bash args: @@ -103,10 +107,10 @@ steps: docker build -t gcr.io/${PROJECT_ID}/schema_deployer:${TAG_NAME} --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} . docker tag gcr.io/${PROJECT_ID}/schema_deployer:${TAG_NAME} \ gcr.io/${PROJECT_ID}/schema_deployer:latest - docker push gcr.io/${PROJECT_ID}/schema_deployer:latest docker push gcr.io/${PROJECT_ID}/schema_deployer:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/schema_deployer:latest dir: 'release/schema-deployer/' -# Build the schema_verifier image and upload it to GCR. +# Build and upload the schema_verifier image. - name: 'gcr.io/cloud-builders/docker' entrypoint: /bin/bash args: @@ -116,33 +120,21 @@ steps: docker build -t gcr.io/${PROJECT_ID}/schema_verifier:${TAG_NAME} --build-arg TAG_NAME=${TAG_NAME} --build-arg PROJECT_ID=${PROJECT_ID} . docker tag gcr.io/${PROJECT_ID}/schema_verifier:${TAG_NAME} \ gcr.io/${PROJECT_ID}/schema_verifier:latest - docker push gcr.io/${PROJECT_ID}/schema_verifier:latest docker push gcr.io/${PROJECT_ID}/schema_verifier:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/schema_verifier:latest dir: 'release/schema-verifier/' -# Do text replacement in the schema-deploy, schema-verify and -# prober_cert_updater configs. +# Do text replacement in the cloud build YAML files. - name: 'gcr.io/cloud-builders/gcloud' entrypoint: /bin/bash args: - -c - | set -e - builder_digest=$( \ - gcloud container images list-tags gcr.io/${PROJECT_ID}/builder \ + builder_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/builder \ --format='get(digest)' --filter='tags = ${TAG_NAME}') - schema_deployer_digest=$( \ - gcloud container images list-tags gcr.io/${PROJECT_ID}/schema_deployer \ + schema_deployer_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/schema_deployer \ --format='get(digest)' --filter='tags = ${TAG_NAME}') - schema_verifier_digest=$( \ - gcloud container images list-tags gcr.io/${PROJECT_ID}/schema_verifier \ - --format='get(digest)' --filter='tags = ${TAG_NAME}') - prober_cert_updater_digest=$( \ - gcloud container images list-tags \ - gcr.io/${PROJECT_ID}/prober_cert_updater \ - --format='get(digest)' --filter='tags = ${TAG_NAME}') - db_object_updater_digest=$( \ - gcloud container images list-tags \ - gcr.io/${PROJECT_ID}/db_object_updater \ + schema_verifier_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/schema_verifier \ --format='get(digest)' --filter='tags = ${TAG_NAME}') sed -i s/builder:latest/builder@$builder_digest/g \ release/cloudbuild-schema-deploy.yaml @@ -157,7 +149,7 @@ steps: sed -i s/schema_verifier:latest/schema_verifier@$schema_verifier_digest/g \ release/cloudbuild-schema-verify.yaml sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-schema-deploy.yaml - for environment in alpha crash sandbox production; do + for environment in alpha crash qa sandbox production; do sed s/'$${_ENV}'/${environment}/g release/cloudbuild-schema-deploy.yaml \ > release/cloudbuild-schema-deploy-${environment}.yaml sed s/'$${_ENV}'/${environment}/g release/cloudbuild-schema-verify.yaml \ @@ -167,7 +159,46 @@ steps: sed s/'$${_ENV}'/${environment}/g release/cloudbuild-sync-db-objects.yaml \ > release/cloudbuild-sync-db-objects-${environment}.yaml done -# Upload the gradle binary to GCS if it does not exist and point URL in gradle wrapper to it. +# Do text replacement in the k8s manifests. +- name: 'gcr.io/cloud-builders/gcloud' + entrypoint: /bin/bash + args: + - -c + - | + set -e + for env in alpha crash qa sandbox production + do + based_domain=$(grep baseDomain \ + ./core/src/main/java/google/registry/config/files/nomulus-config-${env}.yaml | \ + awk '{print $2}') + for service in frontend backend pubapi console + do + # non-canary + sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \ + sed s/ENVIRONMENT/${env}/g | \ + sed s/PROXY_ENV/${env}/g | \ + sed s/EPP/epp/g | \ + sed s/WHOIS/whois/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml + # canary + sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \ + sed s/ENVIRONMENT/${env}/g | \ + sed s/PROXY_ENV/${env}_canary/g | \ + sed s/EPP/epp-canary/g | \ + sed s/WHOIS/whois-canary/g | \ + sed s/${service}/${service}-canary/g \ + > ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml + # gateway + sed s/BASE_DOMAIN/${base_domain}/g \ + ./jetty/kubernetes/gateway/nomulus-route-${service}.yaml \ + > ./jetty/kubernetes/gateway/nomulus-route-${env}-${service}.yaml + # IAP + sed s/SERVICE/${service}/g ./jetty/kubernetes/gateway/nomulus-iap-${env}.yaml \ + > ./jetty/kubernetes/gateway/nomulus-iap-${env}-${service}.yaml + sed s/SERVICE/${service}-canary/g ./jetty/kubernetes/gateway/nomulus-iap-${env}.yaml \ + > ./jetty/kubernetes/gateway/nomulus-iap-${env}-${service}-canary.yaml + done + done +# Upload the Gradle binary to GCS if it does not exist and point URL in Gradle wrapper to it. - name: 'gcr.io/cloud-builders/gsutil' entrypoint: /bin/bash args: