diff --git a/release/cloudbuild-delete.yaml b/release/cloudbuild-delete.yaml index 8ae7e9633..76a2ed2ba 100644 --- a/release/cloudbuild-delete.yaml +++ b/release/cloudbuild-delete.yaml @@ -43,7 +43,7 @@ steps: gcloud auth activate-service-account --key-file=tool-credential.json - for service in default pubapi backend bsa tools + for service in default pubapi backend bsa tools console do for version in $(gcloud app versions list \ --filter="SERVICE:$service AND SERVING_STATUS:STOPPED" \ diff --git a/release/cloudbuild-dev-resource.yaml b/release/cloudbuild-dev-resource.yaml index 847cc2769..94fd4e2c5 100644 --- a/release/cloudbuild-dev-resource.yaml +++ b/release/cloudbuild-dev-resource.yaml @@ -10,21 +10,21 @@ # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds steps: # Compile javadoc -- name: 'gcr.io/${PROJECT_ID}/builder:latest' +- name: 'gcr.io/${PROJECT_ID}/builder:live' entrypoint: /bin/bash args: ['./gradlew', ':javadoc'] # Upload the files to GCS # We don't use GCB's built-in artifacts uploader because we want to delete # the existing files in the bucket first, and we want to parallelize the # uploading process. -- name: 'gcr.io/${PROJECT_ID}/builder:latest' +- name: 'gcr.io/${PROJECT_ID}/builder:live' entrypoint: /bin/bash args: ['gsutil', '-m', 'rsync', '-d', '-r', 'build/docs/javadoc', 'gs://${PROJECT_ID}-javadoc'] # Upload the files to GCS # We don't use GCB's built-in artifacts uploader because we want to delete # the existing files in the bucket first, and we want to parallelize the # uploading process. -- name: 'gcr.io/${PROJECT_ID}/builder:latest' +- name: 'gcr.io/${PROJECT_ID}/builder:live' entrypoint: /bin/bash args: ['gsutil', '-m', 'rsync', '-d', '-r', 'db/src/main/resources/sql/er_diagram', 'gs://${PROJECT_ID}-er-diagram'] diff --git a/release/cloudbuild-kythe.yaml b/release/cloudbuild-kythe.yaml index fba1abad3..91ecd2f7d 100644 --- a/release/cloudbuild-kythe.yaml +++ b/release/cloudbuild-kythe.yaml @@ -9,7 +9,7 @@ # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds steps: # Download Kythe -- name: 'gcr.io/${PROJECT_ID}/builder:latest' +- name: 'gcr.io/${PROJECT_ID}/builder:live' entrypoint: /bin/bash args: - -c @@ -20,7 +20,7 @@ steps: rm kythe-${_KYTHE_VERSION}.tar.gz mv kythe-${_KYTHE_VERSION} kythe # Build Nomulus with the Kythe wrapper -- name: 'gcr.io/${PROJECT_ID}/builder:latest' +- name: 'gcr.io/${PROJECT_ID}/builder:live' entrypoint: /bin/bash args: - -c @@ -44,7 +44,7 @@ steps: ./gradlew clean testClasses \ -Dno_werror=true -PenableCrossReferencing=true # Merge kzip files -- name: 'gcr.io/${PROJECT_ID}/builder:latest' +- name: 'gcr.io/${PROJECT_ID}/builder:live' entrypoint: /bin/bash args: - -c @@ -53,13 +53,11 @@ steps: ./kythe/tools/kzip merge \ --output $${KYTHE_OUTPUT_DIRECTORY}/merged/${COMMIT_SHA}.kzip \ $${KYTHE_OUTPUT_DIRECTORY}/*.kzip - artifacts: objects: location: 'gs://${PROJECT_ID}-codesearch' paths: - 'kythe_output/merged/${COMMIT_SHA}.kzip' - timeout: 3600s options: machineType: 'E2_HIGHCPU_32' diff --git a/release/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml index 15adfef7e..dd1aa9721 100644 --- a/release/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -92,11 +92,8 @@ steps: set -e digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/nomulus-tool \ --format="get(digest)" --filter="tags = ${TAG_NAME}") - sed -i s/'$${_IMAGE}'/nomulus-tool/g release/cloudbuild-tag.yaml - sed -i s/':$${TAG_NAME}'/@$digest/g release/cloudbuild-tag.yaml - sed -i s/'nomulus-tool:latest'/nomulus-tool@$digest/g release/cloudbuild-deploy-*.yaml # schema-deploy and schema-verify scripts - sed -i s/'nomulus-tool:latest'/nomulus-tool@$digest/g release/cloudbuild-schema-*.yaml + sed -i s/nomulus-tool:latest/nomulus-tool@$digest/g release/cloudbuild-schema-*.yaml # Build and upload the prober_cert_updater image. This image extends from the `builder` and the # nomulus.jar built earlier. - name: 'gcr.io/${PROJECT_ID}/builder:latest' @@ -114,17 +111,6 @@ steps: docker push gcr.io/${PROJECT_ID}/prober_cert_updater:${TAG_NAME} docker push gcr.io/${PROJECT_ID}/prober_cert_updater:latest dir: 'release/prober-cert-updater/' -# Update the prober_updater image digest in relevant GCB files. -- name: 'gcr.io/${PROJECT_ID}/builder:latest' - entrypoint: /bin/bash - args: - - -c - - | - set -e - digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/prober_cert_updater \ - --format="get(digest)" --filter="tags = ${TAG_NAME}") - sed -i s/prober_cert_updater:latest/prober_cert_updater@$digest/g \ - release/cloudbuild-renew-prober-certs-*.yaml # Build and upload the db_object_updater image. This image extends from the `builder` and the # nomulus.jar built earlier. - name: 'gcr.io/${PROJECT_ID}/builder:latest' @@ -142,17 +128,6 @@ steps: docker push gcr.io/${PROJECT_ID}/db_object_updater:${TAG_NAME} docker push gcr.io/${PROJECT_ID}/db_object_updater:latest dir: 'release/db-object-updater/' -# Update the db_object_updater image digest in relevant GCB files. -- name: 'gcr.io/${PROJECT_ID}/builder:latest' - entrypoint: /bin/bash - args: - - -c - - | - set -e - digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/db_object_updater \ - --format="get(digest)" --filter="tags = ${TAG_NAME}") - sed -i s/db_object_updater:latest/db_object_updater@$digest/g \ - release/cloudbuild-sync-db-objects-*.yaml # Build and stage Dataflow Flex templates. - name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash @@ -215,14 +190,11 @@ artifacts: - 'output/tag_name' - 'output/nomulus.jar' - 'output/schema.jar' - - 'release/cloudbuild-tag.yaml' - - 'release/cloudbuild-sync.yaml' + - 'release/cloudbuild-sync-and-tag.yaml' - 'release/cloudbuild-deploy-*.yaml' - 'release/cloudbuild-delete-*.yaml' - - 'release/cloudbuild-renew-prober-certs-*.yaml' - 'release/cloudbuild-schema-deploy-*.yaml' - 'release/cloudbuild-schema-verify-*.yaml' - - 'release/cloudbuild-sync-db-objects-*.yaml' - 'jetty/kubernetes/*.yaml' - 'jetty/kubernetes/gateway/*.yaml' # The images are already uploaded, but we still need to include them there so that @@ -232,4 +204,4 @@ images: - 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}' timeout: 7200s options: - machineType: 'E2_HIGHCPU_32' + machineType: 'E2_HIGHCPU_32' \ No newline at end of file diff --git a/release/cloudbuild-proxy.yaml b/release/cloudbuild-proxy.yaml index 054d51a11..5a5a3055b 100644 --- a/release/cloudbuild-proxy.yaml +++ b/release/cloudbuild-proxy.yaml @@ -43,8 +43,6 @@ steps: --keyversion-project=${PROJECT_ID} --keyversion-location=global \ --keyversion-keyring=attestor-keys --keyversion-key=signing \ --keyversion=1 - sed -i s/'$${_IMAGE}'/proxy/g release/cloudbuild-tag.yaml - sed -i s/':$${TAG_NAME}'/@$digest/g release/cloudbuild-tag.yaml # Images to upload to GCR. Even though the image has already been uploaded, we still include it # here so that the GCB pubsub message contains it (for Spinnaker to consume). images: ['gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}'] @@ -55,7 +53,6 @@ artifacts: # This cannot be regexs because of how Spinnaker constructs artifact paths. paths: - 'proxy/kubernetes/proxy-*.yaml' - - 'release/cloudbuild-tag.yaml' timeout: 3600s options: machineType: 'E2_HIGHCPU_32' diff --git a/release/cloudbuild-release.yaml b/release/cloudbuild-release.yaml index 1a5f8603f..69e473f18 100644 --- a/release/cloudbuild-release.yaml +++ b/release/cloudbuild-release.yaml @@ -83,13 +83,12 @@ steps: sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-proxy.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-nomulus.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-deploy.yaml - sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-sync.yaml - sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-tag.yaml + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-sync-and-tag.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete.yaml sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml - sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync.yaml + sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync-and-tag.yaml sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-deploy.yaml for environment in alpha crash qa sandbox production; do sed s/'$${_ENV}'/${environment}/g release/cloudbuild-deploy.yaml \ @@ -154,10 +153,6 @@ steps: > release/cloudbuild-schema-deploy-${environment}.yaml sed s/'$${_ENV}'/${environment}/g release/cloudbuild-schema-verify.yaml \ > release/cloudbuild-schema-verify-${environment}.yaml - sed s/'$${_ENV}'/${environment}/g release/cloudbuild-renew-prober-certs.yaml \ - > release/cloudbuild-renew-prober-certs-${environment}.yaml - sed s/'$${_ENV}'/${environment}/g release/cloudbuild-sync-db-objects.yaml \ - > release/cloudbuild-sync-db-objects-${environment}.yaml done # Do text replacement in the k8s manifests. - name: 'gcr.io/cloud-builders/gcloud' @@ -168,6 +163,14 @@ steps: set -e for env in alpha crash qa sandbox production do + # This is the project where Nomulus runs, and we need it to correctly bind the k8s service + # account to the GCP service account. + if [ ${env} == production ] + then + project="domain-registry" + else + project="domain-registry-${env}" + fi base_domain=$(grep baseDomain \ ./core/src/main/java/google/registry/config/files/nomulus-config-${env}.yaml | \ awk '{print $2}') @@ -179,6 +182,11 @@ steps: sed s/PROXY_ENV/${env}/g | \ sed s/EPP/epp/g | \ sed s/WHOIS/whois/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml + if [ ${service} == frontend ] + then + sed -i s/${PROJECT_ID}.iam.gserviceaccount.com/${project}.iam.gserviceaccount.com/g \ + ./jetty/kubernetes/nomulus-${env}-${service}.yaml + fi # canary sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \ sed s/ENVIRONMENT/${env}/g | \ @@ -187,6 +195,11 @@ steps: sed s/WHOIS/whois-canary/g | \ sed s/${service}/${service}-canary/g \ > ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml + if [ ${service} == frontend ] + then + sed -i s/${PROJECT_ID}.iam.gserviceaccount.com/${project}.iam.gserviceaccount.com/g \ + ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml + fi # gateway sed s/BASE_DOMAIN/${base_domain}/g \ ./jetty/kubernetes/gateway/nomulus-route-${service}.yaml \ diff --git a/release/cloudbuild-renew-prober-certs.yaml b/release/cloudbuild-renew-prober-certs.yaml index 68898184a..a032e0c41 100644 --- a/release/cloudbuild-renew-prober-certs.yaml +++ b/release/cloudbuild-renew-prober-certs.yaml @@ -21,7 +21,7 @@ # Please refer to ./cloudbuild-release.yaml for more details. steps: # Generate new SSL certs -- name: 'gcr.io/$PROJECT_ID/builder:latest' +- name: 'gcr.io/$PROJECT_ID/builder:live' entrypoint: /bin/bash args: - -c @@ -46,7 +46,7 @@ steps: -password file:./passphrase.txt # Download the nomulus-tools credential, which has the privilege to invoke tools # commands. Also download the list of probers. -- name: 'gcr.io/$PROJECT_ID/builder:latest' +- name: 'gcr.io/$PROJECT_ID/builder:live' entrypoint: /bin/bash args: - -c @@ -63,7 +63,7 @@ steps: > nomulus_tool_credential.json # Install the new pem cert in the Nomulus server. After this step, both the # current cert and the new cert are accepted for login by the server. -- name: 'gcr.io/$PROJECT_ID/prober_cert_updater:latest' +- name: 'gcr.io/$PROJECT_ID/prober_cert_updater:live' args: - ${_ENV} - ./prober-client-tls.pem @@ -71,7 +71,7 @@ steps: - ./nomulus_tool_credential.json # Add the p12 cert to SecretManager. Prober instances will start using this # cert when they restart. -- name: 'gcr.io/$PROJECT_ID/builder:latest' +- name: 'gcr.io/$PROJECT_ID/builder:live' entrypoint: /bin/bash args: - -c diff --git a/release/cloudbuild-sync-and-tag.yaml b/release/cloudbuild-sync-and-tag.yaml new file mode 100644 index 000000000..8365f51a4 --- /dev/null +++ b/release/cloudbuild-sync-and-tag.yaml @@ -0,0 +1,102 @@ +# This pipeline syncs the folder gs://[PROJECT_ID]-deploy/[TAG] to gs://[PROJECT_ID]-deploy/live. +# It also adds a "live" tag to various images in gcr.io/[PROJECT_ID]/[IMAGE]:[TAG]. +# The PROJECT_ID is the current project name that gcloud uses. +# +# To manually trigger a build on GCB, run: +# gcloud builds submit --config cloudbuild-sync-and-tag.yaml --substitutions TAG_NAME=[TAG] .. +# +# To trigger a build automatically, follow the instructions below and add a trigger: +# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds +# +# Note: to work around issue in Spinnaker's 'Deployment Manifest' stage, +# variable references must avoid the ${var} format. Valid formats include +# $var or ${"${var}"}. This file use the former. Since TAG_NAME is +# expanded in the copies sent to Spinnaker, we preserve the brackets around +# them for safe pattern matching during release. +# See https://github.com/spinnaker/spinnaker/issues/3028 for more information. +steps: +# Rsync the folder where deployment artifacts are uploaded. +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gsutil + - -m + - rsync + - -d + - -r + - gs://$PROJECT_ID-deploy/${TAG_NAME} + - gs://$PROJECT_ID-deploy/live +- # Tag nomulus +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/nomulus:${TAG_NAME} + - gcr.io/$PROJECT_ID/nomulus:live +- # Tag proxy +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/proxy:${TAG_NAME} + - gcr.io/$PROJECT_ID/proxy:live +- # Tag nomulus tool +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/nomulus-tool:${TAG_NAME} + - gcr.io/$PROJECT_ID/nomulus-tool:live +# Tag builder +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/builder:${TAG_NAME} + - gcr.io/$PROJECT_ID/builder:live +# Update db_object_updater +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/db_object_updater:${TAG_NAME} + - gcr.io/$PROJECT_ID/db_object_updater:live +# Update prober cert updater +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/prober_cert_updater:${TAG_NAME} + - gcr.io/$PROJECT_ID/prober_cert_updater:live +# Update schema verifier +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/schema_verifier:${TAG_NAME} + - gcr.io/$PROJECT_ID/schema_verifier:live +# Update schema deployer +- name: 'gcr.io/$PROJECT_ID/builder:latest' + args: + - gcloud + - container + - images + - add-tag + - gcr.io/$PROJECT_ID/schema_deployer:${TAG_NAME} + - gcr.io/$PROJECT_ID/schema_deployer:live +timeout: 3600s +options: + machineType: 'E2_HIGHCPU_32' diff --git a/release/cloudbuild-sync-db-objects.yaml b/release/cloudbuild-sync-db-objects.yaml index 8b573de71..96f652462 100644 --- a/release/cloudbuild-sync-db-objects.yaml +++ b/release/cloudbuild-sync-db-objects.yaml @@ -15,7 +15,7 @@ # steps: # Check out the internal repo. -- name: 'gcr.io/cloud-builders/git' +- name: 'gcr.io/cloud-builders/git:latest' entrypoint: /bin/bash args: - -c @@ -28,7 +28,7 @@ steps: ./gcompute-tools/git-cookie-authdaemon git clone ${_INTERNAL_REPO_URL} nomulus-internal # Download and decrypt the nomulus tool credential -- name: 'gcr.io/$PROJECT_ID/builder:latest' +- name: 'gcr.io/$PROJECT_ID/builder:live' entrypoint: /bin/bash args: - -c @@ -38,21 +38,21 @@ steps: --secret nomulus-tool-cloudbuild-credential \ > nomulus_tool_credential.json # Configure the TLDs using the stored configuration files in the internal repo -- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest' +- name: 'gcr.io/$PROJECT_ID/db_object_updater:live' args: - ${_ENV} - ./nomulus_tool_credential.json - configure_tld - nomulus-internal/core/src/main/java/google/registry/config/files/tld/ # Configure the premium lists using the stored configuration files in the internal repo -- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest' +- name: 'gcr.io/$PROJECT_ID/db_object_updater:live' args: - ${_ENV} - ./nomulus_tool_credential.json - update_premium_list - nomulus-internal/core/src/main/java/google/registry/config/files/premium/ # Configure the reserved lists using the stored configuration files in the internal repo -- name: 'gcr.io/$PROJECT_ID/db_object_updater:latest' +- name: 'gcr.io/$PROJECT_ID/db_object_updater:live' args: - ${_ENV} - ./nomulus_tool_credential.json diff --git a/release/cloudbuild-sync.yaml b/release/cloudbuild-sync.yaml deleted file mode 100644 index 0f7eadb39..000000000 --- a/release/cloudbuild-sync.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# This will sync the folder gs://[PROJECT_ID]-deploy/[TAG] to gs://[PROJECT_ID]-deploy/live. -# The PROJECT_ID is the current project name that gcloud uses. -# -# To manually trigger a build on GCB, run: -# gcloud builds submit --config cloudbuild-sync.yaml --substitutions TAG_NAME=[TAG] .. -# -# To trigger a build automatically, follow the instructions below and add a trigger: -# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds -# -# Note: to work around issue in Spinnaker's 'Deployment Manifest' stage, -# variable references must avoid the ${var} format. Valid formats include -# $var or ${"${var}"}. This file use the former. Since TAG_NAME is -# expanded in the copies sent to Spinnaker, we preserve the brackets around -# them for safe pattern matching during release. -# See https://github.com/spinnaker/spinnaker/issues/3028 for more information. -steps: -# Rsync the folder. -- name: 'gcr.io/$PROJECT_ID/builder:latest' - args: - - gsutil - - -m - - rsync - - -d - - -r - - gs://$PROJECT_ID-deploy/${TAG_NAME} - - gs://$PROJECT_ID-deploy/live -timeout: 3600s -options: - machineType: 'E2_HIGHCPU_32' diff --git a/release/cloudbuild-tag.yaml b/release/cloudbuild-tag.yaml deleted file mode 100644 index 7e0720420..000000000 --- a/release/cloudbuild-tag.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# This will add a "live" tag to the image in gcr.io/[PROJECT_ID]/[IMAGE]:[TAG]. -# The PROJECT_ID is the current project name that gcloud uses. -# -# To manually trigger a build on GCB, run: -# gcloud builds submit --config cloudbuild-tag.yaml --substitutions \ -# TAG_NAME=[TAG],_IMAGE=[IMAGE] .. -# -# To trigger a build automatically, follow the instructions below and add a trigger: -# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds -# -# Note: to work around issue in Spinnaker's 'Deployment Manifest' stage, -# variable references must avoid the ${var} format. Valid formats include -# $var or ${"${var}"}. This file use the former. Since TAG_NAME is -# expanded in the copies sent to Spinnaker, we preserve the brackets around -# them for safe pattern matching during release. -# See https://github.com/spinnaker/spinnaker/issues/3028 for more information. -steps: -- name: 'gcr.io/$PROJECT_ID/builder:latest' - args: - - gcloud - - container - - images - - add-tag - - gcr.io/$PROJECT_ID/${_IMAGE}:${TAG_NAME} - - gcr.io/$PROJECT_ID/${_IMAGE}:live -timeout: 3600s -options: - machineType: 'E2_HIGHCPU_32'