Update Cloud Scheduler and Cloud Tasks deployment process (#2666)

build.gradle

@@ -561,7 +561,7 @@ task deployCloudSchedulerAndQueue {
         commandLine 'go', 'run',
                 "./deployCloudSchedulerAndQueue.go",
                 "${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${env}.yaml",
-                "${rootDir}/core/src/main/java/google/registry/env/${env}/default/WEB-INF/cloud-scheduler-tasks.xml",
+                "${rootDir}/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-${env}.xml",
                 "domain-registry-${env}"
       }
       exec {
@@ -569,7 +569,7 @@ task deployCloudSchedulerAndQueue {
         commandLine 'go', 'run',
                 "./deployCloudSchedulerAndQueue.go",
                 "${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${env}.yaml",
-                "${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
+                "${rootDir}/core/src/main/java/google/registry/config/files/cloud-tasks-queue.xml",
                 "domain-registry-${env}"
       }
     }

console-webapp/staged/app.yaml

@@ -1,4 +1,6 @@
 runtime: nodejs20
 service: console
+basic_scaling:
+  max_instances: 10
 build_env_variables:
   GOOGLE_NODE_RUN_SCRIPTS: ''

jetty/build.gradle

@@ -60,6 +60,7 @@ tasks.register('tagNomulusImage', Exec) {
 }
 
 tasks.register('pushNomulusImage', Exec) {
+    configure verifyDeploymentConfig
     commandLine 'docker', 'push', "gcr.io/${rootProject.gcpProject}/nomulus"
     dependsOn(tasks.named('tagNomulusImage'))
 }
@@ -89,6 +90,7 @@ tasks.register('buildDeployer', Exec) {
 
 // Once GKE is the only option, we can use the same task in the root project instead.
 tasks.register('deployCloudSchedulerAndQueue') {
+    configure verifyDeploymentConfig
     dependsOn(tasks.named('deployCloudScheduler'), tasks.named('deployQueue'))
 }
 
@@ -97,7 +99,7 @@ tasks.register('deployCloudScheduler', Exec) {
     workingDir("$buildDir")
     commandLine './deployer',
         "${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${rootProject.environment}.yaml",
-        "${rootDir}/core/src/main/java/google/registry/env/${rootProject.environment}/default/WEB-INF/cloud-scheduler-tasks.xml",
+        "${rootDir}/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-${rootProject.environment}.xml",
         rootProject.gcpProject, '--gke'
     // Only deploy the tasks after Nomulus itself is deployed.
     mustRunAfter(tasks.named('deployToGke'))
@@ -108,7 +110,7 @@ tasks.register('deployQueue', Exec) {
     workingDir("$buildDir")
     commandLine './deployer',
         "${rootDir}/core/src/main/java/google/registry/config/files/nomulus-config-${rootProject.environment}.yaml",
-        "${rootDir}/core/src/main/java/google/registry/env/common/default/WEB-INF/cloud-tasks-queue.xml",
+        "${rootDir}/core/src/main/java/google/registry/config/files/cloud-tasks-queue.xml",
         rootProject.gcpProject, '--gke'
     // Only deploy the queues after Nomulus itself is deployed.
     mustRunAfter(tasks.named('deployToGke'))
@@ -116,7 +118,6 @@ tasks.register('deployQueue', Exec) {
 
 tasks.register('deployToGke', Exec) {
     dependsOn('pushNomulusImage', ':proxy:pushProxyImage')
-    configure verifyDeploymentConfig
     commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}", "${rootProject.baseDomain}"
 }
 
@@ -125,7 +126,13 @@ tasks.register('deployNomulus') {
 }
 
 tasks.register('getEndpoints', Exec) {
-    configure verifyDeploymentConfig
+    doFirst {
+        if (rootProject.ext.gcpProject == null) {
+            def error = 'You must specify -Penvironment={alpha,crash,qa}'
+            System.err.println("\033[33;1m${error}\033[0m")
+            throw GradleException("Aborting:  ${error}")
+        }
+    }
     commandLine './get-endpoints.py', "${rootProject.gcpProject}"
 }
 

jetty/get-endpoints.py

@@ -45,7 +45,8 @@ class UseCluster(PreserveContext):
 
     def __enter__(self):
         super().__enter__()
-        cmd = f'gcloud container clusters get-credentials {self._cluster} --location {self._region} --project {self._project}'
+        cmd = (f'gcloud container fleet memberships get-credentials'
+                   f' {self._cluster} --project {self._project}')
         run_command(cmd)
 
 

proxy/build.gradle

@@ -23,11 +23,11 @@ task tagProxyImage(dependsOn: buildProxyImage, type: Exec) {
 }
 
 task pushProxyImage(dependsOn: tagProxyImage, type: Exec) {
+  configure verifyDeploymentConfig
   commandLine 'docker', 'push', "gcr.io/${rootProject.gcpProject}/proxy"
 }
 
 task deployProxy(dependsOn: pushProxyImage, type: Exec) {
-  configure verifyDeploymentConfig
   commandLine './deploy-proxy-for-env.sh', "${rootProject.environment}"
 }
 

release/cloudbuild-deploy-gke.yaml

@@ -0,0 +1,65 @@
+# This job deploys auxiliary resources for Nomulus, such as Cloud Scheduler tasks
+# and Cloud Tasks queues.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config=cloudbuild-deploy-gke.yaml \
+# --substitutions=TAG_NAME=[TAG],_ENV=[ENV] ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+# Note: to work around the issue in Spinnaker's 'Deployment Manifest' stage,
+# variable references must avoid the ${var} format. Valid formats include
+# $var or ${"${var}"}. This file uses the former. Since TAG_NAME and _ENV are
+# expanded in the copies sent to Spinnaker, we preserve the brackets around
+# them for safe pattern matching during release.
+# See https://github.com/spinnaker/spinnaker/issues/3028 for more information.
+steps:
+# Pull the credential for nomulus tool.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gcloud secrets versions access latest \
+      --secret nomulus-tool-cloudbuild-credential > tool-credential.json
+# Create/Update cloud scheduler and cloud tasks based on a cloud-scheduler-tasks.xml
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gcloud auth activate-service-account --key-file=tool-credential.json
+    if [ ${_ENV} == production ]; then
+      project_id="domain-registry"
+    else
+      project_id="domain-registry-${_ENV}"
+    fi
+    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/nomulus-config-${_ENV}.yaml .
+    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/cloud-scheduler-tasks-${_ENV}.xml .
+    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/cloud-tasks-queue.xml .
+    deployCloudSchedulerAndQueue nomulus-config-${_ENV}.yaml cloud-scheduler-tasks-${_ENV}.xml $project_id --gke
+    deployCloudSchedulerAndQueue nomulus-config-${_ENV}.yaml cloud-tasks-queue.xml $project_id --gke
+# Save the deployed tag for the current environment on GCS, and update the
+# mappings from Nomulus releases to deployment timestamp.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    echo ${TAG_NAME} | \
+      gsutil cp - gs://$PROJECT_ID-deployed-tags/nomulus-gke.${_ENV}.tag
+    now=$(TZ=UTC date '+%Y-%m-%dT%H:%M:%S.%3NZ')
+    echo "${TAG_NAME},$now" | \
+      gsutil cp - gs://$PROJECT_ID-deployed-tags/nomulus-gke.${_ENV}.tmp
+    # Atomically append uploaded tmp file to nomulus-gke.${_ENV}.versions
+    gsutil compose \
+      gs://$PROJECT_ID-deployed-tags/nomulus-gke.${_ENV}.versions \
+      gs://$PROJECT_ID-deployed-tags/nomulus-gke.${_ENV}.tmp \
+      gs://$PROJECT_ID-deployed-tags/nomulus-gke.${_ENV}.versions
+timeout: 3600s
+options:
+  machineType: 'E2_HIGHCPU_32'

release/cloudbuild-deploy.yaml

@@ -34,11 +34,11 @@ steps:
     else
       project_id="domain-registry-${_ENV}"
     fi
-    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
+    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/nomulus-config-${_ENV}.yaml .
-    tar -xvf ${_ENV}.tar
+    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/cloud-scheduler-tasks-${_ENV}.xml .
-    unzip default/WEB-INF/lib/core.jar
+    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/cloud-tasks-queue.xml .
-    deployCloudSchedulerAndQueue google/registry/config/files/nomulus-config-${_ENV}.yaml default/WEB-INF/cloud-scheduler-tasks.xml $project_id
+    deployCloudSchedulerAndQueue nomulus-config-${_ENV}.yaml cloud-scheduler-tasks-${_ENV}.xml $project_id
-    deployCloudSchedulerAndQueue google/registry/config/files/nomulus-config-${_ENV}.yaml default/WEB-INF/cloud-tasks-queue.xml $project_id
+    deployCloudSchedulerAndQueue nomulus-config-${_ENV}.yaml cloud-tasks-queue.xml $project_id
 # Deploy the GAE config files.
 # First authorize the gcloud tool to use the credential json file, then
 # download and unzip the tarball that contains the relevant config files
@@ -54,6 +54,9 @@ steps:
     else
       project_id="domain-registry-${_ENV}"
     fi
+    gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
+    tar -xvf ${_ENV}.tar
+    unzip default/WEB-INF/lib/core.jar
     gcloud -q --project $project_id app deploy default/WEB-INF/appengine-generated/dispatch.yaml
 # Save the deployed tag for the current environment on GCS, and update the
 # mappings from Nomulus releases to Appengine versions.

release/cloudbuild-nomulus.yaml

@@ -190,6 +190,9 @@ artifacts:
     - 'output/tag_name'
     - 'output/nomulus.jar'
     - 'output/schema.jar'
+    - 'core/src/main/java/google/registry/config/files/nomulus-config-*.yaml'
+    - 'core/src/main/java/google/registry/config/files/cloud-tasks-queue.xml'
+    - 'core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-*.xml'
     - 'release/cloudbuild-sync-and-tag.yaml'
     - 'release/cloudbuild-deploy-*.yaml'
     - 'release/cloudbuild-delete-*.yaml'

release/cloudbuild-release.yaml

@@ -83,6 +83,7 @@ steps:
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-proxy.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-nomulus.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-deploy.yaml
+    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-deploy-gke.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-sync-and-tag.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml
@@ -90,9 +91,12 @@ steps:
     sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml
     sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync-and-tag.yaml
     sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-deploy.yaml
+    sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-deploy-gke.yaml
     for environment in alpha crash qa sandbox production; do
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-deploy.yaml \
         > release/cloudbuild-deploy-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-deploy-gke.yaml \
+        > release/cloudbuild-deploy-gke-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete.yaml \
         > release/cloudbuild-delete-${environment}.yaml
     done
@@ -182,6 +186,11 @@ steps:
           sed s/PROXY_ENV/${env}/g | \
           sed s/EPP/epp/g | \
           sed s/WHOIS/whois/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml
+        # Proxy '--log' flag does not work on production.
+        if [ ${env} == production ]
+        then
+          sed -i s/--log,//g ./jetty/kubernetes/nomulus-${env}-${service}.yaml
+        fi
         if [ ${service} == frontend ]
         then
           sed -i s/${PROJECT_ID}.iam.gserviceaccount.com/${project}.iam.gserviceaccount.com/g \
@@ -195,6 +204,11 @@ steps:
           sed s/WHOIS/whois-canary/g | \
           sed s/${service}/${service}-canary/g \
           > ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml
+        # Proxy '--log' flag does not work on production.
+        if [ ${env} == production ]
+        then
+          sed -i s/--log,//g ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml
+        fi
         if [ ${service} == frontend ]
         then
           sed -i s/${PROJECT_ID}.iam.gserviceaccount.com/${project}.iam.gserviceaccount.com/g \