Expose EPP via saidcar proxy (#2680)

jetty/deploy-nomulus-for-env.sh

@@ -37,15 +37,19 @@ for service in frontend backend pubapi console
 do
   sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
   sed s/ENVIRONMENT/"${environment}"/g | \
+  sed s/PROXY_ENV/"${environment}"/g | \
+  sed s/EPP/"epp"/g | \
   kubectl apply -f -
+  kubectl rollout restart deployment/${service}
   # canary
   sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
   sed s/ENVIRONMENT/"${environment}"/g | \
+  sed s/PROXY_ENV/"${environment}_canary"/g | \
+  sed s/EPP/"epp-canary"/g | \
   sed s/"${service}"/"${service}-canary"/g | \
   kubectl apply -f -
+  kubectl rollout restart deployment/${service}-canary
 done
-# Kills all running pods, new pods created will be pulling the new image.
-kubectl delete pods --all
 kubectl apply -f "./kubernetes/gateway/nomulus-gateway.yaml"
 kubectl apply -f "./kubernetes/gateway/nomulus-iap-${environment}.yaml"
 for service in frontend backend console pubapi
@@ -57,4 +61,16 @@ do
   sed s/SERVICE/"${service}-canary"/g "./kubernetes/gateway/nomulus-backend-policy-${environment}.yaml" | \
   kubectl apply -f -
 done
+
+# Restart proxies
+while read line
+do
+  parts=(${line})
+  echo "Updating cluster ${parts[0]} in location ${parts[1]}..."
+  gcloud container clusters get-credentials ${parts[0]} \
+    --project ${project} --location ${parts[1]}
+  kubectl rollout restart deployment/proxy-deployment
+  kubectl rollout restart deployment/proxy-deployment-canary
+done < <(gcloud container clusters list --project ${project} | grep proxy-cluster)
+
 kubectl config use-context "$current_context"

jetty/kubernetes/nomulus-backend.yaml

@@ -12,6 +12,8 @@ spec:
         service: backend
     spec:
       serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
       containers:
       - name: backend
         image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
           name: http
         resources:
           requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
         args: [ENVIRONMENT]
         env:
         - name: POD_ID

jetty/kubernetes/nomulus-console.yaml

@@ -12,6 +12,8 @@ spec:
         service: console
     spec:
       serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
       containers:
       - name: console
         image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
           name: http
         resources:
           requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
         args: [ENVIRONMENT]
         env:
         - name: POD_ID

jetty/kubernetes/nomulus-frontend.yaml

@@ -12,6 +12,8 @@ spec:
         service: frontend
     spec:
       serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
       containers:
       - name: frontend
         image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
           name: http
         resources:
           requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
         args: [ENVIRONMENT]
         env:
         - name: POD_ID
@@ -37,6 +40,27 @@ spec:
               fieldPath: metadata.namespace
         - name: CONTAINER_NAME
           value: frontend
+      - name: EPP
+        image: gcr.io/GCP_PROJECT/proxy
+        ports:
+        - containerPort: 30002
+          name: epp
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "512Mi"
+        args: [--env, PROXY_ENV, --log, --local]
+        env:
+        - name: POD_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONTAINER_NAME
+          value: EPP
 ---
 # Only need to define the service account once per cluster.
 apiVersion: v1
@@ -55,7 +79,7 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: frontend
-  minReplicas: 5
+  minReplicas: 15
   maxReplicas: 15
   metrics:
     - type: Resource
@@ -77,6 +101,26 @@ spec:
       targetPort: http
       name: http
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: EPP
+  annotations:
+    cloud.google.com/l4-rbs: enabled
+    networking.gke.io/weighted-load-balancing: pods-per-node
+spec:
+  type: LoadBalancer
+  # Traffic is directly delivered to a node, preserving the original source IP.
+  externalTrafficPolicy: Local
+  ipFamilies: [IPv4, IPv6]
+  ipFamilyPolicy: RequireDualStack
+  selector:
+    service: frontend
+  ports:
+  - port: 700
+    targetPort: epp
+    name: epp
+---
 apiVersion: net.gke.io/v1
 kind: ServiceExport
 metadata:

jetty/kubernetes/nomulus-pubapi.yaml

@@ -12,6 +12,8 @@ spec:
         service: pubapi
     spec:
       serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
       containers:
       - name: pubapi
         image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
           name: http
         resources:
           requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
         args: [ENVIRONMENT]
         env:
         - name: POD_ID

release/builder/build.sh

@@ -58,6 +58,8 @@ apt-get install postgresql-client-17 procps -y
 # Install gcloud
 apt-get install google-cloud-cli -y
 apt-get install google-cloud-sdk-app-engine-java -y
+apt-get install kubectl -y
+apt-get install google-cloud-cli-gke-gcloud-auth-plugin -y
 
 # Install git
 apt-get install git -y

release/cloudbuild-nomulus.yaml

@@ -198,6 +198,7 @@ artifacts:
     - 'release/cloudbuild-delete-*.yaml'
     - 'release/cloudbuild-schema-deploy-*.yaml'
     - 'release/cloudbuild-schema-verify-*.yaml'
+    - 'release/cloudbuild-restart-proxies-*.yaml'
     - 'jetty/kubernetes/*.yaml'
     - 'jetty/kubernetes/gateway/*.yaml'
 # The images are already uploaded, but we still need to include them there so that

release/cloudbuild-release.yaml

@@ -88,6 +88,7 @@ steps:
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml
     sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete.yaml
+    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-restart-proxies.yaml
     sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml
     sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync-and-tag.yaml
     sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-deploy.yaml
@@ -99,6 +100,11 @@ steps:
         > release/cloudbuild-deploy-gke-${environment}.yaml
       sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete.yaml \
         > release/cloudbuild-delete-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml \
+        > release/cloudbuild-restart-proxies-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml | \
+        sed s/proxy-deployment/proxy-deployment-canary/g \
+        > release/cloudbuild-restart-proxies-${environment}-canary.yaml
     done
 # Build and upload the schema_deployer image.
 - name: 'gcr.io/cloud-builders/docker'
@@ -178,11 +184,13 @@ steps:
       base_domain=$(grep baseDomain \
         ./core/src/main/java/google/registry/config/files/nomulus-config-${env}.yaml | \
         awk '{print $2}')
-      for service in frontend backend pubapi console 
+      for service in frontend backend pubapi console
       do
         # non-canary
         sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \
-          sed s/ENVIRONMENT/${env}/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml
+          sed s/ENVIRONMENT/${env}/g | \
+          sed s/PROXY_ENV/"${env}"/g | \
+          sed s/EPP/"epp"/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml
         # Proxy '--log' flag does not work on production.
         if [ ${env} == production ]
         then
@@ -196,6 +204,8 @@ steps:
         # canary
         sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \
           sed s/ENVIRONMENT/${env}/g | \
+          sed s/PROXY_ENV/"${env}_canary"/g | \
+          sed s/EPP/"epp-canary"/g | \
           sed s/${service}/${service}-canary/g \
           > ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml
         # Proxy '--log' flag does not work on production.
@@ -264,7 +274,7 @@ steps:
       $(gcloud auth list --format='get(account)' --filter=active)
     git add .
     git commit -m "Release commit for tag ${TAG_NAME}"
-    git push -o nokeycheck origin master 
+    git push -o nokeycheck origin master
     git tag ${TAG_NAME}
     git push -o nokeycheck origin ${TAG_NAME}
 timeout: 3600s

release/cloudbuild-restart-proxies.yaml

@@ -0,0 +1,54 @@
+# This will do rolling restarts of all proxies. This forces the client to reconnect
+# and resets the sessions.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config=cloudbuild-restart-proxies.yaml \
+# --substitutions=_ENV=[ENV] ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+# Note: to work around the issue in Spinnaker's 'Deployment Manifest' stage,
+# variable references must avoid the ${var} format. Valid formats include
+# $var or ${"${var}"}. This file uses the former. Since TAG_NAME and _ENV are
+# expanded in the copies sent to Spinnaker, we preserve the brackets around
+# them for safe pattern matching during release.
+# See https://github.com/spinnaker/spinnaker/issues/3028 for more information.
+steps:
+# Pull the credential for nomulus tool.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gcloud secrets versions access latest \
+      --secret nomulus-tool-cloudbuild-credential > tool-credential.json
+# Do rolling restarts of all proxies in all environments.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    if [ ${_ENV} == production ]
+    then
+      project_id="domain-registry"
+    else
+      project_id="domain-registry-${_ENV}"
+    fi
+
+    gcloud auth activate-service-account --key-file=tool-credential.json
+
+    while read line
+    do
+      name=$(echo $line | awk '{print $1}')
+      location=$(echo $line | awk '{print $2}')
+      echo $name $region
+      echo "Updating cluster $name in location $location..."
+      gcloud container clusters get-credentials $name \
+        --project $project_id --location $location
+      kubectl rollout restart deployment/proxy-deployment
+    done < <(gcloud container clusters list --project $project_id | grep proxy-cluster)
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'