Expose EPP via saidcar proxy (#2680)

2025-12-23 06:15:42 +00:00 · 2025-02-19 13:57:25 -05:00
parent b73e342820
commit 3f2a42ab8d
9 changed files with 146 additions and 10 deletions
--- a/jetty/deploy-nomulus-for-env.sh
+++ b/jetty/deploy-nomulus-for-env.sh
@@ -37,15 +37,19 @@ for service in frontend backend pubapi console
 do
  sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
  sed s/ENVIRONMENT/"${environment}"/g | \
+  sed s/PROXY_ENV/"${environment}"/g | \
+  sed s/EPP/"epp"/g | \
  kubectl apply -f -
+  kubectl rollout restart deployment/${service}
  # canary
  sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
  sed s/ENVIRONMENT/"${environment}"/g | \
+  sed s/PROXY_ENV/"${environment}_canary"/g | \
+  sed s/EPP/"epp-canary"/g | \
  sed s/"${service}"/"${service}-canary"/g | \
  kubectl apply -f -
+  kubectl rollout restart deployment/${service}-canary
 done
-# Kills all running pods, new pods created will be pulling the new image.
-kubectl delete pods --all
 kubectl apply -f "./kubernetes/gateway/nomulus-gateway.yaml"
 kubectl apply -f "./kubernetes/gateway/nomulus-iap-${environment}.yaml"
 for service in frontend backend console pubapi
@@ -57,4 +61,16 @@ do
  sed s/SERVICE/"${service}-canary"/g "./kubernetes/gateway/nomulus-backend-policy-${environment}.yaml" | \
  kubectl apply -f -
 done
+
+# Restart proxies
+while read line
+do
+  parts=(${line})
+  echo "Updating cluster ${parts[0]} in location ${parts[1]}..."
+  gcloud container clusters get-credentials ${parts[0]} \
+    --project ${project} --location ${parts[1]}
+  kubectl rollout restart deployment/proxy-deployment
+  kubectl rollout restart deployment/proxy-deployment-canary
+done < <(gcloud container clusters list --project ${project} | grep proxy-cluster)
+
 kubectl config use-context "$current_context"
--- a/jetty/kubernetes/nomulus-backend.yaml
+++ b/jetty/kubernetes/nomulus-backend.yaml
@@ -12,6 +12,8 @@ spec:
        service: backend
    spec:
      serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
      containers:
      - name: backend
        image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
          name: http
        resources:
          requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
        args: [ENVIRONMENT]
        env:
        - name: POD_ID
--- a/jetty/kubernetes/nomulus-console.yaml
+++ b/jetty/kubernetes/nomulus-console.yaml
@@ -12,6 +12,8 @@ spec:
        service: console
    spec:
      serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
      containers:
      - name: console
        image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
          name: http
        resources:
          requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
        args: [ENVIRONMENT]
        env:
        - name: POD_ID
--- a/jetty/kubernetes/nomulus-frontend.yaml
+++ b/jetty/kubernetes/nomulus-frontend.yaml
@@ -12,6 +12,8 @@ spec:
        service: frontend
    spec:
      serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
      containers:
      - name: frontend
        image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
          name: http
        resources:
          requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
        args: [ENVIRONMENT]
        env:
        - name: POD_ID
@@ -37,6 +40,27 @@ spec:
              fieldPath: metadata.namespace
        - name: CONTAINER_NAME
          value: frontend
+      - name: EPP
+        image: gcr.io/GCP_PROJECT/proxy
+        ports:
+        - containerPort: 30002
+          name: epp
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "512Mi"
+        args: [--env, PROXY_ENV, --log, --local]
+        env:
+        - name: POD_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE_ID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CONTAINER_NAME
+          value: EPP
 ---
 # Only need to define the service account once per cluster.
 apiVersion: v1
@@ -55,7 +79,7 @@ spec:
    apiVersion: apps/v1
    kind: Deployment
    name: frontend
-  minReplicas: 5
+  minReplicas: 15
  maxReplicas: 15
  metrics:
    - type: Resource
@@ -77,6 +101,26 @@ spec:
      targetPort: http
      name: http
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: EPP
+  annotations:
+    cloud.google.com/l4-rbs: enabled
+    networking.gke.io/weighted-load-balancing: pods-per-node
+spec:
+  type: LoadBalancer
+  # Traffic is directly delivered to a node, preserving the original source IP.
+  externalTrafficPolicy: Local
+  ipFamilies: [IPv4, IPv6]
+  ipFamilyPolicy: RequireDualStack
+  selector:
+    service: frontend
+  ports:
+  - port: 700
+    targetPort: epp
+    name: epp
+---
 apiVersion: net.gke.io/v1
 kind: ServiceExport
 metadata:
--- a/jetty/kubernetes/nomulus-pubapi.yaml
+++ b/jetty/kubernetes/nomulus-pubapi.yaml
@@ -12,6 +12,8 @@ spec:
        service: pubapi
    spec:
      serviceAccountName: nomulus
+      nodeSelector:
+        cloud.google.com/compute-class: "Performance"
      containers:
      - name: pubapi
        image: gcr.io/GCP_PROJECT/nomulus
@@ -20,7 +22,8 @@ spec:
          name: http
        resources:
          requests:
-            cpu: "500m"
+            cpu: "100m"
+            memory: "512Mi"
        args: [ENVIRONMENT]
        env:
        - name: POD_ID
--- a/release/builder/build.sh
+++ b/release/builder/build.sh
@@ -58,6 +58,8 @@ apt-get install postgresql-client-17 procps -y
 # Install gcloud
 apt-get install google-cloud-cli -y
 apt-get install google-cloud-sdk-app-engine-java -y
+apt-get install kubectl -y
+apt-get install google-cloud-cli-gke-gcloud-auth-plugin -y

 # Install git
 apt-get install git -y
--- a/release/cloudbuild-nomulus.yaml
+++ b/release/cloudbuild-nomulus.yaml
@@ -198,6 +198,7 @@ artifacts:
    - 'release/cloudbuild-delete-*.yaml'
    - 'release/cloudbuild-schema-deploy-*.yaml'
    - 'release/cloudbuild-schema-verify-*.yaml'
+    - 'release/cloudbuild-restart-proxies-*.yaml'
    - 'jetty/kubernetes/*.yaml'
    - 'jetty/kubernetes/gateway/*.yaml'
 # The images are already uploaded, but we still need to include them there so that
--- a/release/cloudbuild-release.yaml
+++ b/release/cloudbuild-release.yaml
@@ -88,6 +88,7 @@ steps:
    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-deploy.yaml
    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-schema-verify.yaml
    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-delete.yaml
+    sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-restart-proxies.yaml
    sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy/kubernetes/proxy-*.yaml
    sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-sync-and-tag.yaml
    sed -i s/'$${TAG_NAME}'/${TAG_NAME}/g release/cloudbuild-deploy.yaml
@@ -99,6 +100,11 @@ steps:
        > release/cloudbuild-deploy-gke-${environment}.yaml
      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-delete.yaml \
        > release/cloudbuild-delete-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml \
+        > release/cloudbuild-restart-proxies-${environment}.yaml
+      sed s/'$${_ENV}'/${environment}/g release/cloudbuild-restart-proxies.yaml | \
+        sed s/proxy-deployment/proxy-deployment-canary/g \
+        > release/cloudbuild-restart-proxies-${environment}-canary.yaml
    done
 # Build and upload the schema_deployer image.
 - name: 'gcr.io/cloud-builders/docker'
@@ -182,7 +188,9 @@ steps:
      do
        # non-canary
        sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \
-          sed s/ENVIRONMENT/${env}/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml
+          sed s/ENVIRONMENT/${env}/g | \
+          sed s/PROXY_ENV/"${env}"/g | \
+          sed s/EPP/"epp"/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml
        # Proxy '--log' flag does not work on production.
        if [ ${env} == production ]
        then
@@ -196,6 +204,8 @@ steps:
        # canary
        sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \
          sed s/ENVIRONMENT/${env}/g | \
+          sed s/PROXY_ENV/"${env}_canary"/g | \
+          sed s/EPP/"epp-canary"/g | \
          sed s/${service}/${service}-canary/g \
          > ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml
        # Proxy '--log' flag does not work on production.
--- a/release/cloudbuild-restart-proxies.yaml
+++ b/release/cloudbuild-restart-proxies.yaml
@@ -0,0 +1,54 @@
+# This will do rolling restarts of all proxies. This forces the client to reconnect
+# and resets the sessions.
+#
+# To manually trigger a build on GCB, run:
+# gcloud builds submit --config=cloudbuild-restart-proxies.yaml \
+# --substitutions=_ENV=[ENV] ..
+#
+# To trigger a build automatically, follow the instructions below and add a trigger:
+# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
+#
+# Note: to work around the issue in Spinnaker's 'Deployment Manifest' stage,
+# variable references must avoid the ${var} format. Valid formats include
+# $var or ${"${var}"}. This file uses the former. Since TAG_NAME and _ENV are
+# expanded in the copies sent to Spinnaker, we preserve the brackets around
+# them for safe pattern matching during release.
+# See https://github.com/spinnaker/spinnaker/issues/3028 for more information.
+steps:
+# Pull the credential for nomulus tool.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gcloud secrets versions access latest \
+      --secret nomulus-tool-cloudbuild-credential > tool-credential.json
+# Do rolling restarts of all proxies in all environments.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    if [ ${_ENV} == production ]
+    then
+      project_id="domain-registry"
+    else
+      project_id="domain-registry-${_ENV}"
+    fi
+
+    gcloud auth activate-service-account --key-file=tool-credential.json
+
+    while read line
+    do
+      name=$(echo $line | awk '{print $1}')
+      location=$(echo $line | awk '{print $2}')
+      echo $name $region
+      echo "Updating cluster $name in location $location..."
+      gcloud container clusters get-credentials $name \
+        --project $project_id --location $location
+      kubectl rollout restart deployment/proxy-deployment
+    done < <(gcloud container clusters list --project $project_id | grep proxy-cluster)
+timeout: 3600s
+options:
+  machineType: 'N1_HIGHCPU_8'