Update Nomulus deployment script (#2677)

We only deploy to the us-central1 cluster in order to minimize database
locality issue.

jetty/deploy-nomulus-for-env.sh

@@ -29,48 +29,37 @@ environment=${1}
 base_domain=${2}
 project="domain-registry-"${environment}
 current_context=$(kubectl config current-context)
-while read line
+line=$(gcloud container clusters list --project "${project}" | grep nomulus | grep main)
+parts=(${line})
+echo "Updating cluster ${parts[0]} in location ${parts[1]}..."
+gcloud container fleet memberships get-credentials "${parts[0]}" --project "${project}"
+for service in frontend backend pubapi console
 do
-  parts=(${line})
+  sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
-  echo "Updating cluster ${parts[0]} in location ${parts[1]}..."
+  sed s/ENVIRONMENT/"${environment}"/g | \
-  gcloud container fleet memberships get-credentials "${parts[0]}" --project "${project}"
+  kubectl apply -f -
-  for service in frontend backend pubapi console
+  # canary
-  do
+  sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
-    sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
+  sed s/ENVIRONMENT/"${environment}"/g | \
-    sed s/ENVIRONMENT/"${environment}"/g | \
+  sed s/"${service}"/"${service}-canary"/g | \
-    sed s/PROXY_ENV/"${environment}"/g | \
+  kubectl apply -f -
-    sed s/EPP/"epp"/g | \
+done
-    sed s/WHOIS/"whois"/g | \
+# Kills all running pods, new pods created will be pulling the new image.
-    kubectl apply -f -
+kubectl delete pods --all
-    # canary
+kubectl apply -f "./kubernetes/gateway/nomulus-gateway.yaml"
-    sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
+kubectl apply -f "./kubernetes/gateway/nomulus-iap-${environment}.yaml"
-    sed s/ENVIRONMENT/"${environment}"/g | \
+for service in frontend backend console pubapi
-    sed s/PROXY_ENV/"${environment}_canary"/g | \
+do
-    sed s/EPP/"epp-canary"/g | \
+  sed s/BASE_DOMAIN/"${base_domain}"/g "./kubernetes/gateway/nomulus-route-${service}.yaml" | \
-    sed s/WHOIS/"whois-canary"/g | \
+  kubectl apply -f -
-    sed s/"${service}"/"${service}-canary"/g | \
+  # Don't enable IAP on pubapi.
-    kubectl apply -f -
+  if [[ "${service}" == pubapi ]]
-  done
-  # Kills all running pods, new pods created will be pulling the new image.
-  kubectl delete pods --all
-  # The multi-cluster gateway is only deployed to one cluster (the one in the US).
-  if [[ "${parts[1]}" == us-* ]]
   then
-    kubectl apply -f "./kubernetes/gateway/nomulus-gateway.yaml"
+    continue
-    for service in frontend backend console pubapi
-    do
-      sed s/BASE_DOMAIN/"${base_domain}"/g "./kubernetes/gateway/nomulus-route-${service}.yaml" | \
-      kubectl apply -f -
-      # Don't enable IAP on pubapi.
-      if [[ "${service}" == pubapi ]]
-      then
-        continue
-      fi
-      sed s/SERVICE/"${service}"/g "./kubernetes/gateway/nomulus-iap-${environment}.yaml" | \
-      kubectl apply -f -
-      sed s/SERVICE/"${service}-canary"/g "./kubernetes/gateway/nomulus-iap-${environment}.yaml" | \
-      kubectl apply -f -
-    done
   fi
-done < <(gcloud container clusters list --project "${project}" | grep nomulus)
+  sed s/SERVICE/"${service}"/g "./kubernetes/gateway/nomulus-backend-policy-${environment}.yaml" | \
+  kubectl apply -f -
+  sed s/SERVICE/"${service}-canary"/g "./kubernetes/gateway/nomulus-backend-policy-${environment}.yaml" | \
+  kubectl apply -f -
+done
 kubectl config use-context "$current_context"

jetty/kubernetes/nomulus-backend.yaml

@@ -43,8 +43,8 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: backend
-  minReplicas: 1
+  minReplicas: 2
-  maxReplicas: 20
+  maxReplicas: 5
   metrics:
     - type: Resource
       resource:

jetty/kubernetes/nomulus-console.yaml

@@ -44,7 +44,7 @@ spec:
     kind: Deployment
     name: console
   minReplicas: 1
-  maxReplicas: 20
+  maxReplicas: 5
   metrics:
     - type: Resource
       resource:

jetty/kubernetes/nomulus-frontend.yaml

@@ -33,26 +33,6 @@ spec:
               fieldPath: metadata.namespace
         - name: CONTAINER_NAME
           value: frontend
-      - name: EPP
-        image: gcr.io/GCP_PROJECT/proxy
-        ports:
-        - containerPort: 30002
-          name: epp
-        resources:
-          requests:
-            cpu: "500m"
-        args: [--env, PROXY_ENV, --log, --local]
-        env:
-        - name: POD_ID
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: NAMESPACE_ID
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: CONTAINER_NAME
-          value: EPP
 ---
 # Only need to define the service account once per cluster.
 apiVersion: v1
@@ -71,8 +51,8 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: frontend
-  minReplicas: 1
+  minReplicas: 5
-  maxReplicas: 20
+  maxReplicas: 15
   metrics:
     - type: Resource
       resource:
@@ -93,26 +73,6 @@ spec:
       targetPort: http
       name: http
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: EPP
-  annotations:
-    cloud.google.com/l4-rbs: enabled
-    networking.gke.io/weighted-load-balancing: pods-per-node
-spec:
-  type: LoadBalancer
-  # Traffic is directly delivered to a node, preserving the original source IP.
-  externalTrafficPolicy: Local
-  ipFamilies: [IPv4, IPv6]
-  ipFamilyPolicy: RequireDualStack
-  selector:
-    service: frontend
-  ports:
-  - port: 700
-    targetPort: epp
-    name: epp
----
 apiVersion: net.gke.io/v1
 kind: ServiceExport
 metadata:

jetty/kubernetes/nomulus-pubapi.yaml

@@ -33,30 +33,6 @@ spec:
               fieldPath: metadata.namespace
         - name: CONTAINER_NAME
           value: pubapi
-      - name: WHOIS
-        image: gcr.io/GCP_PROJECT/proxy
-        ports:
-        - containerPort: 30001
-          name: whois
-        - containerPort: 30010
-          name: http-whois
-        - containerPort: 30011
-          name: https-whois
-        resources:
-          requests:
-            cpu: "500m"
-        args: [ --env, PROXY_ENV, --log, --local ]
-        env:
-        - name: POD_ID
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: NAMESPACE_ID
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: CONTAINER_NAME
-          value: WHOIS
 ---
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
@@ -67,8 +43,8 @@ spec:
     apiVersion: apps/v1
     kind: Deployment
     name: pubapi
-  minReplicas: 1
+  minReplicas: 5
-  maxReplicas: 20
+  maxReplicas: 15
   metrics:
     - type: Resource
       resource:
@@ -89,32 +65,6 @@ spec:
       targetPort: http
       name: http
 ---
-apiVersion: v1
-kind: Service
-metadata:
-  name: WHOIS
-  annotations:
-    cloud.google.com/l4-rbs: enabled
-    networking.gke.io/weighted-load-balancing: pods-per-node
-spec:
-  type: LoadBalancer
-  # Traffic is directly delivered to a node, preserving the original source IP.
-  externalTrafficPolicy: Local
-  ipFamilies: [IPv4, IPv6]
-  ipFamilyPolicy: RequireDualStack
-  selector:
-    service: pubapi
-  ports:
-  - port: 43
-    targetPort: whois
-    name: whois
-  - port: 80
-    targetPort: http-whois
-    name: http-whois
-  - port: 443
-    targetPort: https-whois
-    name: https-whois
----
 apiVersion: net.gke.io/v1
 kind: ServiceExport
 metadata:

release/cloudbuild-release.yaml

@@ -182,10 +182,7 @@ steps:
       do
         # non-canary
         sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \
-          sed s/ENVIRONMENT/${env}/g | \
+          sed s/ENVIRONMENT/${env}/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml
-          sed s/PROXY_ENV/${env}/g | \
-          sed s/EPP/epp/g | \
-          sed s/WHOIS/whois/g > ./jetty/kubernetes/nomulus-${env}-${service}.yaml
         # Proxy '--log' flag does not work on production.
         if [ ${env} == production ]
         then
@@ -199,9 +196,6 @@ steps:
         # canary
         sed s/GCP_PROJECT/${PROJECT_ID}/g ./jetty/kubernetes/nomulus-${service}.yaml | \
           sed s/ENVIRONMENT/${env}/g | \
-          sed s/PROXY_ENV/${env}_canary/g | \
-          sed s/EPP/epp-canary/g | \
-          sed s/WHOIS/whois-canary/g | \
           sed s/${service}/${service}-canary/g \
           > ./jetty/kubernetes/nomulus-${env}-${service}-canary.yaml
         # Proxy '--log' flag does not work on production.
@@ -218,11 +212,11 @@ steps:
         sed s/BASE_DOMAIN/${base_domain}/g \
           ./jetty/kubernetes/gateway/nomulus-route-${service}.yaml \
           > ./jetty/kubernetes/gateway/nomulus-route-${env}-${service}.yaml
-        # IAP
+        # GCP backend policy
-          sed s/SERVICE/${service}/g ./jetty/kubernetes/gateway/nomulus-iap-${env}.yaml \
+          sed s/SERVICE/${service}/g ./jetty/kubernetes/gateway/nomulus-backend-policy-${env}.yaml \
-            > ./jetty/kubernetes/gateway/nomulus-iap-${env}-${service}.yaml
+            > ./jetty/kubernetes/gateway/nomulus-backend-policy-${env}-${service}.yaml
-          sed s/SERVICE/${service}-canary/g ./jetty/kubernetes/gateway/nomulus-iap-${env}.yaml \
+          sed s/SERVICE/${service}-canary/g ./jetty/kubernetes/gateway/nomulus-backend-policy-${env}.yaml \
-            > ./jetty/kubernetes/gateway/nomulus-iap-${env}-${service}-canary.yaml
+            > ./jetty/kubernetes/gateway/nomulus-backend-policy-${env}-${service}-canary.yaml
       done
     done
 # Upload the Gradle binary to GCS if it does not exist and point URL in Gradle wrapper to it.