Make GKE networking work more properly (#2531)

jetty/build.gradle

@@ -85,7 +85,7 @@ tasks.register('run', JavaExec) {
 tasks.register('deployNomulus', Exec) {
     dependsOn('pushNomulusImage', ':proxy:pushProxyImage')
     configure verifyDeploymentConfig
-    commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}"
+    commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}", "${rootProject.baseDomain}"
 }
 
 project.build.dependsOn(tasks.named('buildNomulusImage'))

jetty/deploy-nomulus-for-env.sh

@@ -17,12 +17,13 @@
 # kills all running pods to force k8s to create new pods using the just-pushed
 # manifest.
 
-if [[ $# -ne 1 ]]; then
-  echo "Usage: $0 alpha|crash|qa"
+if [[ $# -ne 2 ]]; then
+  echo "Usage: $0 alpha|crash|qa [base_domain]}"
   exit 1
 fi
 
 environment=${1}
+base_domain=${2}
 project="domain-registry-"${environment}
 current_context=$(kubectl config current-context)
 while read line
@@ -31,16 +32,25 @@ do
   echo "Updating cluster ${parts[0]} in location ${parts[1]}..."
   gcloud container clusters get-credentials "${parts[0]}" \
     --project "${project}" --location "${parts[1]}"
-  sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-deployment.yaml" | \
-  sed s/ENVIRONMENT/"${environment}"/g | \
-  kubectl apply -f -
-  kubectl apply -f "./kubernetes/nomulus-service.yaml"
+  for service in frontend backend pubapi console
+  do
+    sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \
+    sed s/ENVIRONMENT/"${environment}"/g | \
+    kubectl apply -f -
+  done
   # Kills all running pods, new pods created will be pulling the new image.
   kubectl delete pods --all
   # The multi-cluster gateway is only deployed to one cluster (the one in the US).
   if [[ "${parts[1]}" == us-* ]]
   then
-    kubectl apply -f "./kubernetes/nomulus-gateway.yaml"
+    kubectl apply -f "./kubernetes/gateway/nomulus-gateway.yaml"
+    for service in frontend backend pubapi console
+    do
+      sed s/BASE_DOMAIN/"${base_domain}"/g "./kubernetes/gateway/nomulus-route-${service}.yaml" | \
+      kubectl apply -f -
+      sed s/SERVICE/"${service}"/g "./kubernetes/gateway/nomulus-iap-${environment}.yaml" | \
+      kubectl apply -f -
+    done
   fi
 done < <(gcloud container clusters list --project "${project}" | grep nomulus)
 kubectl config use-context "$current_context"

jetty/kubernetes/gateway/nomulus-gateway.yaml

@@ -0,0 +1,17 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: Gateway
+metadata:
+  name: nomulus
+spec:
+  gatewayClassName: gke-l7-global-external-managed-mc
+  listeners:
+  - name: https
+    protocol: HTTPS
+    port: 443
+    tls:
+      mode: Terminate
+      options:
+        networking.gke.io/pre-shared-certs: nomulus
+    allowedRoutes:
+      kinds:
+      - kind: HTTPRoute

jetty/kubernetes/gateway/nomulus-route-backend.yaml

@@ -0,0 +1,47 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: backend
+spec:
+  parentRefs:
+  - kind: Gateway
+    name: nomulus
+  hostnames:
+    - "backend.BASE_DOMAIN"
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /_dr/task
+    - path:
+        type: PathPrefix
+        value: /_dr/cron
+    - path:
+        type: PathPrefix
+        value: /_dr/admin
+    - path:
+        type: PathPrefix
+        value: /_dr/epptool
+    - path:
+        type: PathPrefix
+        value: /loadtest
+    backendRefs:
+    - group: net.gke.io
+      kind: ServiceImport
+      name: backend
+      port: 80
+---
+apiVersion: networking.gke.io/v1
+kind: HealthCheckPolicy
+metadata:
+  name: backend
+spec:
+  default:
+    config:
+      type: HTTP
+      httpHealthCheck:
+        requestPath: /healthz/
+  targetRef:
+    group: net.gke.io
+    kind: ServiceImport
+    name: backend

jetty/kubernetes/gateway/nomulus-route-console.yaml

@@ -1,38 +1,31 @@
-kind: Gateway
 apiVersion: gateway.networking.k8s.io/v1beta1
-metadata:
-  name: nomulus
-spec:
-  gatewayClassName: gke-l7-global-external-managed-mc
-  listeners:
-  - name: http
-    protocol: HTTP
-    port: 80
-    allowedRoutes:
-      kinds:
-      - kind: HTTPRoute
----
 kind: HTTPRoute
-apiVersion: gateway.networking.k8s.io/v1beta1
 metadata:
-  name: nomulus
-  labels:
-    app: nomulus
+  name: console
 spec:
   parentRefs:
   - kind: Gateway
     name: nomulus
+  hostnames:
+    - "console.BASE_DOMAIN"
   rules:
-  - backendRefs:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /console-api
+    - path:
+        type: PathPrefix
+        value: /console
+    backendRefs:
     - group: net.gke.io
       kind: ServiceImport
-      name: nomulus
+      name: console
       port: 80
 ---
 apiVersion: networking.gke.io/v1
 kind: HealthCheckPolicy
 metadata:
-  name: nomulus
+  name: console
 spec:
   default:
     config:
@@ -42,5 +35,4 @@ spec:
   targetRef:
     group: net.gke.io
     kind: ServiceImport
-    name: nomulus
-
+    name: console

jetty/kubernetes/gateway/nomulus-route-frontend.yaml

@@ -0,0 +1,35 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: frontend
+spec:
+  parentRefs:
+  - kind: Gateway
+    name: nomulus
+  hostnames:
+    - "frontend.BASE_DOMAIN"
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /_dr/epp
+    backendRefs:
+    - group: net.gke.io
+      kind: ServiceImport
+      name: frontend
+      port: 80
+---
+apiVersion: networking.gke.io/v1
+kind: HealthCheckPolicy
+metadata:
+  name: frontend
+spec:
+  default:
+    config:
+      type: HTTP
+      httpHealthCheck:
+        requestPath: /healthz/
+  targetRef:
+    group: net.gke.io
+    kind: ServiceImport
+    name: frontend

jetty/kubernetes/gateway/nomulus-route-pubapi.yaml

@@ -0,0 +1,44 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: pubapi
+spec:
+  parentRefs:
+  - kind: Gateway
+    name: nomulus
+  hostnames:
+    - "pubapi.BASE_DOMAIN"
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /_dr/whois
+    - path:
+        type: PathPrefix
+        value: /check
+    - path:
+        type: PathPrefix
+        value: /whois
+    - path:
+        type: PathPrefix
+        value: /rdap
+    backendRefs:
+    - group: net.gke.io
+      kind: ServiceImport
+      name: pubapi
+      port: 80
+---
+apiVersion: networking.gke.io/v1
+kind: HealthCheckPolicy
+metadata:
+  name: pubapi
+spec:
+  default:
+    config:
+      type: HTTP
+      httpHealthCheck:
+        requestPath: /healthz/
+  targetRef:
+    group: net.gke.io
+    kind: ServiceImport
+    name: pubapi

jetty/kubernetes/nomulus-backend.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend
+spec:
+  selector:
+    matchLabels:
+      service: backend
+  template:
+    metadata:
+      labels:
+        service: backend
+    spec:
+      serviceAccountName: nomulus
+      containers:
+      - name: backend
+        image: gcr.io/GCP_PROJECT/nomulus
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: "500m"
+        args: [ENVIRONMENT]
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: backend
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: backend
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 100
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: backend
+spec:
+  selector:
+    service: backend
+  ports:
+    - port: 80
+      targetPort: http
+      name: http
+---
+apiVersion: net.gke.io/v1
+kind: ServiceExport
+metadata:
+  name: backend

jetty/kubernetes/nomulus-console.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: console
+spec:
+  selector:
+    matchLabels:
+      service: console
+  template:
+    metadata:
+      labels:
+        service: console
+    spec:
+      serviceAccountName: nomulus
+      containers:
+      - name: console
+        image: gcr.io/GCP_PROJECT/nomulus
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: "500m"
+        args: [ENVIRONMENT]
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: console
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: console
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 100
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: console
+spec:
+  selector:
+    service: console
+  ports:
+    - port: 80
+      targetPort: http
+      name: http
+---
+apiVersion: net.gke.io/v1
+kind: ServiceExport
+metadata:
+  name: console

jetty/kubernetes/nomulus-frontend.yaml

@@ -1,21 +1,19 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: nomulus
-  labels:
-    app: nomulus
+  name: frontend
 spec:
   selector:
     matchLabels:
-      app: nomulus
+      service: frontend
   template:
     metadata:
       labels:
-        app: nomulus
+        service: frontend
     spec:
       serviceAccountName: nomulus
       containers:
-      - name: nomulus
+      - name: frontend
         image: gcr.io/GCP_PROJECT/nomulus
         ports:
         - containerPort: 8080
@@ -50,14 +48,12 @@ spec:
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
-  name: nomulus
-  labels:
-    app: nomulus
+  name: frontend
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: nomulus
+    name: frontend
   minReplicas: 1
   maxReplicas: 20
   metrics:
@@ -67,4 +63,26 @@ spec:
         target:
           type: Utilization
           averageUtilization: 100
-
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frontend
+spec:
+  selector:
+    service: frontend
+  ports:
+    - port: 80
+      targetPort: http
+      name: http
+    - port: 43
+      targetPort: whois
+      name: whois
+    - port: 700
+      targetPort: epp
+      name: epp
+---
+apiVersion: net.gke.io/v1
+kind: ServiceExport
+metadata:
+  name: frontend

jetty/kubernetes/nomulus-pubapi.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: pubapi
+spec:
+  selector:
+    matchLabels:
+      service: pubapi
+  template:
+    metadata:
+      labels:
+        service: pubapi
+    spec:
+      serviceAccountName: nomulus
+      containers:
+      - name: pubapi
+        image: gcr.io/GCP_PROJECT/nomulus
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: "500m"
+        args: [ENVIRONMENT]
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: pubapi
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: pubapi
+  minReplicas: 1
+  maxReplicas: 20
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 100
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: pubapi
+spec:
+  selector:
+    service: pubapi
+  ports:
+    - port: 80
+      targetPort: http
+      name: http
+---
+apiVersion: net.gke.io/v1
+kind: ServiceExport
+metadata:
+  name: pubapi

jetty/kubernetes/nomulus-service.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: nomulus
-spec:
-  selector:
-    app: nomulus
-  ports:
-    - port: 80
-      targetPort: http
-      name: http
-    - port: 43
-      targetPort: whois
-      name: whois
-    - port: 700
-      targetPort: epp
-      name: epp
----
-kind: ServiceExport
-apiVersion: net.gke.io/v1
-metadata:
-  name: nomulus