From 7641b05f122241c5cd0fdc395afc5b1645784891 Mon Sep 17 00:00:00 2001 From: Lai Jiang Date: Thu, 26 Dec 2024 10:25:02 -0500 Subject: [PATCH] Expose EPP and WHOIS endpoints on reginal load balancers (#2627) k8s does not have a way to expose a global load balancer with TCP endpoints, and setting up node port-based routing is a chore, even with Terraform (which is what we did with the standalone proxy). We will use Cloud DNS's geolocation routing policy to ensure that clients connect to the endpoint closest to them. --- jetty/build.gradle | 5 + jetty/deploy-nomulus-for-env.sh | 6 +- jetty/get-endpoints.py | 157 +++++++++++++++++++++++++ jetty/kubernetes/nomulus-frontend.yaml | 27 ++++- jetty/kubernetes/nomulus-pubapi.yaml | 27 ++++- 5 files changed, 210 insertions(+), 12 deletions(-) create mode 100755 jetty/get-endpoints.py diff --git a/jetty/build.gradle b/jetty/build.gradle index 0b93a173f..7946db043 100644 --- a/jetty/build.gradle +++ b/jetty/build.gradle @@ -116,4 +116,9 @@ tasks.register('deployNomulus', Exec) { commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}", "${rootProject.baseDomain}" } +tasks.register('getEndpoints', Exec) { + configure verifyDeploymentConfig + commandLine './get-endpoints.py', "${rootProject.gcpProject}" +} + project.build.dependsOn(tasks.named('buildNomulusImage')) diff --git a/jetty/deploy-nomulus-for-env.sh b/jetty/deploy-nomulus-for-env.sh index 1de215680..bf0e8c47e 100755 --- a/jetty/deploy-nomulus-for-env.sh +++ b/jetty/deploy-nomulus-for-env.sh @@ -37,13 +37,15 @@ do sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \ sed s/ENVIRONMENT/"${environment}"/g | \ sed s/PROXY_ENV/"${environment}"/g | \ - sed s/PROXY_NAME/"proxy"/g | \ + sed s/EPP/"epp"/g | \ + sed s/WHOIS/"whois"/g | \ kubectl apply -f - # canary sed s/GCP_PROJECT/"${project}"/g "./kubernetes/nomulus-${service}.yaml" | \ sed s/ENVIRONMENT/"${environment}"/g | \ sed s/PROXY_ENV/"${environment}_canary"/g | \ - sed s/PROXY_NAME/"proxy-canary"/g | \ + sed s/EPP/"epp-canary"/g | \ + sed s/WHOIS/"whois-canary"/g | \ sed s/"${service}"/"${service}-canary"/g | \ kubectl apply -f - done diff --git a/jetty/get-endpoints.py b/jetty/get-endpoints.py new file mode 100755 index 000000000..d2307fb03 --- /dev/null +++ b/jetty/get-endpoints.py @@ -0,0 +1,157 @@ +#! /bin/env python3 +# Copyright 2024 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +A script that outputs the IP endpoints of various load balancers, to be run +after Nomulus is deployed. +''' + +import ipaddress +import json +import subprocess +import sys +from dataclasses import dataclass +from ipaddress import IPv4Address +from ipaddress import IPv6Address +from operator import attrgetter +from operator import methodcaller + + +class PreserveContext: + def __enter__(self): + self._context = run_command('kubectl config current-context') + + def __exit__(self, type, value, traceback): + run_command('kubectl config use-context ' + self._context) + + +class UseCluster(PreserveContext): + def __init__(self, cluster: str, region: str, project: str): + self._cluster = cluster + self._region = region + self._project = project + + def __enter__(self): + super().__enter__() + cmd = f'gcloud container clusters get-credentials {self._cluster} --location {self._region} --project {self._project}' + run_command(cmd) + + +def run_command(cmd: str, print_output=False) -> str: + proc = subprocess.run(cmd, text=True, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if print_output: + print(proc.stdout) + return proc.stdout + + +def get_clusters(project: str) -> dict[str, str]: + cmd = f'gcloud container clusters list --project {project} --format=json' + content = json.loads(run_command(cmd)) + res = {} + for item in content: + name = item['name'] + region = item['location'] + if not name.startswith('nomulus-cluster'): + continue + res[name] = region + return res + + +def get_endpoints(resource: str, service: str, jsonpath: str) -> list[ + str]: + content = run_command( + f'kubectl get {resource}/{service} -o jsonpath={jsonpath}', ) + return content.split() + + +def get_region_symbol(region: str) -> str: + if region.startswith('us'): + return 'amer' + if region.startswith('europe'): + return 'emea' + if region.startswith('asia'): + return 'apac' + return 'other' + + +@dataclass +class IP: + service: str + region: str + address: IPv4Address | IPv6Address + + def is_ipv6(self) -> bool: + return self.address.version == 6 + + def __str__(self) -> str: + return f'{self.service} {self.region}: {self.address}' + + +def terraform_str(item) -> str: + res = "" + if (isinstance(item, dict)): + res += '{\n' + for key, value in item.items(): + res += f'{key} = {terraform_str(value)}\n' + res += '}' + elif (isinstance(item, list)): + res += '[' + for i, value in enumerate(item): + if i != 0: + res += ', ' + res += terraform_str(value) + res += ']' + else: + res += f'"{item}"' + return res + + +if __name__ == '__main__': + if len(sys.argv) != 2: + raise ValueError('Usage: get-endpoints.py ') + project = sys.argv[1] + print(f'Project: {project}') + clusters = get_clusters(project) + ips = [] + res = {} + for cluster, region in clusters.items(): + with UseCluster(cluster, region, project): + for service in ['whois', 'whois-canary', 'epp', 'epp-canary']: + map_key = service.replace('-', '_') + for ip in get_endpoints('services', service, + '{.status.loadBalancer.ingress[*].ip}'): + ip = ipaddress.ip_address(ip) + if isinstance(ip, IPv4Address): + map_key_with_iptype = map_key + '_ipv4' + else: + map_key_with_iptype = map_key + '_ipv6' + if map_key_with_iptype not in res: + res[map_key_with_iptype] = {} + res[map_key_with_iptype][get_region_symbol(region)] = [ip] + ips.append(IP(service, get_region_symbol(region), ip)) + if not region.startswith('us'): + continue + ip = get_endpoints('gateways.gateway.networking.k8s.io', 'nomulus', + '{.status.addresses[*].value}')[0] + print(f'nomulus: {ip}') + res['https_ip'] = ipaddress.ip_address(ip) + ips.sort(key=attrgetter('region')) + ips.sort(key=methodcaller('is_ipv6')) + ips.sort(key=attrgetter('service')) + for ip in ips: + print(ip) + print("Terraform friendly output:") + print(terraform_str(res)) diff --git a/jetty/kubernetes/nomulus-frontend.yaml b/jetty/kubernetes/nomulus-frontend.yaml index 460ba7a3d..aa950d788 100644 --- a/jetty/kubernetes/nomulus-frontend.yaml +++ b/jetty/kubernetes/nomulus-frontend.yaml @@ -33,7 +33,7 @@ spec: fieldPath: metadata.namespace - name: CONTAINER_NAME value: frontend - - name: PROXY_NAME + - name: EPP image: gcr.io/GCP_PROJECT/proxy ports: - containerPort: 30002 @@ -52,7 +52,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CONTAINER_NAME - value: PROXY_NAME + value: EPP --- # Only need to define the service account once per cluster. apiVersion: v1 @@ -92,9 +92,26 @@ spec: - port: 80 targetPort: http name: http - - port: 700 - targetPort: epp - name: epp +--- +apiVersion: v1 +kind: Service +metadata: + name: EPP + annotations: + cloud.google.com/l4-rbs: enabled + networking.gke.io/weighted-load-balancing: pods-per-node +spec: + type: LoadBalancer + # Traffic is directly delivered to a node, preserving the original source IP. + externalTrafficPolicy: Local + ipFamilies: [IPv4, IPv6] + ipFamilyPolicy: RequireDualStack + selector: + service: frontend + ports: + - port: 700 + targetPort: epp + name: epp --- apiVersion: net.gke.io/v1 kind: ServiceExport diff --git a/jetty/kubernetes/nomulus-pubapi.yaml b/jetty/kubernetes/nomulus-pubapi.yaml index 42d3daa3b..579c7d4cb 100644 --- a/jetty/kubernetes/nomulus-pubapi.yaml +++ b/jetty/kubernetes/nomulus-pubapi.yaml @@ -33,7 +33,7 @@ spec: fieldPath: metadata.namespace - name: CONTAINER_NAME value: pubapi - - name: PROXY_NAME + - name: WHOIS image: gcr.io/GCP_PROJECT/proxy ports: - containerPort: 30001 @@ -52,7 +52,7 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CONTAINER_NAME - value: PROXY_NAME + value: WHOIS --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler @@ -84,9 +84,26 @@ spec: - port: 80 targetPort: http name: http - - port: 43 - targetPort: whois - name: whois +--- +apiVersion: v1 +kind: Service +metadata: + name: WHOIS + annotations: + cloud.google.com/l4-rbs: enabled + networking.gke.io/weighted-load-balancing: pods-per-node +spec: + type: LoadBalancer + # Traffic is directly delivered to a node, preserving the original source IP. + externalTrafficPolicy: Local + ipFamilies: [IPv4, IPv6] + ipFamilyPolicy: RequireDualStack + selector: + service: pubapi + ports: + - port: 43 + targetPort: whois + name: whois --- apiVersion: net.gke.io/v1 kind: ServiceExport