From e434528cd3d9b579c1106e1fcdea29f3e0d096d6 Mon Sep 17 00:00:00 2001 From: Lai Jiang Date: Thu, 11 Apr 2024 15:01:09 -0400 Subject: [PATCH] Add nomulus deployment and service manifests (#2389) --- jetty/build.gradle | 15 ++++ jetty/deploy-nomulus-for-env.sh | 42 +++++++++++ jetty/kubernetes/nomulus-deployment.yaml | 69 +++++++++++++++++++ jetty/kubernetes/nomulus-gateway.yaml | 31 +++++++++ jetty/kubernetes/nomulus-service.yaml | 22 ++++++ proxy/build.gradle | 22 +++--- proxy/deploy-proxy-for-env.sh | 4 +- .../registry/proxy/EppProtocolModule.java | 17 +++-- .../proxy/HttpsRelayProtocolModule.java | 35 +++++++--- .../java/google/registry/proxy/Protocol.java | 19 ++++- .../google/registry/proxy/ProxyConfig.java | 1 + .../google/registry/proxy/ProxyModule.java | 17 ++++- .../registry/proxy/WhoisProtocolModule.java | 14 ++-- .../registry/proxy/config/default-config.yaml | 2 +- .../registry/proxy/ProtocolModuleTest.java | 19 +++-- 15 files changed, 287 insertions(+), 42 deletions(-) create mode 100755 jetty/deploy-nomulus-for-env.sh create mode 100644 jetty/kubernetes/nomulus-deployment.yaml create mode 100644 jetty/kubernetes/nomulus-gateway.yaml create mode 100644 jetty/kubernetes/nomulus-service.yaml diff --git a/jetty/build.gradle b/jetty/build.gradle index 4d57325a4..f1a1edc55 100644 --- a/jetty/build.gradle +++ b/jetty/build.gradle @@ -49,6 +49,15 @@ tasks.register('buildNomulusImage', Exec) { dependsOn(tasks.named('stage')) } +tasks.register('tagNomulusImage', Exec) { + commandLine 'docker', 'tag', 'nomulus', "gcr.io/${rootProject.gcpProject}/nomulus" + dependsOn(tasks.named('buildNomulusImage')) +} + +tasks.register('pushNomulusImage', Exec) { + commandLine 'docker', 'push', "gcr.io/${rootProject.gcpProject}/nomulus" +} + tasks.register('run', JavaExec) { // We do the check when the task actually runs, not when we define it. // This way if one doesn't set the value, one can still run other tasks. @@ -66,4 +75,10 @@ tasks.register('run', JavaExec) { dependsOn(tasks.named('stage')) } +tasks.register('deployNomulus', Exec) { + dependsOn(tasks.named('pushNomulusImage'), tasks.named(":proxy:pushProxyImage")) + configure verifyDeploymentConfig + commandLine './deploy-nomulus-for-env.sh', "${rootProject.environment}" +} + project.build.dependsOn(tasks.named('buildNomulusImage')) diff --git a/jetty/deploy-nomulus-for-env.sh b/jetty/deploy-nomulus-for-env.sh new file mode 100755 index 000000000..7c3c0de65 --- /dev/null +++ b/jetty/deploy-nomulus-for-env.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2024 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script prepares the proxy k8s manifest, pushes it to the clusters, and +# kills all running pods to force k8s to create new pods using the just-pushed +# manifest. + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 alpha|crash|qa" + exit 1 +fi + +environment=${1} +project="domain-registry-"${environment} +current_context=$(kubectl config current-context) +while read line +do + parts=(${line}) + echo "Updating cluster ${parts[0]} in location ${parts[1]}..." + gcloud container clusters get-credentials "${parts[0]}" \ + --project "${project}" --location "${parts[1]}" + sed s/GCP_PROJECT/${project}/g "./kubernetes/nomulus-deployment.yaml" | \ + sed s/ENVIRONMENT/${environment}/g | \ + kubectl apply -f - + kubectl apply -f "./kubernetes/nomulus-service.yaml" + #kubectl apply -f "./kubernetes/nomulus-gateway.yaml" + # Kills all running pods, new pods created will be pulling the new image. + kubectl delete pods --all +done < <(gcloud container clusters list --project ${project} | grep nomulus) +kubectl config use-context "$current_context" diff --git a/jetty/kubernetes/nomulus-deployment.yaml b/jetty/kubernetes/nomulus-deployment.yaml new file mode 100644 index 000000000..910b11d4d --- /dev/null +++ b/jetty/kubernetes/nomulus-deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nomulus + labels: + app: nomulus +spec: + selector: + matchLabels: + app: nomulus + template: + metadata: + labels: + app: nomulus + spec: + containers: + - name: nomulus + image: gcr.io/GCP_PROJECT/nomulus + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: "500m" + args: [ENVIRONMENT] + - name: proxy + image: gcr.io/GCP_PROJECT/proxy + ports: + - containerPort: 30001 + name: whois + - containerPort: 30002 + name: epp + resources: + requests: + cpu: "500m" + args: [--env, ENVIRONMENT, --log, --local] + env: + - name: POD_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE_ID + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONTAINER_NAME + value: proxy +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: nomulus + labels: + app: nomulus +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nomulus + minReplicas: 1 + maxReplicas: 20 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 100 + diff --git a/jetty/kubernetes/nomulus-gateway.yaml b/jetty/kubernetes/nomulus-gateway.yaml new file mode 100644 index 000000000..43de71b79 --- /dev/null +++ b/jetty/kubernetes/nomulus-gateway.yaml @@ -0,0 +1,31 @@ +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: nomulus +spec: + gatewayClassName: gke-l7-global-external-managed-mc + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: HTTPRoute +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1beta1 +metadata: + name: nomulus + labels: + app: nomulus +spec: + parentRefs: + - kind: Gateway + name: nomulus + rules: + - backendRefs: + - group: net.gke.io + kind: ServiceImport + name: nomulus + port: 80 + diff --git a/jetty/kubernetes/nomulus-service.yaml b/jetty/kubernetes/nomulus-service.yaml new file mode 100644 index 000000000..0a0d02687 --- /dev/null +++ b/jetty/kubernetes/nomulus-service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: nomulus +spec: + selector: + app: nomulus + ports: + - port: 80 + targetPort: http + name: http + - port: 43 + targetPort: whois + name: whois + - port: 700 + targetPort: epp + name: epp +#--- +#kind: ServiceExport +#apiVersion: net.gke.io/v1 +#metadata: +# name: nomulus diff --git a/proxy/build.gradle b/proxy/build.gradle index 6c46e533b..0584481fc 100644 --- a/proxy/build.gradle +++ b/proxy/build.gradle @@ -18,19 +18,17 @@ task buildProxyImage(dependsOn: deployJar, type: Exec) { commandLine 'docker', 'build', '-t', 'proxy', '.' } -task deployProxy(dependsOn: buildProxyImage) { +task tagProxyImage(dependsOn: buildProxyImage, type: Exec) { + commandLine 'docker', 'tag', 'proxy', "gcr.io/${rootProject.gcpProject}/proxy" +} + +task pushProxyImage(dependsOn: tagProxyImage, type: Exec) { + commandLine 'docker', 'push', "gcr.io/${rootProject.gcpProject}/proxy" +} + +task deployProxy(dependsOn: pushProxyImage, type: Exec) { configure verifyDeploymentConfig - doLast { - exec { - commandLine 'docker', 'tag', 'proxy', "gcr.io/${rootProject.gcpProject}/proxy" - } - exec { - commandLine 'docker', 'push', "gcr.io/${rootProject.gcpProject}/proxy" - } - exec { - commandLine './deploy-proxy-for-env.sh', "${rootProject.environment}" - } - } + commandLine './deploy-proxy-for-env.sh', "${rootProject.environment}" } project.build.dependsOn buildProxyImage diff --git a/proxy/deploy-proxy-for-env.sh b/proxy/deploy-proxy-for-env.sh index ada39e065..9330268d6 100755 --- a/proxy/deploy-proxy-for-env.sh +++ b/proxy/deploy-proxy-for-env.sh @@ -18,8 +18,8 @@ # manifest. if [[ $# -ne 1 ]]; then - echo "Usage: $0 alpha|crash" - exit 1 + echo "Usage: $0 alpha|crash|qa" + exit 1 fi environment=${1} diff --git a/proxy/src/main/java/google/registry/proxy/EppProtocolModule.java b/proxy/src/main/java/google/registry/proxy/EppProtocolModule.java index 833548114..3982eac2c 100644 --- a/proxy/src/main/java/google/registry/proxy/EppProtocolModule.java +++ b/proxy/src/main/java/google/registry/proxy/EppProtocolModule.java @@ -70,12 +70,14 @@ public final class EppProtocolModule { ProxyConfig config, @EppProtocol int eppPort, @EppProtocol ImmutableList> handlerProviders, - @HttpsRelayProtocol BackendProtocol.Builder backendProtocolBuilder) { + @HttpsRelayProtocol BackendProtocol.Builder backendProtocolBuilder, + @HttpsRelayProtocol boolean localRelay) { return Protocol.frontendBuilder() .name(PROTOCOL_NAME) .port(eppPort) .handlerProviders(handlerProviders) - .relayProtocol(backendProtocolBuilder.host(config.epp.relayHost).build()) + .relayProtocol( + backendProtocolBuilder.host(localRelay ? "localhost" : config.epp.relayHost).build()) .build(); } @@ -114,7 +116,7 @@ public final class EppProtocolModule { config.epp.headerLengthBytes, // Adjustment applied to the header field value in order to obtain message length. -config.epp.headerLengthBytes, - // Initial bytes to strip (i.e. strip the length header). + // Initial bytes to strip (i.e., strip the length header). config.epp.headerLengthBytes); } @@ -150,9 +152,14 @@ public final class EppProtocolModule { @Named("idToken") Supplier idTokenSupplier, @Named("hello") byte[] helloBytes, FrontendMetrics metrics, - ProxyConfig config) { + ProxyConfig config, + @HttpsRelayProtocol boolean localRelay) { return new EppServiceHandler( - config.epp.relayHost, config.epp.relayPath, idTokenSupplier, helloBytes, metrics); + localRelay ? "localhost" : config.epp.relayHost, + config.epp.relayPath, + idTokenSupplier, + helloBytes, + metrics); } @Singleton diff --git a/proxy/src/main/java/google/registry/proxy/HttpsRelayProtocolModule.java b/proxy/src/main/java/google/registry/proxy/HttpsRelayProtocolModule.java index f625e2f89..109268112 100644 --- a/proxy/src/main/java/google/registry/proxy/HttpsRelayProtocolModule.java +++ b/proxy/src/main/java/google/registry/proxy/HttpsRelayProtocolModule.java @@ -35,11 +35,19 @@ import javax.inject.Provider; import javax.inject.Qualifier; /** - * Module that provides a {@link BackendProtocol.Builder} for HTTPS protocol. + * Module that provides a {@link BackendProtocol.Builder} for HTTP(S) protocol. * *

Only a builder is provided because the client protocol itself depends on the remote host * address, which is provided in the server protocol module that relays to this client protocol - * module, e. g. {@link WhoisProtocolModule}. + * module, e.g., {@link WhoisProtocolModule}. + * + *

The protocol can be configured without TLS. In this case, the remote host has to be + * "localhost". Plan HTTP is only expected to be used when communication with Nomulus is via local + * loopback (for security reasons), as is the case when both the proxy and Nomulus container live in + * the same Kubernetes pod. + * + * @see The Kubernetes network + * model */ @Module public class HttpsRelayProtocolModule { @@ -54,10 +62,12 @@ public class HttpsRelayProtocolModule { @HttpsRelayProtocol static BackendProtocol.Builder provideProtocolBuilder( ProxyConfig config, + @HttpsRelayProtocol boolean localRelay, @HttpsRelayProtocol ImmutableList> handlerProviders) { return Protocol.backendBuilder() .name(PROTOCOL_NAME) - .port(config.httpsRelay.port) + .isLocal(localRelay) + .port(localRelay ? config.httpsRelay.localPort : config.httpsRelay.port) .handlerProviders(handlerProviders); } @@ -74,6 +84,7 @@ public class HttpsRelayProtocolModule { @Provides @HttpsRelayProtocol static ImmutableList> provideHandlerProviders( + @HttpsRelayProtocol boolean localRelay, @HttpsRelayProtocol Provider> sslClientInitializerProvider, Provider httpClientCodecProvider, @@ -81,13 +92,17 @@ public class HttpsRelayProtocolModule { Provider backendMetricsHandlerProvider, Provider loggingHandlerProvider, Provider relayHandlerProvider) { - return ImmutableList.of( - sslClientInitializerProvider, - httpClientCodecProvider, - httpObjectAggregatorProvider, - backendMetricsHandlerProvider, - loggingHandlerProvider, - relayHandlerProvider); + ImmutableList.Builder> builder = + new ImmutableList.Builder<>(); + if (!localRelay) { + builder.add(sslClientInitializerProvider); + } + builder.add(httpClientCodecProvider); + builder.add(httpObjectAggregatorProvider); + builder.add(backendMetricsHandlerProvider); + builder.add(loggingHandlerProvider); + builder.add(relayHandlerProvider); + return builder.build(); } @Provides diff --git a/proxy/src/main/java/google/registry/proxy/Protocol.java b/proxy/src/main/java/google/registry/proxy/Protocol.java index 2f00a20d2..1770a0e08 100644 --- a/proxy/src/main/java/google/registry/proxy/Protocol.java +++ b/proxy/src/main/java/google/registry/proxy/Protocol.java @@ -47,8 +47,9 @@ public interface Protocol { return new AutoValue_Protocol_FrontendProtocol.Builder().hasBackend(true); } + /** A builder for {@link FrontendProtocol}, by default it connects to a remote host. */ static BackendProtocol.Builder backendBuilder() { - return new AutoValue_Protocol_BackendProtocol.Builder(); + return new AutoValue_Protocol_BackendProtocol.Builder().isLocal(false); } /** @@ -121,10 +122,26 @@ public interface Protocol { /** The hostname that the proxy connects to. */ public abstract String host(); + /** Whether the protocol is expected to connect to localhost. */ + public abstract boolean isLocal(); + /** Builder of {@link BackendProtocol}. */ @AutoValue.Builder public abstract static class Builder extends Protocol.Builder { public abstract Builder host(String value); + + public abstract Builder isLocal(boolean value); + + abstract BackendProtocol autoBuild(); + + @Override + public BackendProtocol build() { + BackendProtocol protocol = autoBuild(); + Preconditions.checkState( + !protocol.isLocal() || protocol.host().equals("localhost"), + "Local backend protocol must connect to localhost"); + return autoBuild(); + } } } } diff --git a/proxy/src/main/java/google/registry/proxy/ProxyConfig.java b/proxy/src/main/java/google/registry/proxy/ProxyConfig.java index c9c41818d..0fd492356 100644 --- a/proxy/src/main/java/google/registry/proxy/ProxyConfig.java +++ b/proxy/src/main/java/google/registry/proxy/ProxyConfig.java @@ -103,6 +103,7 @@ public class ProxyConfig { /** Configuration options that apply to HTTPS relay protocol. */ public static class HttpsRelay { public int port; + public int localPort; public int maxMessageLengthBytes; } diff --git a/proxy/src/main/java/google/registry/proxy/ProxyModule.java b/proxy/src/main/java/google/registry/proxy/ProxyModule.java index 623d622db..0bf8dd6fd 100644 --- a/proxy/src/main/java/google/registry/proxy/ProxyModule.java +++ b/proxy/src/main/java/google/registry/proxy/ProxyModule.java @@ -37,6 +37,7 @@ import google.registry.networking.module.CertificateSupplierModule; import google.registry.networking.module.CertificateSupplierModule.Mode; import google.registry.proxy.EppProtocolModule.EppProtocol; import google.registry.proxy.HealthCheckProtocolModule.HealthCheckProtocol; +import google.registry.proxy.HttpsRelayProtocolModule.HttpsRelayProtocol; import google.registry.proxy.Protocol.FrontendProtocol; import google.registry.proxy.ProxyConfig.Environment; import google.registry.proxy.WebWhoisProtocolsModule.HttpWhoisProtocol; @@ -91,6 +92,13 @@ public class ProxyModule { @Parameter(names = "--https_whois", description = "Port for HTTPS WHOIS") private Integer httpsWhoisPort; + @Parameter( + names = "--local", + description = + "Whether EPP/WHOIS traffic should be forwarded to localhost using HTTP on port defined in" + + " httpsRelay.localPort") + private boolean local = false; + @Parameter(names = "--env", description = "Environment to run the proxy in") private Environment env = Environment.LOCAL; @@ -168,6 +176,13 @@ public class ProxyModule { return config.oauthClientId; } + @Provides + @HttpsRelayProtocol + @Singleton + boolean provideIsLocal() { + return local; + } + @Provides @WhoisProtocol int provideWhoisPort(ProxyConfig config) { @@ -204,7 +219,7 @@ public class ProxyModule { } /** - * Provides shared logging handler. + * Provides a shared logging handler. * *

Note that this handler always records logs at {@code LogLevel.DEBUG}, it is up to the JUL * logger that it contains to decide if logs at this level should actually be captured. The log diff --git a/proxy/src/main/java/google/registry/proxy/WhoisProtocolModule.java b/proxy/src/main/java/google/registry/proxy/WhoisProtocolModule.java index 108d1ef49..dcf79902a 100644 --- a/proxy/src/main/java/google/registry/proxy/WhoisProtocolModule.java +++ b/proxy/src/main/java/google/registry/proxy/WhoisProtocolModule.java @@ -61,12 +61,14 @@ public final class WhoisProtocolModule { ProxyConfig config, @WhoisProtocol int whoisPort, @WhoisProtocol ImmutableList> handlerProviders, - @HttpsRelayProtocol BackendProtocol.Builder backendProtocolBuilder) { + @HttpsRelayProtocol BackendProtocol.Builder backendProtocolBuilder, + @HttpsRelayProtocol boolean localRelay) { return Protocol.frontendBuilder() .name(PROTOCOL_NAME) .port(whoisPort) .handlerProviders(handlerProviders) - .relayProtocol(backendProtocolBuilder.host(config.whois.relayHost).build()) + .relayProtocol( + backendProtocolBuilder.host(localRelay ? "localhost" : config.whois.relayHost).build()) .build(); } @@ -94,9 +96,13 @@ public final class WhoisProtocolModule { static WhoisServiceHandler provideWhoisServiceHandler( ProxyConfig config, @Named("idToken") Supplier idTokenSupplier, - FrontendMetrics metrics) { + FrontendMetrics metrics, + @HttpsRelayProtocol boolean localRelay) { return new WhoisServiceHandler( - config.whois.relayHost, config.whois.relayPath, idTokenSupplier, metrics); + localRelay ? "localhost" : config.whois.relayHost, + config.whois.relayPath, + idTokenSupplier, + metrics); } @Provides diff --git a/proxy/src/main/java/google/registry/proxy/config/default-config.yaml b/proxy/src/main/java/google/registry/proxy/config/default-config.yaml index 3ded21bfd..ed6fd0eab 100644 --- a/proxy/src/main/java/google/registry/proxy/config/default-config.yaml +++ b/proxy/src/main/java/google/registry/proxy/config/default-config.yaml @@ -177,7 +177,7 @@ healthCheck: httpsRelay: port: 443 - + localPort: 8080 # Maximum size of an HTTP message in bytes. maxMessageLengthBytes: 524288 diff --git a/proxy/src/test/java/google/registry/proxy/ProtocolModuleTest.java b/proxy/src/test/java/google/registry/proxy/ProtocolModuleTest.java index de79804b2..3a6c45b5f 100644 --- a/proxy/src/test/java/google/registry/proxy/ProtocolModuleTest.java +++ b/proxy/src/test/java/google/registry/proxy/ProtocolModuleTest.java @@ -67,7 +67,7 @@ import org.junit.jupiter.api.BeforeEach; /** * Base class for end-to-end tests of a {@link Protocol}. * - *

The end-to-end tests ensures that the business logic that a {@link Protocol} defines are + *

The end-to-end tests ensure that the business logic that a {@link Protocol} defines are * correctly performed by various handlers attached to its pipeline. Non-business essential handlers * should be excluded. * @@ -91,7 +91,7 @@ public abstract class ProtocolModuleTest { // The PROXY protocol is only used when the proxy is behind the GCP load balancer. It is // not part of any business logic. ProxyProtocolHandler.class, - // SSL is part of the business logic for some protocol (EPP for example), but its + // SSL is part of the business logic for some protocol (EPP, for example), but its // impact is isolated. Including it makes tests much more complicated. It should be tested // separately in its own unit tests. SslClientInitializer.class, @@ -152,7 +152,7 @@ public abstract class ProtocolModuleTest { void initializeChannel(Consumer initializer) { channel = new EmbeddedChannel( - new ChannelInitializer() { + new ChannelInitializer<>() { @Override protected void initChannel(Channel ch) { initializer.accept(ch); @@ -218,8 +218,8 @@ public abstract class ProtocolModuleTest { * *

Most of the binding provided in this module should be either a fake, or a {@link * ChannelHandler} that is excluded, and annotated with {@code @Singleton}. This module acts as a - * replacement for {@link ProxyModule} used in production component. Providing a handler that is - * part of the business logic of a {@link Protocol} from this module is a sign that the binding + * replacement for {@link ProxyModule} used in the production component. Providing a handler that + * is part of the business logic of a {@link Protocol} from this module is a sign that the binding * should be provided in the respective {@code ProtocolModule} instead. */ @Module @@ -306,12 +306,19 @@ public abstract class ProtocolModuleTest { } // This method is only here to satisfy Dagger binding, but is never used. In test environment, - // it is the self-signed certificate and its key that end up being used. + // it is the self-signed certificate and its key that ends up being used. @Singleton @Provides @Named("pemBytes") static byte[] providePemBytes() { return new byte[0]; } + + @Singleton + @Provides + @HttpsRelayProtocol + static boolean provideLocalRelay() { + return false; + } } }