From 1096f201cd9e9f740e1b28edc7ef998a700418a9 Mon Sep 17 00:00:00 2001 From: Pavlo Tkach <3469726+ptkach@users.noreply.github.com> Date: Fri, 4 Apr 2025 17:33:43 -0400 Subject: [PATCH] Add GKE readiness probe (#2735) --- .../registry/module/ReadinessProbeAction.java | 96 +++++++++++++++++++ .../registry/module/RequestComponent.java | 9 ++ .../frontend/FrontendRequestComponent.java | 6 ++ .../module/pubapi/PubApiRequestComponent.java | 6 +- .../registry/request/RequestHandler.java | 7 +- .../module/frontend/frontend_routing.txt | 2 + .../registry/module/pubapi/pubapi_routing.txt | 1 + .../google/registry/module/routing.txt | 3 + jetty/kubernetes/nomulus-console.yaml | 9 ++ jetty/kubernetes/nomulus-pubapi.yaml | 27 ++++-- 10 files changed, 153 insertions(+), 13 deletions(-) create mode 100644 core/src/main/java/google/registry/module/ReadinessProbeAction.java diff --git a/core/src/main/java/google/registry/module/ReadinessProbeAction.java b/core/src/main/java/google/registry/module/ReadinessProbeAction.java new file mode 100644 index 000000000..39984675a --- /dev/null +++ b/core/src/main/java/google/registry/module/ReadinessProbeAction.java @@ -0,0 +1,96 @@ +// Copyright 2025 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.module; + +import static google.registry.persistence.transaction.TransactionManagerFactory.tm; +import static jakarta.servlet.http.HttpServletResponse.SC_OK; + +import com.google.common.flogger.FluentLogger; +import google.registry.request.Action; +import google.registry.request.Action.GaeService; +import google.registry.request.Action.GkeService; +import google.registry.request.auth.Auth; +import jakarta.inject.Inject; +import jakarta.servlet.http.HttpServletResponse; + +public class ReadinessProbeAction implements Runnable { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final HttpServletResponse rsp; + + public ReadinessProbeAction(HttpServletResponse rsp) { + this.rsp = rsp; + } + + /** + * Executes the readiness check. + * + *

Performs a simple database query and sets the HTTP response status to OK (200) upon + * successful completion. Throws a runtime exception if the database query fails. + */ + @Override + public final void run() { + logger.atInfo().log("Performing readiness check database query..."); + try { + tm().transact(() -> tm().query("SELECT version()", Void.class)); + rsp.setStatus(SC_OK); + logger.atInfo().log("Readiness check successful."); + } catch (Exception e) { + logger.atWarning().withCause(e).log("Readiness check failed:"); + throw new RuntimeException("Readiness check failed during database query", e); + } + } + + @Action( + service = GaeService.DEFAULT, + gkeService = GkeService.CONSOLE, + path = ReadinessProbeConsoleAction.PATH, + auth = Auth.AUTH_PUBLIC) + public static class ReadinessProbeConsoleAction extends ReadinessProbeAction { + public static final String PATH = "/ready/console"; + + @Inject + public ReadinessProbeConsoleAction(HttpServletResponse rsp) { + super(rsp); + } + } + + @Action( + service = GaeService.PUBAPI, + gkeService = GkeService.PUBAPI, + path = ReadinessProbeActionPubApi.PATH, + auth = Auth.AUTH_PUBLIC) + public static class ReadinessProbeActionPubApi extends ReadinessProbeAction { + public static final String PATH = "/ready/pubapi"; + + @Inject + public ReadinessProbeActionPubApi(HttpServletResponse rsp) { + super(rsp); + } + } + + @Action( + service = GaeService.DEFAULT, + gkeService = GkeService.FRONTEND, + path = ReadinessProbeActionFrontend.PATH, + auth = Auth.AUTH_PUBLIC) + public static final class ReadinessProbeActionFrontend extends ReadinessProbeAction { + public static final String PATH = "/ready/frontend"; + + @Inject + public ReadinessProbeActionFrontend(HttpServletResponse rsp) { + super(rsp); + } + } +} diff --git a/core/src/main/java/google/registry/module/RequestComponent.java b/core/src/main/java/google/registry/module/RequestComponent.java index 2ccd703e1..5db0bc38d 100644 --- a/core/src/main/java/google/registry/module/RequestComponent.java +++ b/core/src/main/java/google/registry/module/RequestComponent.java @@ -58,6 +58,9 @@ import google.registry.flows.TlsCredentials.EppTlsModule; import google.registry.flows.custom.CustomLogicModule; import google.registry.loadtest.LoadTestAction; import google.registry.loadtest.LoadTestModule; +import google.registry.module.ReadinessProbeAction.ReadinessProbeActionFrontend; +import google.registry.module.ReadinessProbeAction.ReadinessProbeActionPubApi; +import google.registry.module.ReadinessProbeAction.ReadinessProbeConsoleAction; import google.registry.monitoring.whitebox.WhiteboxModule; import google.registry.rdap.RdapAutnumAction; import google.registry.rdap.RdapDomainAction; @@ -256,6 +259,12 @@ interface RequestComponent { PublishSpec11ReportAction publishSpec11ReportAction(); + ReadinessProbeConsoleAction readinessProbeConsoleAction(); + + ReadinessProbeActionPubApi readinessProbeActionPubApi(); + + ReadinessProbeActionFrontend readinessProbeActionFrontend(); + RdapAutnumAction rdapAutnumAction(); RdapDomainAction rdapDomainAction(); diff --git a/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java b/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java index 0728e4624..85824aadc 100644 --- a/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java +++ b/core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java @@ -21,6 +21,8 @@ import google.registry.dns.DnsModule; import google.registry.flows.EppTlsAction; import google.registry.flows.FlowComponent; import google.registry.flows.TlsCredentials.EppTlsModule; +import google.registry.module.ReadinessProbeAction.ReadinessProbeActionFrontend; +import google.registry.module.ReadinessProbeAction.ReadinessProbeConsoleAction; import google.registry.monitoring.whitebox.WhiteboxModule; import google.registry.request.RequestComponentBuilder; import google.registry.request.RequestModule; @@ -82,6 +84,10 @@ public interface FrontendRequestComponent { FlowComponent.Builder flowComponentBuilder(); + ReadinessProbeActionFrontend readinessProbeActionFrontend(); + + ReadinessProbeConsoleAction readinessProbeConsoleAction(); + RegistrarsAction registrarsAction(); SecurityAction securityAction(); diff --git a/core/src/main/java/google/registry/module/pubapi/PubApiRequestComponent.java b/core/src/main/java/google/registry/module/pubapi/PubApiRequestComponent.java index 0a6eef6d6..1141ffa6d 100644 --- a/core/src/main/java/google/registry/module/pubapi/PubApiRequestComponent.java +++ b/core/src/main/java/google/registry/module/pubapi/PubApiRequestComponent.java @@ -20,6 +20,7 @@ import google.registry.dns.DnsModule; import google.registry.flows.CheckApiAction; import google.registry.flows.CheckApiAction.CheckApiModule; import google.registry.flows.TlsCredentials.EppTlsModule; +import google.registry.module.ReadinessProbeAction.ReadinessProbeActionPubApi; import google.registry.monitoring.whitebox.WhiteboxModule; import google.registry.rdap.RdapAutnumAction; import google.registry.rdap.RdapDomainAction; @@ -56,15 +57,16 @@ public interface PubApiRequestComponent { RdapAutnumAction rdapAutnumAction(); RdapDomainAction rdapDomainAction(); RdapDomainSearchAction rdapDomainSearchAction(); - RdapEmptyAction rdapEmptyAction(); - RdapEntityAction rdapEntityAction(); RdapEntitySearchAction rdapEntitySearchAction(); RdapHelpAction rdapHelpAction(); RdapIpAction rdapDefaultAction(); RdapNameserverAction rdapNameserverAction(); RdapNameserverSearchAction rdapNameserverSearchAction(); + + ReadinessProbeActionPubApi readinessProbeActionPubApi(); + WhoisHttpAction whoisHttpAction(); WhoisAction whoisAction(); diff --git a/core/src/main/java/google/registry/request/RequestHandler.java b/core/src/main/java/google/registry/request/RequestHandler.java index 7273fd0e3..bdc1b89d9 100644 --- a/core/src/main/java/google/registry/request/RequestHandler.java +++ b/core/src/main/java/google/registry/request/RequestHandler.java @@ -143,8 +143,11 @@ public class RequestHandler { GkeService service = Action.ServiceGetter.get(route.get().action()); String expectedDomain = RegistryConfig.getServiceUrl(service).getHost(); String actualDomain = req.getServerName(); - // If the hostname is "localhost", it must have come from the sidecar proxy. - if (!Objects.equals("localhost", actualDomain) + // If the request doesn't come from GKE readiness prober + String maybeUserAgent = Optional.ofNullable(req.getHeader("User-Agent")).orElse(""); + if (!maybeUserAgent.startsWith("kube-probe") + // If the hostname is "localhost", it must have come from the sidecar proxy. + && !Objects.equals("localhost", actualDomain) && !Objects.equals(actualDomain, expectedDomain)) { logger.atWarning().log( "Actual domain %s does not match expected domain %s", actualDomain, expectedDomain); diff --git a/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt b/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt index 4470947da..fa7a028f7 100644 --- a/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt +++ b/core/src/test/resources/google/registry/module/frontend/frontend_routing.txt @@ -1,5 +1,6 @@ SERVICE PATH CLASS METHODS OK MIN USER_POLICY FRONTEND /_dr/epp EppTlsAction POST n APP ADMIN +FRONTEND /ready/frontend ReadinessProbeActionFrontend GET n NONE PUBLIC CONSOLE /console-api/bulk-domain ConsoleBulkDomainAction POST n USER PUBLIC CONSOLE /console-api/domain ConsoleDomainGetAction GET n USER PUBLIC CONSOLE /console-api/domain-list ConsoleDomainListAction GET n USER PUBLIC @@ -15,3 +16,4 @@ CONSOLE /console-api/settings/security SecurityAction POST CONSOLE /console-api/settings/whois-fields WhoisRegistrarFieldsAction POST n USER PUBLIC CONSOLE /console-api/userdata ConsoleUserDataAction GET n USER PUBLIC CONSOLE /console-api/users ConsoleUsersAction GET,POST,DELETE,PUT n USER PUBLIC +CONSOLE /ready/console ReadinessProbeConsoleAction GET n NONE PUBLIC diff --git a/core/src/test/resources/google/registry/module/pubapi/pubapi_routing.txt b/core/src/test/resources/google/registry/module/pubapi/pubapi_routing.txt index aa0b8f86d..5269f018b 100644 --- a/core/src/test/resources/google/registry/module/pubapi/pubapi_routing.txt +++ b/core/src/test/resources/google/registry/module/pubapi/pubapi_routing.txt @@ -11,4 +11,5 @@ PUBAPI /rdap/help(*) RdapHelpAction GET,HEAD n NONE PUBLIC PUBAPI /rdap/ip/(*) RdapIpAction GET,HEAD n NONE PUBLIC PUBAPI /rdap/nameserver/(*) RdapNameserverAction GET,HEAD n NONE PUBLIC PUBAPI /rdap/nameservers RdapNameserverSearchAction GET,HEAD n NONE PUBLIC +PUBAPI /ready/pubapi ReadinessProbeActionPubApi GET n NONE PUBLIC PUBAPI /whois/(*) WhoisHttpAction GET n NONE PUBLIC diff --git a/core/src/test/resources/google/registry/module/routing.txt b/core/src/test/resources/google/registry/module/routing.txt index 53c8ab5a7..800fdc4f5 100644 --- a/core/src/test/resources/google/registry/module/routing.txt +++ b/core/src/test/resources/google/registry/module/routing.txt @@ -1,5 +1,6 @@ SERVICE PATH CLASS METHODS OK MIN USER_POLICY FRONTEND /_dr/epp EppTlsAction POST n APP ADMIN +FRONTEND /ready/frontend ReadinessProbeActionFrontend GET n NONE PUBLIC BACKEND /_dr/admin/createGroups CreateGroupsAction POST n APP ADMIN BACKEND /_dr/admin/list/domains ListDomainsAction GET,POST n APP ADMIN BACKEND /_dr/admin/list/hosts ListHostsAction GET,POST n APP ADMIN @@ -66,6 +67,7 @@ PUBAPI /rdap/help(*) RdapHelpAction PUBAPI /rdap/ip/(*) RdapIpAction GET,HEAD n NONE PUBLIC PUBAPI /rdap/nameserver/(*) RdapNameserverAction GET,HEAD n NONE PUBLIC PUBAPI /rdap/nameservers RdapNameserverSearchAction GET,HEAD n NONE PUBLIC +PUBAPI /ready/pubapi ReadinessProbeActionPubApi GET n NONE PUBLIC PUBAPI /whois/(*) WhoisHttpAction GET n NONE PUBLIC CONSOLE /console-api/bulk-domain ConsoleBulkDomainAction POST n USER PUBLIC CONSOLE /console-api/domain ConsoleDomainGetAction GET n USER PUBLIC @@ -82,3 +84,4 @@ CONSOLE /console-api/settings/security SecurityAction CONSOLE /console-api/settings/whois-fields WhoisRegistrarFieldsAction POST n USER PUBLIC CONSOLE /console-api/userdata ConsoleUserDataAction GET n USER PUBLIC CONSOLE /console-api/users ConsoleUsersAction GET,POST,DELETE,PUT n USER PUBLIC +CONSOLE /ready/console ReadinessProbeConsoleAction GET n NONE PUBLIC diff --git a/jetty/kubernetes/nomulus-console.yaml b/jetty/kubernetes/nomulus-console.yaml index db74d8500..1e3eda3eb 100644 --- a/jetty/kubernetes/nomulus-console.yaml +++ b/jetty/kubernetes/nomulus-console.yaml @@ -20,6 +20,15 @@ spec: ports: - containerPort: 8080 name: http + startupProbe: + httpGet: + port: 8080 + path: /ready/console + initialDelaySeconds: 1 + timeoutSeconds: 60 + successThreshold: 1 + failureThreshold: 3 + periodSeconds: 30 resources: requests: # explicit pod-slots 0 is required in order to downgrade node diff --git a/jetty/kubernetes/nomulus-pubapi.yaml b/jetty/kubernetes/nomulus-pubapi.yaml index 421a84acc..dc27b739e 100644 --- a/jetty/kubernetes/nomulus-pubapi.yaml +++ b/jetty/kubernetes/nomulus-pubapi.yaml @@ -20,6 +20,15 @@ spec: ports: - containerPort: 8080 name: http + startupProbe: + httpGet: + port: 8080 + path: /ready/pubapi + initialDelaySeconds: 1 + timeoutSeconds: 60 + successThreshold: 1 + failureThreshold: 3 + periodSeconds: 30 resources: requests: # explicit pod-slots 0 is required in order to downgrade node @@ -62,12 +71,12 @@ spec: minReplicas: 5 maxReplicas: 15 metrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 100 + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 100 --- apiVersion: v1 kind: Service @@ -77,9 +86,9 @@ spec: selector: service: pubapi ports: - - port: 80 - targetPort: http - name: http + - port: 80 + targetPort: http + name: http --- apiVersion: net.gke.io/v1 kind: ServiceExport