Fix ICANN reporting and add rdap-queries field (#2081)

This includes two changes, the second necessary for testing the first.
1. We add the rdap-queries field as mandated by the amendment to the
   registry agreement,
   https://itp.cdn.icann.org/en/files/registry-agreement/proposed-global-amendment-base-gtld-registry-agreement-12-04-2023-en.pdf.
   This is fairly similar to the whois-queries field where we just query
   the logs, but instead of searching for "whois" we search for "rdap".
2. BigQuery doesn't use MAX to refer to the bigger of two fields; MAX
   accepts an array as an argument. In order to do what we want (and to
   have the BigQuery statements succeed), we need to use GREATEST.
   Tested both versions in alpha and production BigQuery instances.

build.gradle

@@ -516,7 +516,6 @@ task javaIncrementalFormatCheck {
       println("Omitting format check: not in a git directory.")
     }
   }
-  dependsOn('console-webapp:checkFormatting')
 }
 
 // Shows how modified lines in Java source files will change after formatting.
@@ -534,7 +533,6 @@ task javaIncrementalFormatApply {
   doLast {
     invokeJavaDiffFormatScript("format")
   }
-  dependsOn('console-webapp:applyFormatting')
 }
 
 task javadoc(type: Javadoc) {
@@ -561,6 +559,7 @@ tasks.build.dependsOn(tasks.javadoc)
 // core Nomulus codebase, and runs all presubmits.
 task coreDev {
   dependsOn 'javaIncrementalFormatApply'
+  dependsOn 'console-webapp:applyFormatting'
   dependsOn 'javadoc'
   dependsOn 'checkDependenciesDotGradle'
   dependsOn 'checkLicense'

console-webapp/build.gradle

@@ -66,3 +66,4 @@ tasks.runConsoleWebappUnitTests.dependsOn(tasks.npmInstallDeps)
 tasks.buildConsoleWebappProd.dependsOn(tasks.npmInstallDeps)
 tasks.applyFormatting.dependsOn(tasks.npmInstallDeps)
 tasks.checkFormatting.dependsOn(tasks.npmInstallDeps)
+tasks.build.dependsOn(tasks.checkFormatting)

core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java

@@ -81,9 +81,6 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
 
   public static final String PATH = "/_dr/task/publishDnsUpdates";
   public static final String LOCK_NAME = "DNS updates";
-  // TODO(b/236726584): Remove App Engine header once CloudTasksUtils is refactored to create HTTP
-  // tasks.
-  public static final String APP_ENGINE_RETRY_HEADER = "X-AppEngine-TaskRetryCount";
   public static final String CLOUD_TASKS_RETRY_HEADER = "X-CloudTasks-TaskRetryCount";
   public static final int RETRIES_BEFORE_PERMANENT_FAILURE = 20;
 
@@ -140,8 +137,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
       @Config("registrySupportEmail") Lazy<InternetAddress> registrySupportEmail,
       @Config("registryCcEmail") Lazy<InternetAddress> registryCcEmail,
       @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
-      @Header(APP_ENGINE_RETRY_HEADER) Optional<Integer> appEngineRetryCount,
-      @Header(CLOUD_TASKS_RETRY_HEADER) Optional<Integer> cloudTasksRetryCount,
+      @Header(CLOUD_TASKS_RETRY_HEADER) int retryCount,
       DnsWriterProxy dnsWriterProxy,
       DnsMetrics dnsMetrics,
       LockHandler lockHandler,
@@ -153,10 +149,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     this.dnsMetrics = dnsMetrics;
     this.timeout = timeout;
     this.sendEmailService = sendEmailService;
-    retryCount =
-        cloudTasksRetryCount.orElse(
-            appEngineRetryCount.orElseThrow(
-                () -> new IllegalStateException("Missing a valid retry count header")));
+    this.retryCount = retryCount;
     this.dnsWriter = dnsWriter;
     this.enqueuedTime = enqueuedTime;
     this.itemsCreateTime = itemsCreateTime;

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -50,6 +50,7 @@ import com.google.common.collect.Maps;
 import com.google.common.collect.Range;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
+import com.google.gson.annotations.Expose;
 import com.google.re2j.Pattern;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
@@ -253,7 +254,7 @@ public class Registrar extends UpdateAutoTimestampEntity implements Buildable, J
   // Authentication.
 
   /** X.509 PEM client certificate(s) used to authenticate registrar to EPP service. */
-  String clientCertificate;
+  @Expose String clientCertificate;
 
   /** Base64 encoded SHA256 hash of {@link #clientCertificate}. */
   String clientCertificateHash;
@@ -263,13 +264,13 @@ public class Registrar extends UpdateAutoTimestampEntity implements Buildable, J
    *
    * <p>This allows registrars to migrate certificates without downtime.
    */
-  String failoverClientCertificate;
+  @Expose String failoverClientCertificate;
 
   /** Base64 encoded SHA256 hash of {@link #failoverClientCertificate}. */
   String failoverClientCertificateHash;
 
   /** An allow list of netmasks (in CIDR notation) which the client is allowed to connect from. */
-  List<CidrAddressBlock> ipAddressAllowList;
+  @Expose List<CidrAddressBlock> ipAddressAllowList;
 
   /** A hashed password for EPP access. The hash is a base64 encoded SHA256 string. */
   String passwordHash;

core/src/main/java/google/registry/module/frontend/FrontendRequestComponent.java

@@ -28,6 +28,7 @@ import google.registry.request.RequestScope;
 import google.registry.ui.server.console.ConsoleDomainGetAction;
 import google.registry.ui.server.console.RegistrarsAction;
 import google.registry.ui.server.console.settings.ContactAction;
+import google.registry.ui.server.console.settings.SecurityAction;
 import google.registry.ui.server.registrar.ConsoleOteSetupAction;
 import google.registry.ui.server.registrar.ConsoleRegistrarCreatorAction;
 import google.registry.ui.server.registrar.ConsoleUiAction;
@@ -70,6 +71,8 @@ interface FrontendRequestComponent {
 
   RegistrarsAction registrarsAction();
 
+  SecurityAction securityAction();
+
   @Subcomponent.Builder
   abstract class Builder implements RequestComponentBuilder<FrontendRequestComponent> {
     @Override public abstract Builder requestModule(RequestModule requestModule);

core/src/main/java/google/registry/request/RequestModule.java

@@ -15,11 +15,10 @@
 package google.registry.request;
 
 import static com.google.common.net.MediaType.JSON_UTF_8;
-import static google.registry.dns.PublishDnsUpdatesAction.APP_ENGINE_RETRY_HEADER;
 import static google.registry.dns.PublishDnsUpdatesAction.CLOUD_TASKS_RETRY_HEADER;
 import static google.registry.model.tld.Tlds.assertTldExists;
 import static google.registry.model.tld.Tlds.assertTldsExist;
-import static google.registry.request.RequestParameters.extractOptionalHeader;
+import static google.registry.request.RequestParameters.extractRequiredHeader;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static google.registry.request.RequestParameters.extractSetOfParameters;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -131,8 +130,8 @@ public final class RequestModule {
   @FullServletPath
   static String provideFullServletPath(HttpServletRequest req) {
     // Include the port only if it differs from the default for the scheme.
-    if ((req.getScheme().equals("http") && (req.getServerPort() == 80))
-        || (req.getScheme().equals("https") && (req.getServerPort() == 443))) {
+    if (("http".equals(req.getScheme()) && (req.getServerPort() == 80))
+        || ("https".equals(req.getScheme()) && (req.getServerPort() == 443))) {
       return String.format("%s://%s%s", req.getScheme(), req.getServerName(), req.getServletPath());
     } else {
       return String.format(
@@ -237,16 +236,10 @@ public final class RequestModule {
     return params.build();
   }
 
-  @Provides
-  @Header(APP_ENGINE_RETRY_HEADER)
-  static Optional<Integer> provideAppEngineRetryCount(HttpServletRequest req) {
-    return extractOptionalHeader(req, APP_ENGINE_RETRY_HEADER).map(Integer::parseInt);
-  }
-
   @Provides
   @Header(CLOUD_TASKS_RETRY_HEADER)
-  static Optional<Integer> provideCloudTasksRetryCount(HttpServletRequest req) {
-    return extractOptionalHeader(req, CLOUD_TASKS_RETRY_HEADER).map(Integer::parseInt);
+  static int provideCloudTasksRetryCount(HttpServletRequest req) {
+    return Integer.parseInt(extractRequiredHeader(req, CLOUD_TASKS_RETRY_HEADER));
   }
 
   @Provides

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.CreateCancellationsForBillingEventsCommand;
+import google.registry.tools.javascrap.RecreateBillingRecurrencesCommand;
 
 /** Container class to create and run remote commands against a server instance. */
 public final class RegistryTool {
@@ -93,6 +94,7 @@ public final class RegistryTool {
           .put("login", LoginCommand.class)
           .put("logout", LogoutCommand.class)
           .put("pending_escrow", PendingEscrowCommand.class)
+          .put("recreate_billing_recurrences", RecreateBillingRecurrencesCommand.class)
           .put("registrar_poc", RegistrarPocCommand.class)
           .put("renew_domain", RenewDomainCommand.class)
           .put("save_sql_credential", SaveSqlCredentialCommand.class)

core/src/main/java/google/registry/tools/javascrap/RecreateBillingRecurrencesCommand.java

@@ -0,0 +1,146 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import google.registry.model.EppResourceUtils;
+import google.registry.model.billing.BillingRecurrence;
+import google.registry.model.common.TimeOfYear;
+import google.registry.model.domain.Domain;
+import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.QueryComposer.Comparator;
+import google.registry.tools.ConfirmingCommand;
+import java.util.List;
+import org.joda.time.DateTime;
+
+/**
+ * Command to recreate closed {@link BillingRecurrence}s for domains.
+ *
+ * <p>This can be used to fix situations where BillingRecurrences were inadvertently closed. The new
+ * recurrences will start at the recurrenceTimeOfYear that has most recently occurred in the past,
+ * so that billing will restart upon the next date that the domain would have normally been billed
+ * for autorenew.
+ */
+@Parameters(
+    separators = " =",
+    commandDescription = "Recreate inadvertently-closed BillingRecurrences.")
+public class RecreateBillingRecurrencesCommand extends ConfirmingCommand {
+
+  @Parameter(
+      description = "Domain name(s) for which we wish to recreate a BillingRecurrence",
+      required = true)
+  private List<String> mainParameters;
+
+  @Override
+  protected String prompt() throws Exception {
+    checkArgument(!mainParameters.isEmpty(), "Must provide at least one domain name");
+    return tm().transact(
+            () -> {
+              ImmutableList<BillingRecurrence> existingRecurrences = loadRecurrences();
+              ImmutableList<BillingRecurrence> newRecurrences =
+                  convertRecurrencesWithoutSaving(existingRecurrences);
+              return String.format(
+                  "Create new BillingRecurrence(s)?\n"
+                      + "Existing recurrences:\n"
+                      + "%s\n"
+                      + "New recurrences:\n"
+                      + "%s",
+                  Joiner.on('\n').join(existingRecurrences), Joiner.on('\n').join(newRecurrences));
+            });
+  }
+
+  @Override
+  protected String execute() throws Exception {
+    ImmutableList<BillingRecurrence> newBillingRecurrences = tm().transact(this::internalExecute);
+    return "Created new recurrence(s): " + newBillingRecurrences;
+  }
+
+  private ImmutableList<BillingRecurrence> internalExecute() {
+    ImmutableList<BillingRecurrence> newRecurrences =
+        convertRecurrencesWithoutSaving(loadRecurrences());
+    newRecurrences.forEach(
+        recurrence -> {
+          tm().put(recurrence);
+          Domain domain = tm().loadByKey(VKey.create(Domain.class, recurrence.getDomainRepoId()));
+          tm().put(domain.asBuilder().setAutorenewBillingEvent(recurrence.createVKey()).build());
+        });
+    return newRecurrences;
+  }
+
+  private ImmutableList<BillingRecurrence> convertRecurrencesWithoutSaving(
+      ImmutableList<BillingRecurrence> existingRecurrences) {
+    return existingRecurrences.stream()
+        .map(
+            existingRecurrence -> {
+              TimeOfYear timeOfYear = existingRecurrence.getRecurrenceTimeOfYear();
+              DateTime newLastExpansion =
+                  timeOfYear.getLastInstanceBeforeOrAt(tm().getTransactionTime());
+              // event time should be the next date of billing in the future
+              DateTime eventTime = timeOfYear.getNextInstanceAtOrAfter(tm().getTransactionTime());
+              return existingRecurrence
+                  .asBuilder()
+                  .setRecurrenceEndTime(END_OF_TIME)
+                  .setRecurrenceLastExpansion(newLastExpansion)
+                  .setEventTime(eventTime)
+                  .setId(0)
+                  .build();
+            })
+        .collect(toImmutableList());
+  }
+
+  private ImmutableList<BillingRecurrence> loadRecurrences() {
+    ImmutableList.Builder<BillingRecurrence> result = new ImmutableList.Builder<>();
+    DateTime now = tm().getTransactionTime();
+    for (String domainName : mainParameters) {
+      Domain domain =
+          EppResourceUtils.loadByForeignKey(Domain.class, domainName, now)
+              .orElseThrow(
+                  () ->
+                      new IllegalArgumentException(
+                          String.format(
+                              "Domain %s does not exist or has been deleted", domainName)));
+      BillingRecurrence billingRecurrence = tm().loadByKey(domain.getAutorenewBillingEvent());
+      checkArgument(
+          !billingRecurrence.getRecurrenceEndTime().equals(END_OF_TIME),
+          "Domain %s's recurrence's end date is already END_OF_TIME",
+          domainName);
+      // Double-check that there are no non-linked BillingRecurrences that have an END_OF_TIME end.
+      // If this is the case, something has been mis-linked.
+      ImmutableList<BillingRecurrence> allRecurrencesForDomain =
+          tm().createQueryComposer(BillingRecurrence.class)
+              .where("domainRepoId", Comparator.EQ, domain.getRepoId())
+              .list();
+      allRecurrencesForDomain.forEach(
+          recurrence ->
+              checkArgument(
+                  !recurrence.getRecurrenceEndTime().equals(END_OF_TIME),
+                  "There exists a recurrence with id %s for domain %s with an end date of"
+                      + " END_OF_TIME",
+                  recurrence.getId(),
+                  domainName));
+
+      result.add(billingRecurrence);
+    }
+    return result.build();
+  }
+}

core/src/main/java/google/registry/ui/server/console/settings/SecurityAction.java

@@ -0,0 +1,171 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.ui.server.console.settings;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.request.Action.Method.GET;
+import static google.registry.request.Action.Method.POST;
+
+import avro.shaded.com.google.common.collect.ImmutableList;
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.gson.Gson;
+import google.registry.flows.certs.CertificateChecker;
+import google.registry.flows.certs.CertificateChecker.InsecureCertificateException;
+import google.registry.model.console.ConsolePermission;
+import google.registry.model.console.User;
+import google.registry.model.registrar.Registrar;
+import google.registry.request.Action;
+import google.registry.request.Parameter;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.request.auth.AuthResult;
+import google.registry.request.auth.AuthenticatedRegistrarAccessor;
+import google.registry.request.auth.AuthenticatedRegistrarAccessor.RegistrarAccessDeniedException;
+import google.registry.ui.server.registrar.JsonGetAction;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+
+@Action(
+    service = Action.Service.DEFAULT,
+    path = SecurityAction.PATH,
+    method = {GET, POST},
+    auth = Auth.AUTH_PUBLIC_LOGGED_IN)
+public class SecurityAction implements JsonGetAction {
+
+  static final String PATH = "/console-api/settings/security";
+  private final HttpServletRequest req;
+  private final AuthResult authResult;
+  private final Response response;
+  private final Gson gson;
+  private final String registrarId;
+  private AuthenticatedRegistrarAccessor registrarAccessor;
+  private Optional<Registrar> registrar;
+  private CertificateChecker certificateChecker;
+
+  @Inject
+  public SecurityAction(
+      HttpServletRequest req,
+      AuthResult authResult,
+      Response response,
+      Gson gson,
+      CertificateChecker certificateChecker,
+      AuthenticatedRegistrarAccessor registrarAccessor,
+      @Parameter("registrarId") String registrarId,
+      @Parameter("registrar") Optional<Registrar> registrar) {
+    this.req = req;
+    this.authResult = authResult;
+    this.response = response;
+    this.gson = gson;
+    this.registrarId = registrarId;
+    this.registrarAccessor = registrarAccessor;
+    this.registrar = registrar;
+    this.certificateChecker = certificateChecker;
+  }
+
+  @Override
+  public void run() {
+    if (req.getMethod().equals(GET.toString())) {
+      getHandler();
+    } else {
+      postHandler();
+    }
+  }
+
+  private void getHandler() {
+    try {
+      Registrar registrar = registrarAccessor.getRegistrar(registrarId);
+      response.setStatus(HttpStatusCodes.STATUS_CODE_OK);
+      response.setPayload(gson.toJson(registrar));
+    } catch (RegistrarAccessDeniedException e) {
+      response.setStatus(HttpStatusCodes.STATUS_CODE_FORBIDDEN);
+      response.setPayload(e.getMessage());
+    }
+  }
+
+  private void postHandler() {
+    User user = authResult.userAuthInfo().get().consoleUser().get();
+    if (!user.getUserRoles().hasPermission(registrarId, ConsolePermission.EDIT_REGISTRAR_DETAILS)) {
+      response.setStatus(HttpStatusCodes.STATUS_CODE_FORBIDDEN);
+      return;
+    }
+
+    if (!registrar.isPresent()) {
+      response.setStatus(HttpStatusCodes.STATUS_CODE_BAD_REQUEST);
+      response.setPayload(gson.toJson("'registrar' parameter is not present"));
+      return;
+    }
+
+    Registrar savedRegistrar;
+    try {
+      savedRegistrar = registrarAccessor.getRegistrar(registrarId);
+    } catch (RegistrarAccessDeniedException e) {
+      response.setStatus(HttpStatusCodes.STATUS_CODE_FORBIDDEN);
+      response.setPayload(e.getMessage());
+      return;
+    }
+
+    tm().transact(() -> setResponse(savedRegistrar));
+  }
+
+  private void setResponse(Registrar savedRegistrar) {
+    Registrar registrarParameter = registrar.get();
+    Registrar.Builder updatedRegistrar =
+        savedRegistrar
+            .asBuilder()
+            .setIpAddressAllowList(registrarParameter.getIpAddressAllowList());
+
+    boolean hasInvalidCerts =
+        ImmutableList.of(
+                registrarParameter.getClientCertificate(),
+                registrarParameter.getFailoverClientCertificate())
+            .stream()
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .anyMatch(
+                cert -> {
+                  try {
+                    certificateChecker.validateCertificate(cert);
+                    return false;
+                  } catch (InsecureCertificateException e) {
+                    return true;
+                  }
+                });
+
+    if (hasInvalidCerts) {
+      response.setStatus(HttpStatusCodes.STATUS_CODE_BAD_REQUEST);
+      response.setPayload("Insecure Certificate in parameter");
+      return;
+    }
+
+    registrarParameter
+        .getClientCertificate()
+        .ifPresent(
+            newClientCert -> {
+              updatedRegistrar.setClientCertificate(newClientCert, tm().getTransactionTime());
+            });
+
+    registrarParameter
+        .getFailoverClientCertificate()
+        .ifPresent(
+            failoverCert -> {
+              updatedRegistrar.setFailoverClientCertificate(
+                  failoverCert, tm().getTransactionTime());
+            });
+
+    tm().put(updatedRegistrar.build());
+    response.setStatus(HttpStatusCodes.STATUS_CODE_OK);
+  }
+}

core/src/main/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java

@@ -24,6 +24,7 @@ import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarPoc;
 import google.registry.request.OptionalJsonPayload;
 import google.registry.request.Parameter;
@@ -187,4 +188,15 @@ public final class RegistrarConsoleModule {
   static String provideRegistrarId(HttpServletRequest req) {
     return extractRequiredParameter(req, "registrarId");
   }
+
+  @Provides
+  @Parameter("registrar")
+  public static Optional<Registrar> provideRegistrar(
+      Gson gson, @OptionalJsonPayload Optional<JsonObject> payload) {
+    if (payload.isPresent() && payload.get().has("registrar")) {
+      return Optional.of(gson.fromJson(payload.get().get("registrar"), Registrar.class));
+    }
+
+    return Optional.empty();
+  }
 }

core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_activity_report_aggregation.sql

@@ -58,7 +58,8 @@ SELECT
   SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
   SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
   SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
-  SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update
+  SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update,
+  SUM(IF(metricName = 'rdap-queries', count, 0)) AS rdap_queries
   -- Cross join a list of all TLDs against TLD-specific metrics and then
   -- filter so that only metrics with that TLD or a NULL TLD are counted
   -- towards a given TLD.

core/src/main/resources/google/registry/reporting/icann/sql/cloud_sql_transaction_counts.sql

@@ -47,7 +47,16 @@ FROM (
             END AS clientId,
             tld,
             report_field AS field,
-            report_amount AS amount,
+            -- See b/290228682, there are edge cases in which the net_renew would be negative when
+            -- a domain is cancelled by superusers during renew grace period. The correct thing
+            -- to do is attribute the cancellation to the owning registrar, but that would require
+            -- changing the owing registrar of the the corresponding cancellation DomainHistory,
+            -- which has cascading effects that we don't want to deal with. As such we simply
+            -- floor the number here to zero to prevent any negative value from appearing, which
+            -- should have negligible impact as the edge cage happens very rarely, more specifically
+            -- when a cancellation happens during grace period by a registrar other than the the
+            -- owning one. All the numbers here should be positive to pass ICANN validation.
+            GREATEST(report_amount, 0) AS amount,
             reporting_time AS reportingTime
             FROM EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
             ''' SELECT history_type, history_other_registrar_id, history_registrar_id, domain_repo_id, history_revision_id FROM "DomainHistory";''') AS dh

core/src/main/resources/google/registry/reporting/icann/sql/whois_counts.sql

@@ -23,6 +23,7 @@ SELECT
   CASE
     WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries'
     WHEN SUBSTR(requestPath, 0, 7) = '/whois/' THEN 'web-whois-queries'
+    WHEN SUBSTR(requestPath, 0, 6) = '/rdap/' THEN 'rdap-queries'
   END AS metricName,
   COUNT(requestPath) AS count
 FROM

core/src/test/java/google/registry/dns/PublishDnsUpdatesActionTest.java

@@ -64,7 +64,6 @@ import google.registry.testing.FakeResponse;
 import google.registry.testing.Lazies;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
-import java.util.Optional;
 import java.util.Set;
 import javax.mail.internet.InternetAddress;
 import org.joda.time.DateTime;
@@ -161,8 +160,7 @@ public class PublishDnsUpdatesActionTest {
         registrySupportEmail,
         registryCcEmail,
         outgoingRegistry,
-        Optional.ofNullable(retryCount),
-        Optional.empty(),
+        retryCount,
         new DnsWriterProxy(ImmutableMap.of("correctWriter", dnsWriter)),
         dnsMetrics,
         lockHandler,
@@ -449,20 +447,6 @@ public class PublishDnsUpdatesActionTest {
         .isEqualTo("registry-cc@test.com");
   }
 
-  @Test
-  void testTaskMissingRetryHeaders_throwsException() {
-    IllegalStateException thrown =
-        assertThrows(
-            IllegalStateException.class,
-            () ->
-                createAction(
-                    "xn--q9jyb4c",
-                    ImmutableSet.of(),
-                    ImmutableSet.of("ns1.example.xn--q9jyb4c"),
-                    null));
-    assertThat(thrown).hasMessageThat().contains("Missing a valid retry count header");
-  }
-
   @Test
   void testHostAndDomain_published() {
     ImmutableSet<String> hosts =

core/src/test/java/google/registry/reporting/icann/ActivityReportingQueryBuilderTest.java

@@ -45,7 +45,7 @@ class ActivityReportingQueryBuilderTest {
   }
 
   @Test
-  void testIntermediaryQueryMatch_cloud_sql() {
+  void testIntermediaryQueryMatch_cloudSql() {
     ImmutableList<String> expectedQueryNames =
         ImmutableList.of(
             ActivityReportingQueryBuilder.REGISTRAR_OPERATING_STATUS,

core/src/test/java/google/registry/tools/javascrap/RecreateBillingRecurrencesCommandTest.java

@@ -0,0 +1,143 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.loadAllOf;
+import static google.registry.testing.DatabaseHelper.loadByEntity;
+import static google.registry.testing.DatabaseHelper.loadByKey;
+import static google.registry.testing.DatabaseHelper.persistActiveContact;
+import static google.registry.testing.DatabaseHelper.persistDomainWithDependentResources;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
+import static org.junit.Assert.assertThrows;
+
+import google.registry.model.ImmutableObjectSubject;
+import google.registry.model.billing.BillingRecurrence;
+import google.registry.model.contact.Contact;
+import google.registry.model.domain.Domain;
+import google.registry.tools.CommandTestCase;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Tests for {@link RecreateBillingRecurrencesCommand}. */
+public class RecreateBillingRecurrencesCommandTest
+    extends CommandTestCase<RecreateBillingRecurrencesCommand> {
+
+  private Contact contact;
+  private Domain domain;
+  private BillingRecurrence oldRecurrence;
+
+  @BeforeEach
+  void beforeEach() {
+    fakeClock.setTo(DateTime.parse("2022-09-05TZ"));
+    createTld("tld");
+    contact = persistActiveContact("contact1234");
+    domain =
+        persistDomainWithDependentResources(
+            "example",
+            "tld",
+            contact,
+            fakeClock.nowUtc(),
+            fakeClock.nowUtc(),
+            fakeClock.nowUtc().plusYears(1));
+    oldRecurrence = loadByKey(domain.getAutorenewBillingEvent());
+    oldRecurrence =
+        persistResource(
+            oldRecurrence.asBuilder().setRecurrenceEndTime(fakeClock.nowUtc().plusDays(1)).build());
+    fakeClock.setTo(DateTime.parse("2023-07-11TZ"));
+  }
+
+  @Test
+  void testSuccess_simpleRecreation() throws Exception {
+    runCommandForced("example.tld");
+    // The domain should now be linked to the new recurrence
+    BillingRecurrence newRecurrence = loadByKey(loadByEntity(domain).getAutorenewBillingEvent());
+    assertThat(newRecurrence.getId()).isNotEqualTo(oldRecurrence.getId());
+    // The new recurrence should not end and have last year's event time and last expansion.
+    assertThat(newRecurrence.getRecurrenceEndTime()).isEqualTo(END_OF_TIME);
+    assertThat(newRecurrence.getEventTime()).isEqualTo(DateTime.parse("2023-09-05TZ"));
+    assertThat(newRecurrence.getRecurrenceLastExpansion())
+        .isEqualTo(DateTime.parse("2022-09-05TZ"));
+    assertThat(loadAllOf(BillingRecurrence.class)).containsExactly(oldRecurrence, newRecurrence);
+  }
+
+  @Test
+  void testSuccess_multipleDomains() throws Exception {
+    Domain otherDomain =
+        persistDomainWithDependentResources(
+            "other",
+            "tld",
+            contact,
+            DateTime.parse("2022-09-07TZ"),
+            DateTime.parse("2022-09-07TZ"),
+            DateTime.parse("2023-09-07TZ"));
+    BillingRecurrence otherRecurrence = loadByKey(otherDomain.getAutorenewBillingEvent());
+    otherRecurrence =
+        persistResource(
+            otherRecurrence
+                .asBuilder()
+                .setRecurrenceEndTime(DateTime.parse("2022-09-08TZ"))
+                .build());
+    runCommandForced("example.tld", "other.tld");
+    // Both domains should have new recurrences with END_OF_TIME expirations
+    BillingRecurrence otherNewRecurrence =
+        loadByKey(loadByEntity(otherDomain).getAutorenewBillingEvent());
+    assertThat(otherNewRecurrence.getId()).isNotEqualTo(otherRecurrence.getId());
+    assertThat(otherNewRecurrence.getRecurrenceEndTime()).isEqualTo(END_OF_TIME);
+    assertThat(otherNewRecurrence.getEventTime()).isEqualTo(DateTime.parse("2023-09-07TZ"));
+    assertThat(otherNewRecurrence.getRecurrenceLastExpansion())
+        .isEqualTo(DateTime.parse("2022-09-07TZ"));
+    assertThat(loadAllOf(BillingRecurrence.class))
+        .comparingElementsUsing(ImmutableObjectSubject.immutableObjectCorrespondence("id"))
+        .containsExactly(
+            oldRecurrence,
+            oldRecurrence
+                .asBuilder()
+                .setRecurrenceEndTime(END_OF_TIME)
+                .setEventTime(DateTime.parse("2023-09-05TZ"))
+                .setRecurrenceLastExpansion(DateTime.parse("2022-09-05TZ"))
+                .build(),
+            otherRecurrence,
+            otherNewRecurrence);
+  }
+
+  @Test
+  void testFailure_badDomain() {
+    assertThat(assertThrows(IllegalArgumentException.class, () -> runCommandForced("foo.tld")))
+        .hasMessageThat()
+        .isEqualTo("Domain foo.tld does not exist or has been deleted");
+  }
+
+  @Test
+  void testFailure_alreadyEndOfTime() {
+    persistResource(oldRecurrence.asBuilder().setRecurrenceEndTime(END_OF_TIME).build());
+    assertThat(assertThrows(IllegalArgumentException.class, () -> runCommandForced("example.tld")))
+        .hasMessageThat()
+        .isEqualTo("Domain example.tld's recurrence's end date is already END_OF_TIME");
+  }
+
+  @Test
+  void testFailure_nonLinkedRecurrenceIsEndOfTime() {
+    persistResource(oldRecurrence.asBuilder().setRecurrenceEndTime(END_OF_TIME).setId(0).build());
+    assertThat(assertThrows(IllegalArgumentException.class, () -> runCommandForced("example.tld")))
+        .hasMessageThat()
+        .isEqualTo(
+            "There exists a recurrence with id 9 for domain example.tld with an end date of"
+                + " END_OF_TIME");
+  }
+}

core/src/test/java/google/registry/ui/server/console/settings/SecurityActionTest.java

@@ -0,0 +1,183 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.ui.server.console.settings;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.CertificateSamples.SAMPLE_CERT;
+import static google.registry.testing.CertificateSamples.SAMPLE_CERT2;
+import static google.registry.testing.DatabaseHelper.loadRegistrar;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.SqlHelper.saveRegistrar;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.api.client.http.HttpStatusCodes;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.net.InetAddresses;
+import com.google.gson.Gson;
+import google.registry.flows.certs.CertificateChecker;
+import google.registry.model.console.GlobalRole;
+import google.registry.model.console.User;
+import google.registry.model.console.UserRoles;
+import google.registry.model.registrar.Registrar;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.request.Action;
+import google.registry.request.RequestModule;
+import google.registry.request.auth.AuthResult;
+import google.registry.request.auth.AuthSettings.AuthLevel;
+import google.registry.request.auth.AuthenticatedRegistrarAccessor;
+import google.registry.request.auth.UserAuthInfo;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeResponse;
+import google.registry.ui.server.registrar.RegistrarConsoleModule;
+import google.registry.util.CidrAddressBlock;
+import google.registry.util.UtilsModule;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Optional;
+import javax.servlet.http.HttpServletRequest;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Tests for {@link google.registry.ui.server.console.settings.SecurityAction}. */
+class SecurityActionTest {
+
+  private static String jsonRegistrar1 =
+      String.format(
+          "{\"registrarId\": \"registrarId\", \"clientCertificate\": \"%s\","
+              + " \"ipAddressAllowList\": [\"192.168.1.1/32\"]}",
+          SAMPLE_CERT2);
+  private static final Gson GSON = UtilsModule.provideGson();
+  private final HttpServletRequest request = mock(HttpServletRequest.class);
+  private final FakeClock clock = new FakeClock();
+  private Registrar testRegistrar;
+  private FakeResponse response = new FakeResponse();
+
+  private AuthenticatedRegistrarAccessor registrarAccessor =
+      AuthenticatedRegistrarAccessor.createForTesting(
+          ImmutableSetMultimap.of("registrarId", AuthenticatedRegistrarAccessor.Role.ADMIN));
+
+  private CertificateChecker certificateChecker =
+      new CertificateChecker(
+          ImmutableSortedMap.of(START_OF_TIME, 20825, DateTime.parse("2020-09-01T00:00:00Z"), 398),
+          30,
+          15,
+          2048,
+          ImmutableSet.of("secp256r1", "secp384r1"),
+          clock);
+
+  @RegisterExtension
+  final JpaTestExtensions.JpaIntegrationTestExtension jpa =
+      new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension();
+
+  @BeforeEach
+  void beforeEach() {
+    testRegistrar = saveRegistrar("registrarId");
+  }
+
+  @Test
+  void testSuccess_getRegistrarInfo() throws IOException {
+    persistResource(
+        testRegistrar
+            .asBuilder()
+            .setClientCertificate(SAMPLE_CERT, clock.nowUtc())
+            .setIpAddressAllowList(
+                ImmutableSet.of(
+                    CidrAddressBlock.create(InetAddresses.forString("192.168.1.1"), 32),
+                    CidrAddressBlock.create(InetAddresses.forString("2001:db8::1"), 128)))
+            .build());
+    SecurityAction action =
+        createAction(
+            Action.Method.GET,
+            AuthResult.create(
+                AuthLevel.USER,
+                UserAuthInfo.create(
+                    createUser(new UserRoles.Builder().setGlobalRole(GlobalRole.FTE).build()))),
+            testRegistrar.getRegistrarId());
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(HttpStatusCodes.STATUS_CODE_OK);
+    String payload = response.getPayload().replace("\\n", "").replace("\\u003d", "=");
+    assertThat(payload).contains(SAMPLE_CERT.replace("\n", ""));
+    assertThat(payload).contains("192.168.1.1/32");
+    assertThat(payload).contains("2001:db8:0:0:0:0:0:1/128");
+  }
+
+  @Test
+  void testSuccess_postRegistrarInfo() throws IOException {
+    clock.setTo(DateTime.parse("2020-11-01T00:00:00Z"));
+    SecurityAction action =
+        createAction(
+            Action.Method.POST,
+            AuthResult.create(
+                AuthLevel.USER,
+                UserAuthInfo.create(
+                    createUser(new UserRoles.Builder().setGlobalRole(GlobalRole.FTE).build()))),
+            testRegistrar.getRegistrarId());
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(HttpStatusCodes.STATUS_CODE_OK);
+    Registrar r = loadRegistrar(testRegistrar.getRegistrarId());
+    assertThat(r.getClientCertificateHash().get())
+        .isEqualTo("GNd6ZP8/n91t9UTnpxR8aH7aAW4+CpvufYx9ViGbcMY");
+    assertThat(r.getIpAddressAllowList().get(0).getIp()).isEqualTo("192.168.1.1");
+    assertThat(r.getIpAddressAllowList().get(0).getNetmask()).isEqualTo(32);
+  }
+
+  private User createUser(UserRoles userRoles) {
+    return new User.Builder()
+        .setEmailAddress("email@email.com")
+        .setGaiaId("TestUserId")
+        .setUserRoles(userRoles)
+        .build();
+  }
+
+  private SecurityAction createAction(
+      Action.Method method, AuthResult authResult, String registrarId) throws IOException {
+    when(request.getMethod()).thenReturn(method.toString());
+    if (method.equals(Action.Method.GET)) {
+      return new SecurityAction(
+          request,
+          authResult,
+          response,
+          GSON,
+          certificateChecker,
+          registrarAccessor,
+          registrarId,
+          Optional.empty());
+    } else {
+      doReturn(new BufferedReader(new StringReader("{\"registrar\":" + jsonRegistrar1 + "}")))
+          .when(request)
+          .getReader();
+      Optional<Registrar> maybeRegistrar =
+          RegistrarConsoleModule.provideRegistrar(
+              GSON, RequestModule.provideJsonBody(request, GSON));
+      return new SecurityAction(
+          request,
+          authResult,
+          response,
+          GSON,
+          certificateChecker,
+          registrarAccessor,
+          registrarId,
+          maybeRegistrar);
+    }
+  }
+}

core/src/test/resources/google/registry/module/frontend/frontend_routing.txt

@@ -3,6 +3,7 @@ PATH                           CLASS                         METHODS  OK AUTH_ME
 /console-api/domain            ConsoleDomainGetAction        GET      n  API,LEGACY   USER PUBLIC
 /console-api/registrars        RegistrarsAction              GET      n  API,LEGACY   USER PUBLIC
 /console-api/settings/contacts ContactAction                 GET,POST n  API,LEGACY   USER PUBLIC
+/console-api/settings/security SecurityAction                GET,POST n  API,LEGACY   USER PUBLIC
 /registrar                     ConsoleUiAction               GET      n  API,LEGACY   NONE PUBLIC
 /registrar-create              ConsoleRegistrarCreatorAction POST,GET n  API,LEGACY   NONE PUBLIC
 /registrar-ote-setup           ConsoleOteSetupAction         POST,GET n  API,LEGACY   NONE PUBLIC

core/src/test/resources/google/registry/reporting/icann/activity_report_aggregation_test_cloud_sql.sql

@@ -58,7 +58,8 @@ SELECT
   SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
   SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
   SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
-  SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update
+  SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update,
+  SUM(IF(metricName = 'rdap-queries', count, 0)) AS rdap_queries
   -- Cross join a list of all TLDs against TLD-specific metrics and then
   -- filter so that only metrics with that TLD or a NULL TLD are counted
   -- towards a given TLD.

core/src/test/resources/google/registry/reporting/icann/transaction_counts_test_cloud_sql.sql

@@ -47,7 +47,16 @@ FROM (
             END AS clientId,
             tld,
             report_field AS field,
-            report_amount AS amount,
+            -- See b/290228682, there are edge cases in which the net_renew would be negative when
+            -- a domain is cancelled by superusers during renew grace period. The correct thing
+            -- to do is attribute the cancellation to the owning registrar, but that would require
+            -- changing the owing registrar of the the corresponding cancellation DomainHistory,
+            -- which has cascading effects that we don't want to deal with. As such we simply
+            -- floor the number here to zero to prevent any negative value from appearing, which
+            -- should have negligible impact as the edge cage happens very rarely, more specifically
+            -- when a cancellation happens during grace period by a registrar other than the the
+            -- owning one. All the numbers here should be positive to pass ICANN validation.
+            GREATEST(report_amount, 0) AS amount,
             reporting_time AS reportingTime
             FROM EXTERNAL_QUERY("projects/domain-registry-alpha/locations/us/connections/domain-registry-alpha-sql",
             ''' SELECT history_type, history_other_registrar_id, history_registrar_id, domain_repo_id, history_revision_id FROM "DomainHistory";''') AS dh

core/src/test/resources/google/registry/reporting/icann/whois_counts_test_cloud_sql.sql

@@ -23,6 +23,7 @@ SELECT
   CASE
     WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries'
     WHEN SUBSTR(requestPath, 0, 7) = '/whois/' THEN 'web-whois-queries'
+    WHEN SUBSTR(requestPath, 0, 6) = '/rdap/' THEN 'rdap-queries'
   END AS metricName,
   COUNT(requestPath) AS count
 FROM

util/src/main/java/google/registry/util/CidrAddressBlock.java

@@ -19,6 +19,10 @@ import static com.google.common.base.Preconditions.checkArgument;
 import com.google.common.collect.AbstractSequentialIterator;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.InetAddresses;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonWriter;
+import java.io.IOException;
 import java.io.Serializable;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
@@ -476,4 +480,20 @@ public class CidrAddressBlock implements Iterable<InetAddress>, Serializable {
   public String toString() {
     return getCidrString(ip, netmask);
   }
+
+  public static class CidrAddressBlockAdapter extends TypeAdapter<CidrAddressBlock> {
+    @Override
+    public CidrAddressBlock read(JsonReader reader) throws IOException {
+      String stringValue = reader.nextString();
+      if (stringValue.equals("null")) {
+        return null;
+      }
+      return new CidrAddressBlock(stringValue);
+    }
+
+    @Override
+    public void write(JsonWriter writer, CidrAddressBlock cidrAddressBlock) throws IOException {
+      writer.value(cidrAddressBlock.toString());
+    }
+  }
 }

util/src/main/java/google/registry/util/UtilsModule.java

@@ -19,6 +19,7 @@ import com.google.gson.GsonBuilder;
 import dagger.Binds;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.util.CidrAddressBlock.CidrAddressBlockAdapter;
 import java.security.NoSuchAlgorithmException;
 import java.security.ProviderException;
 import java.security.SecureRandom;
@@ -79,6 +80,7 @@ public abstract class UtilsModule {
   public static Gson provideGson() {
     return new GsonBuilder()
         .registerTypeAdapter(DateTime.class, new DateTimeTypeAdapter())
+        .registerTypeAdapter(CidrAddressBlock.class, new CidrAddressBlockAdapter())
         .excludeFieldsWithoutExposeAnnotation()
         .create();
   }