Use the read-only replica for JPA invoicing (#1494)

* Use the read-only replica for JPA invoicing

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -17,7 +17,7 @@ package google.registry.backup;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.google.common.collect.ImmutableMultimap;
@@ -67,7 +67,8 @@ public final class CommitLogCheckpointAction implements Runnable {
     final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
     logger.atInfo().log(
         "Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
-    tm().transact(
+    ofyTm()
+        .transact(
             () -> {
               DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
               if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {

core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static java.lang.Boolean.FALSE;
 import static java.lang.Boolean.TRUE;
 
@@ -288,7 +288,8 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       }
 
       DeletionResult deletionResult =
-          tm().transactNew(
+          ofyTm()
+              .transactNew(
                   () -> {
                     CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
                     // It is possible that the same manifestKey was run twice, if a shard had to be

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -242,6 +242,9 @@ cloudSql:
   jdbcUrl: jdbc:postgresql://localhost
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
+  # If non-null, we will use this instance for certain read-only actions or
+  # pipelines, e.g. RDE, in order to offload some work from the primary
+  # instance. Expect any write actions on this instance to fail.
   replicaInstanceConnectionName: null
 
 cloudDns:

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -30,6 +30,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.pubapi.PubApiRequestComponent.PubApiRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.persistence.PersistenceModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
@@ -56,6 +57,7 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      PersistenceModule.class,
       PubApiRequestComponentModule.class,
       SecretManagerModule.class,
       ServerTridProviderModule.class,

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -277,6 +277,8 @@ public abstract class PersistenceModule {
     setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides);
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
+    overrides.put(
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -291,6 +293,8 @@ public abstract class PersistenceModule {
     HashMap<String, String> overrides = Maps.newHashMap(beamCloudSqlConfigs);
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
+    overrides.put(
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -141,14 +141,15 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     // Postgresql-specific: 'set transaction' command must be called inside a transaction
     assertInTransaction();
 
-    EntityManager entityManager = getEntityManager();
+    ReadOnlyCheckingEntityManager entityManager =
+        (ReadOnlyCheckingEntityManager) getEntityManager();
     // Isolation is hardcoded to REPEATABLE READ, as specified by parent's Javadoc.
     entityManager
         .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
-        .executeUpdate();
+        .executeUpdateIgnoringReadOnly();
     entityManager
         .createNativeQuery(String.format("SET TRANSACTION SNAPSHOT '%s'", snapshotId))
-        .executeUpdate();
+        .executeUpdateIgnoringReadOnly();
     return this;
   }
 

core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java

@@ -206,7 +206,7 @@ public class ReadOnlyCheckingEntityManager implements EntityManager {
   }
 
   @Override
-  public Query createNativeQuery(String sqlString) {
+  public ReadOnlyCheckingQuery createNativeQuery(String sqlString) {
     return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString));
   }
 

core/src/main/java/google/registry/rdap/RdapDomainSearchAction.java

@@ -18,7 +18,6 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.HEAD;
@@ -38,8 +37,10 @@ import com.google.common.primitives.Booleans;
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.PersistenceModule.ReadOnlyReplicaJpaTm;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.rdap.RdapJsonFormatter.OutputDataType;
 import google.registry.rdap.RdapMetrics.EndpointType;
 import google.registry.rdap.RdapMetrics.SearchType;
@@ -91,7 +92,11 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
   @Inject @Parameter("name") Optional<String> nameParam;
   @Inject @Parameter("nsLdhName") Optional<String> nsLdhNameParam;
   @Inject @Parameter("nsIp") Optional<String> nsIpParam;
-  @Inject public RdapDomainSearchAction() {
+
+  @Inject @ReadOnlyReplicaJpaTm JpaTransactionManager readOnlyJpaTm;
+
+  @Inject
+  public RdapDomainSearchAction() {
     super("domain search", EndpointType.DOMAINS);
   }
 
@@ -223,32 +228,31 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
       resultSet = getMatchingResources(query, true, querySizeLimit);
     } else {
       resultSet =
-          jpaTm()
-              .transact(
-                  () -> {
-                    CriteriaBuilder criteriaBuilder =
-                        jpaTm().getEntityManager().getCriteriaBuilder();
-                    CriteriaQueryBuilder<DomainBase> queryBuilder =
-                        CriteriaQueryBuilder.create(DomainBase.class)
-                            .where(
-                                "fullyQualifiedDomainName",
-                                criteriaBuilder::like,
-                                String.format("%s%%", partialStringQuery.getInitialString()))
-                            .orderByAsc("fullyQualifiedDomainName");
-                    if (cursorString.isPresent()) {
-                      queryBuilder =
-                          queryBuilder.where(
-                              "fullyQualifiedDomainName",
-                              criteriaBuilder::greaterThan,
-                              cursorString.get());
-                    }
-                    if (partialStringQuery.getSuffix() != null) {
-                      queryBuilder =
-                          queryBuilder.where(
-                              "tld", criteriaBuilder::equal, partialStringQuery.getSuffix());
-                    }
-                    return getMatchingResourcesSql(queryBuilder, true, querySizeLimit);
-                  });
+          readOnlyJpaTm.transact(
+              () -> {
+                CriteriaBuilder criteriaBuilder =
+                    readOnlyJpaTm.getEntityManager().getCriteriaBuilder();
+                CriteriaQueryBuilder<DomainBase> queryBuilder =
+                    CriteriaQueryBuilder.create(DomainBase.class)
+                        .where(
+                            "fullyQualifiedDomainName",
+                            criteriaBuilder::like,
+                            String.format("%s%%", partialStringQuery.getInitialString()))
+                        .orderByAsc("fullyQualifiedDomainName");
+                if (cursorString.isPresent()) {
+                  queryBuilder =
+                      queryBuilder.where(
+                          "fullyQualifiedDomainName",
+                          criteriaBuilder::greaterThan,
+                          cursorString.get());
+                }
+                if (partialStringQuery.getSuffix() != null) {
+                  queryBuilder =
+                      queryBuilder.where(
+                          "tld", criteriaBuilder::equal, partialStringQuery.getSuffix());
+                }
+                return getMatchingResourcesSql(queryBuilder, true, querySizeLimit);
+              });
     }
     return makeSearchResults(resultSet);
   }
@@ -270,20 +274,19 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
       resultSet = getMatchingResources(query, true, querySizeLimit);
     } else {
       resultSet =
-          jpaTm()
-              .transact(
-                  () -> {
-                    CriteriaQueryBuilder<DomainBase> builder =
-                        queryItemsSql(
-                                DomainBase.class,
-                                "tld",
-                                tld,
-                                Optional.of("fullyQualifiedDomainName"),
-                                cursorString,
-                                DeletedItemHandling.INCLUDE)
-                            .orderByAsc("fullyQualifiedDomainName");
-                    return getMatchingResourcesSql(builder, true, querySizeLimit);
-                  });
+          readOnlyJpaTm.transact(
+              () -> {
+                CriteriaQueryBuilder<DomainBase> builder =
+                    queryItemsSql(
+                            DomainBase.class,
+                            "tld",
+                            tld,
+                            Optional.of("fullyQualifiedDomainName"),
+                            cursorString,
+                            DeletedItemHandling.INCLUDE)
+                        .orderByAsc("fullyQualifiedDomainName");
+                return getMatchingResourcesSql(builder, true, querySizeLimit);
+              });
     }
     return makeSearchResults(resultSet);
   }
@@ -354,28 +357,28 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
           .map(VKey::from)
           .collect(toImmutableSet());
     } else {
-      return jpaTm()
-          .transact(
-              () -> {
-                CriteriaQueryBuilder<HostResource> builder =
-                    queryItemsSql(
-                        HostResource.class,
-                        "fullyQualifiedHostName",
-                        partialStringQuery,
-                        Optional.empty(),
-                        DeletedItemHandling.EXCLUDE);
-                if (desiredRegistrar.isPresent()) {
-                  builder =
-                      builder.where(
-                          "currentSponsorClientId",
-                          jpaTm().getEntityManager().getCriteriaBuilder()::equal,
-                          desiredRegistrar.get());
-                }
-                return getMatchingResourcesSql(builder, true, maxNameserversInFirstStage)
-                    .resources().stream()
-                    .map(HostResource::createVKey)
-                    .collect(toImmutableSet());
-              });
+      return readOnlyJpaTm.transact(
+          () -> {
+            CriteriaQueryBuilder<HostResource> builder =
+                queryItemsSql(
+                    HostResource.class,
+                    "fullyQualifiedHostName",
+                    partialStringQuery,
+                    Optional.empty(),
+                    DeletedItemHandling.EXCLUDE);
+            if (desiredRegistrar.isPresent()) {
+              builder =
+                  builder.where(
+                      "currentSponsorClientId",
+                      readOnlyJpaTm.getEntityManager().getCriteriaBuilder()::equal,
+                      desiredRegistrar.get());
+            }
+            return getMatchingResourcesSql(builder, true, maxNameserversInFirstStage)
+                .resources()
+                .stream()
+                .map(HostResource::createVKey)
+                .collect(toImmutableSet());
+          });
     }
   }
 
@@ -509,21 +512,20 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         parameters.put("desiredRegistrar", desiredRegistrar.get());
       }
       hostKeys =
-          jpaTm()
-              .transact(
-                  () -> {
-                    javax.persistence.Query query =
-                        jpaTm()
-                            .getEntityManager()
-                            .createNativeQuery(queryBuilder.toString())
-                            .setMaxResults(maxNameserversInFirstStage);
-                    parameters.build().forEach(query::setParameter);
-                    @SuppressWarnings("unchecked")
-                    Stream<String> resultStream = query.getResultStream();
-                    return resultStream
-                        .map(repoId -> VKey.create(HostResource.class, repoId))
-                        .collect(toImmutableSet());
-                  });
+          readOnlyJpaTm.transact(
+              () -> {
+                javax.persistence.Query query =
+                    readOnlyJpaTm
+                        .getEntityManager()
+                        .createNativeQuery(queryBuilder.toString())
+                        .setMaxResults(maxNameserversInFirstStage);
+                parameters.build().forEach(query::setParameter);
+                @SuppressWarnings("unchecked")
+                Stream<String> resultStream = query.getResultStream();
+                return resultStream
+                    .map(repoId -> VKey.create(HostResource.class, repoId))
+                    .collect(toImmutableSet());
+              });
     }
     return searchByNameserverRefs(hostKeys);
   }
@@ -568,39 +570,38 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         }
         stream.forEach(domainSetBuilder::add);
       } else {
-        jpaTm()
-            .transact(
-                () -> {
-                  for (VKey<HostResource> hostKey : hostKeys) {
-                    CriteriaQueryBuilder<DomainBase> queryBuilder =
-                        CriteriaQueryBuilder.create(DomainBase.class)
-                            .whereFieldContains("nsHosts", hostKey)
-                            .orderByAsc("fullyQualifiedDomainName");
-                    CriteriaBuilder criteriaBuilder =
-                        jpaTm().getEntityManager().getCriteriaBuilder();
-                    if (!shouldIncludeDeleted()) {
-                      queryBuilder =
-                          queryBuilder.where(
-                              "deletionTime", criteriaBuilder::greaterThan, getRequestTime());
-                    }
-                    if (cursorString.isPresent()) {
-                      queryBuilder =
-                          queryBuilder.where(
-                              "fullyQualifiedDomainName",
-                              criteriaBuilder::greaterThan,
-                              cursorString.get());
-                    }
-                    jpaTm()
-                        .criteriaQuery(queryBuilder.build())
-                        .getResultStream()
-                        .filter(this::isAuthorized)
-                        .forEach(
-                            (domain) -> {
-                              Hibernate.initialize(domain.getDsData());
-                              domainSetBuilder.add(domain);
-                            });
-                  }
-                });
+        readOnlyJpaTm.transact(
+            () -> {
+              for (VKey<HostResource> hostKey : hostKeys) {
+                CriteriaQueryBuilder<DomainBase> queryBuilder =
+                    CriteriaQueryBuilder.create(DomainBase.class)
+                        .whereFieldContains("nsHosts", hostKey)
+                        .orderByAsc("fullyQualifiedDomainName");
+                CriteriaBuilder criteriaBuilder =
+                    readOnlyJpaTm.getEntityManager().getCriteriaBuilder();
+                if (!shouldIncludeDeleted()) {
+                  queryBuilder =
+                      queryBuilder.where(
+                          "deletionTime", criteriaBuilder::greaterThan, getRequestTime());
+                }
+                if (cursorString.isPresent()) {
+                  queryBuilder =
+                      queryBuilder.where(
+                          "fullyQualifiedDomainName",
+                          criteriaBuilder::greaterThan,
+                          cursorString.get());
+                }
+                readOnlyJpaTm
+                    .criteriaQuery(queryBuilder.build())
+                    .getResultStream()
+                    .filter(this::isAuthorized)
+                    .forEach(
+                        (domain) -> {
+                          Hibernate.initialize(domain.getDsData());
+                          domainSetBuilder.add(domain);
+                        });
+              }
+            });
       }
     }
     List<DomainBase> domains = domainSetBuilder.build().asList();

core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java

@@ -32,6 +32,7 @@ import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
+import google.registry.persistence.PersistenceModule;
 import google.registry.reporting.ReportingModule;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
@@ -120,17 +121,16 @@ public class GenerateInvoicesAction implements Runnable {
               .setContainerSpecGcsPath(
                   String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
               .setParameters(
-                  ImmutableMap.of(
-                      "yearMonth",
-                      yearMonth.toString("yyyy-MM"),
-                      "invoiceFilePrefix",
-                      invoiceFilePrefix,
-                      "database",
-                      database.name(),
-                      "billingBucketUrl",
-                      billingBucketUrl,
-                      "registryEnvironment",
-                      RegistryEnvironment.get().name()));
+                  new ImmutableMap.Builder<String, String>()
+                      .put("yearMonth", yearMonth.toString("yyyy-MM"))
+                      .put("invoiceFilePrefix", invoiceFilePrefix)
+                      .put("database", database.name())
+                      .put("billingBucketUrl", billingBucketUrl)
+                      .put("registryEnvironment", RegistryEnvironment.get().name())
+                      .put(
+                          "jpaTransactionManagerType",
+                          PersistenceModule.JpaTransactionManagerType.READ_ONLY_REPLICA.toString())
+                      .build());
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json

@@ -69,6 +69,12 @@
       "regexes": [
         "^DATASTORE|CLOUD_SQL$"
       ]
+    },
+    {
+      "name": "jpaTransactionManagerType",
+      "label": "The type of JPA transaction manager to use if using SQL",
+      "helpText": "The standard SQL instance or a read-only replica may be used",
+      "regexes": ["^REGULAR|READ_ONLY_REPLICA$"]
     }
   ]
 }

core/src/test/java/google/registry/cron/TldFanoutActionTest.java

@@ -98,11 +98,7 @@ class TldFanoutActionTest {
   }
 
   private void assertTaskWithoutTld() {
-    cloudTasksHelper.assertTasksEnqueued(
-        QUEUE,
-        new TaskMatcher()
-            .url(ENDPOINT)
-            .header("content-type", "application/x-www-form-urlencoded"));
+    cloudTasksHelper.assertTasksEnqueued(QUEUE, new TaskMatcher().url(ENDPOINT));
   }
 
   @Test

core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java

@@ -0,0 +1,292 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaQuery;
+import org.joda.time.DateTime;
+
+/**
+ * A {@link JpaTransactionManager} that simulates a read-only replica SQL instance.
+ *
+ * <p>We accomplish this by delegating all calls to the standard transaction manager except for
+ * calls that start transactions. For these, we create a transaction like normal but set it to READ
+ * ONLY mode before doing any work. This is similar to how the read-only Postgres replica works; it
+ * treats all transactions as read-only transactions.
+ */
+public class ReplicaSimulatingJpaTransactionManager implements JpaTransactionManager {
+
+  private final JpaTransactionManager delegate;
+
+  public ReplicaSimulatingJpaTransactionManager(JpaTransactionManager delegate) {
+    this.delegate = delegate;
+  }
+
+  public void teardown() {
+    delegate.teardown();
+  }
+
+  public EntityManager getStandaloneEntityManager() {
+    return delegate.getStandaloneEntityManager();
+  }
+
+  public EntityManager getEntityManager() {
+    return delegate.getEntityManager();
+  }
+
+  public JpaTransactionManager setDatabaseSnapshot(String snapshotId) {
+    return delegate.setDatabaseSnapshot(snapshotId);
+  }
+
+  public <T> TypedQuery<T> query(String sqlString, Class<T> resultClass) {
+    return delegate.query(sqlString, resultClass);
+  }
+
+  public <T> TypedQuery<T> criteriaQuery(CriteriaQuery<T> criteriaQuery) {
+    return delegate.criteriaQuery(criteriaQuery);
+  }
+
+  public Query query(String sqlString) {
+    return delegate.query(sqlString);
+  }
+
+  public boolean inTransaction() {
+    return delegate.inTransaction();
+  }
+
+  public void assertInTransaction() {
+    delegate.assertInTransaction();
+  }
+
+  public <T> T transact(Supplier<T> work) {
+    return delegate.transact(
+        () -> {
+          delegate.getEntityManager().createQuery("SET TRANSACTION READ ONLY").executeUpdate();
+          return work.get();
+        });
+  }
+
+  public <T> T transactWithoutBackup(Supplier<T> work) {
+    return transact(work);
+  }
+
+  public <T> T transactNoRetry(Supplier<T> work) {
+    return transact(work);
+  }
+
+  public void transact(Runnable work) {
+    transact(
+        () -> {
+          work.run();
+          return null;
+        });
+  }
+
+  public void transactNoRetry(Runnable work) {
+    transact(work);
+  }
+
+  public <T> T transactNew(Supplier<T> work) {
+    return transact(work);
+  }
+
+  public void transactNew(Runnable work) {
+    transact(work);
+  }
+
+  public <T> T transactNewReadOnly(Supplier<T> work) {
+    return transact(work);
+  }
+
+  public void transactNewReadOnly(Runnable work) {
+    transact(work);
+  }
+
+  public <T> T doTransactionless(Supplier<T> work) {
+    return delegate.doTransactionless(work);
+  }
+
+  public DateTime getTransactionTime() {
+    return delegate.getTransactionTime();
+  }
+
+  public void insert(Object entity) {
+    delegate.insert(entity);
+  }
+
+  public void insertAll(ImmutableCollection<?> entities) {
+    delegate.insertAll(entities);
+  }
+
+  public void insertAll(ImmutableObject... entities) {
+    delegate.insertAll(entities);
+  }
+
+  public void insertWithoutBackup(ImmutableObject entity) {
+    delegate.insertWithoutBackup(entity);
+  }
+
+  public void insertAllWithoutBackup(ImmutableCollection<?> entities) {
+    delegate.insertAllWithoutBackup(entities);
+  }
+
+  public void put(Object entity) {
+    delegate.put(entity);
+  }
+
+  public void putAll(ImmutableObject... entities) {
+    delegate.putAll(entities);
+  }
+
+  public void putAll(ImmutableCollection<?> entities) {
+    delegate.putAll(entities);
+  }
+
+  public void putWithoutBackup(ImmutableObject entity) {
+    delegate.putWithoutBackup(entity);
+  }
+
+  public void putAllWithoutBackup(ImmutableCollection<?> entities) {
+    delegate.putAllWithoutBackup(entities);
+  }
+
+  public void update(Object entity) {
+    delegate.update(entity);
+  }
+
+  public void updateAll(ImmutableCollection<?> entities) {
+    delegate.updateAll(entities);
+  }
+
+  public void updateAll(ImmutableObject... entities) {
+    delegate.updateAll(entities);
+  }
+
+  public void updateWithoutBackup(ImmutableObject entity) {
+    delegate.updateWithoutBackup(entity);
+  }
+
+  public void updateAllWithoutBackup(ImmutableCollection<?> entities) {
+    delegate.updateAllWithoutBackup(entities);
+  }
+
+  public <T> boolean exists(VKey<T> key) {
+    return delegate.exists(key);
+  }
+
+  public boolean exists(Object entity) {
+    return delegate.exists(entity);
+  }
+
+  public <T> Optional<T> loadByKeyIfPresent(VKey<T> key) {
+    return delegate.loadByKeyIfPresent(key);
+  }
+
+  public <T> ImmutableMap<VKey<? extends T>, T> loadByKeysIfPresent(
+      Iterable<? extends VKey<? extends T>> vKeys) {
+    return delegate.loadByKeysIfPresent(vKeys);
+  }
+
+  public <T> ImmutableList<T> loadByEntitiesIfPresent(Iterable<T> entities) {
+    return delegate.loadByEntitiesIfPresent(entities);
+  }
+
+  public <T> T loadByKey(VKey<T> key) {
+    return delegate.loadByKey(key);
+  }
+
+  public <T> ImmutableMap<VKey<? extends T>, T> loadByKeys(
+      Iterable<? extends VKey<? extends T>> vKeys) {
+    return delegate.loadByKeys(vKeys);
+  }
+
+  public <T> T loadByEntity(T entity) {
+    return delegate.loadByEntity(entity);
+  }
+
+  public <T> ImmutableList<T> loadByEntities(Iterable<T> entities) {
+    return delegate.loadByEntities(entities);
+  }
+
+  public <T> ImmutableList<T> loadAllOf(Class<T> clazz) {
+    return delegate.loadAllOf(clazz);
+  }
+
+  public <T> Stream<T> loadAllOfStream(Class<T> clazz) {
+    return delegate.loadAllOfStream(clazz);
+  }
+
+  public <T> Optional<T> loadSingleton(Class<T> clazz) {
+    return delegate.loadSingleton(clazz);
+  }
+
+  public void delete(VKey<?> key) {
+    delegate.delete(key);
+  }
+
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    delegate.delete(vKeys);
+  }
+
+  public <T> T delete(T entity) {
+    return delegate.delete(entity);
+  }
+
+  public void deleteWithoutBackup(VKey<?> key) {
+    delegate.deleteWithoutBackup(key);
+  }
+
+  public void deleteWithoutBackup(Iterable<? extends VKey<?>> keys) {
+    delegate.deleteWithoutBackup(keys);
+  }
+
+  public void deleteWithoutBackup(Object entity) {
+    delegate.deleteWithoutBackup(entity);
+  }
+
+  public <T> QueryComposer<T> createQueryComposer(Class<T> entity) {
+    return delegate.createQueryComposer(entity);
+  }
+
+  public void clearSessionCache() {
+    delegate.clearSessionCache();
+  }
+
+  public boolean isOfy() {
+    return delegate.isOfy();
+  }
+
+  public void putIgnoringReadOnlyWithoutBackup(Object entity) {
+    delegate.putIgnoringReadOnlyWithoutBackup(entity);
+  }
+
+  public void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key) {
+    delegate.deleteIgnoringReadOnlyWithoutBackup(key);
+  }
+
+  public <T> void assertDelete(VKey<T> key) {
+    delegate.assertDelete(key);
+  }
+}

core/src/test/java/google/registry/rdap/RdapDomainSearchActionTest.java

@@ -15,6 +15,7 @@
 package google.registry.rdap;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.rdap.RdapTestHelper.assertThat;
 import static google.registry.rdap.RdapTestHelper.parseJsonObject;
 import static google.registry.request.Action.Method.POST;
@@ -45,6 +46,7 @@ import google.registry.model.registrar.Registrar;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.ReplicaSimulatingJpaTransactionManager;
 import google.registry.rdap.RdapMetrics.EndpointType;
 import google.registry.rdap.RdapMetrics.SearchType;
 import google.registry.rdap.RdapMetrics.WildcardType;
@@ -93,39 +95,27 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
   }
 
   private JsonObject generateActualJson(RequestType requestType, String paramValue, String cursor) {
-    action.requestPath = actionPath;
-    action.requestMethod = POST;
-    String requestTypeParam = null;
+    String requestTypeParam;
     switch (requestType) {
       case NAME:
         action.nameParam = Optional.of(paramValue);
-        action.nsLdhNameParam = Optional.empty();
-        action.nsIpParam = Optional.empty();
         requestTypeParam = "name";
         break;
       case NS_LDH_NAME:
-        action.nameParam = Optional.empty();
         action.nsLdhNameParam = Optional.of(paramValue);
-        action.nsIpParam = Optional.empty();
         requestTypeParam = "nsLdhName";
         break;
       case NS_IP:
-        action.nameParam = Optional.empty();
-        action.nsLdhNameParam = Optional.empty();
         action.nsIpParam = Optional.of(paramValue);
         requestTypeParam = "nsIp";
         break;
       default:
-        action.nameParam = Optional.empty();
-        action.nsLdhNameParam = Optional.empty();
-        action.nsIpParam = Optional.empty();
         requestTypeParam = "";
         break;
     }
     if (paramValue != null) {
       if (cursor == null) {
         action.parameterMap = ImmutableListMultimap.of(requestTypeParam, paramValue);
-        action.cursorTokenParam = Optional.empty();
       } else {
         action.parameterMap =
             ImmutableListMultimap.of(requestTypeParam, paramValue, "cursor", cursor);
@@ -381,6 +371,12 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
             clock.nowUtc()));
 
     action.requestMethod = POST;
+    action.nameParam = Optional.empty();
+    action.nsLdhNameParam = Optional.empty();
+    action.nsIpParam = Optional.empty();
+    action.cursorTokenParam = Optional.empty();
+    action.requestPath = actionPath;
+    action.readOnlyJpaTm = jpaTm();
   }
 
   private JsonObject generateExpectedJsonForTwoDomainsNsReply() {
@@ -728,6 +724,18 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
     verifyMetrics(SearchType.BY_DOMAIN_NAME, Optional.of(1L));
   }
 
+  @TestSqlOnly
+  void testDomainMatch_readOnlyReplica() {
+    login("evilregistrar");
+    rememberWildcardType("cat.lol");
+    action.readOnlyJpaTm = new ReplicaSimulatingJpaTransactionManager(jpaTm());
+    action.nameParam = Optional.of("cat.lol");
+    action.parameterMap = ImmutableListMultimap.of("name", "cat.lol");
+    action.run();
+    assertThat(response.getPayload()).contains("Yes Virginia <script>");
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
   @TestOfyAndSql
   void testDomainMatch_foundWithUpperCase() {
     login("evilregistrar");

core/src/test/java/google/registry/testing/CloudTasksHelper.java

@@ -227,22 +227,20 @@ public class CloudTasksHelper implements Serializable {
               });
       headers = headerBuilder.build();
       ImmutableMultimap.Builder<String, String> paramBuilder = new ImmutableMultimap.Builder<>();
-      String query = null;
+      // Note that UriParameters.parse() does not throw an IAE on a bad query string (e.g. one
+      // where parameters are not properly URL-encoded); it always does a best-effort parse.
       if (method == HttpMethod.GET) {
-        query = uri.getQuery();
-      } else if (method == HttpMethod.POST) {
+        paramBuilder.putAll(UriParameters.parse(uri.getQuery()));
+      } else if (method == HttpMethod.POST && !task.getAppEngineHttpRequest().getBody().isEmpty()) {
         assertThat(
                 headers.containsEntry(
                     Ascii.toLowerCase(HttpHeaders.CONTENT_TYPE), MediaType.FORM_DATA.toString()))
             .isTrue();
-        query = task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8);
-      }
-      if (query != null) {
-        // Note that UriParameters.parse() does not throw an IAE on a bad query string (e.g. one
-        // where parameters are not properly URL-encoded); it always does a best-effort parse.
-        paramBuilder.putAll(UriParameters.parse(query));
-        params = paramBuilder.build();
+        paramBuilder.putAll(
+            UriParameters.parse(
+                task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8)));
       }
+      params = paramBuilder.build();
     }
 
     public Map<String, Object> toMap() {

util/src/main/java/google/registry/util/CloudTasksUtils.java

@@ -106,30 +106,34 @@ public class CloudTasksUtils implements Serializable {
     checkArgument(
         path != null && !path.isEmpty() && path.charAt(0) == '/',
         "The path must start with a '/'.");
+    checkArgument(
+        method.equals(HttpMethod.GET) || method.equals(HttpMethod.POST),
+        "HTTP method %s is used. Only GET and POST are allowed.",
+        method);
     AppEngineHttpRequest.Builder requestBuilder =
         AppEngineHttpRequest.newBuilder()
             .setHttpMethod(method)
             .setAppEngineRouting(AppEngineRouting.newBuilder().setService(service).build());
-    Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
-    String encodedParams =
-        Joiner.on("&")
-            .join(
-                params.entries().stream()
-                    .map(
-                        entry ->
-                            String.format(
-                                "%s=%s",
-                                escaper.escape(entry.getKey()), escaper.escape(entry.getValue())))
-                    .collect(toImmutableList()));
-    if (method == HttpMethod.GET) {
-      path = String.format("%s?%s", path, encodedParams);
-    } else if (method == HttpMethod.POST) {
-      requestBuilder
-          .putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
-          .setBody(ByteString.copyFrom(encodedParams, StandardCharsets.UTF_8));
-    } else {
-      throw new IllegalArgumentException(
-          String.format("HTTP method %s is used. Only GET and POST are allowed.", method));
+
+    if (!CollectionUtils.isNullOrEmpty(params)) {
+      Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
+      String encodedParams =
+          Joiner.on("&")
+              .join(
+                  params.entries().stream()
+                      .map(
+                          entry ->
+                              String.format(
+                                  "%s=%s",
+                                  escaper.escape(entry.getKey()), escaper.escape(entry.getValue())))
+                      .collect(toImmutableList()));
+      if (method == HttpMethod.GET) {
+        path = String.format("%s?%s", path, encodedParams);
+      } else {
+        requestBuilder
+            .putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
+            .setBody(ByteString.copyFrom(encodedParams, StandardCharsets.UTF_8));
+      }
     }
     requestBuilder.setRelativeUri(path);
     return Task.newBuilder().setAppEngineHttpRequest(requestBuilder.build()).build();

util/src/main/java/google/registry/util/CollectionUtils.java

@@ -49,6 +49,11 @@ public class CollectionUtils {
     return potentiallyNull == null || potentiallyNull.isEmpty();
   }
 
+  /** Checks if a Multimap is null or empty. */
+  public static boolean isNullOrEmpty(@Nullable Multimap<?, ?> potentiallyNull) {
+    return potentiallyNull == null || potentiallyNull.isEmpty();
+  }
+
   /** Turns a null set into an empty set. JAXB leaves lots of null sets lying around. */
   public static <T> Set<T> nullToEmpty(@Nullable Set<T> potentiallyNull) {
     return firstNonNull(potentiallyNull, ImmutableSet.of());

util/src/test/java/google/registry/util/CloudTasksUtilsTest.java

@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;
 import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.cloud.tasks.v2.Task;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.LinkedListMultimap;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
@@ -80,6 +81,48 @@ public class CloudTasksUtilsTest {
     assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
   }
 
+  @Test
+  void testSuccess_createGetTasks_withNullParams() {
+    Task task = CloudTasksUtils.createGetTask("/the/path", "myservice", null);
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
+  @Test
+  void testSuccess_createPostTasks_withNullParams() {
+    Task task = CloudTasksUtils.createPostTask("/the/path", "myservice", null);
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8)).isEmpty();
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
+  @Test
+  void testSuccess_createGetTasks_withEmptyParams() {
+    Task task = CloudTasksUtils.createGetTask("/the/path", "myservice", ImmutableMultimap.of());
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
+  @Test
+  void testSuccess_createPostTasks_withEmptyParams() {
+    Task task = CloudTasksUtils.createPostTask("/the/path", "myservice", ImmutableMultimap.of());
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8)).isEmpty();
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
   @SuppressWarnings("ProtoTimestampGetSecondsGetNano")
   @Test
   void testSuccess_createGetTasks_withJitterSeconds() {

util/src/test/java/google/registry/util/CollectionUtilsTest.java

@@ -22,6 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Multimap;
 import java.util.Map;
 import org.junit.jupiter.api.Test;
 
@@ -43,6 +45,12 @@ class CollectionUtilsTest {
     assertThat(convertedMap).isEmpty();
   }
 
+  @Test
+  void testNullOrEmptyMultimap() {
+    assertThat(CollectionUtils.isNullOrEmpty((Multimap<?, ?>) null)).isTrue();
+    assertThat(CollectionUtils.isNullOrEmpty(ImmutableMultimap.of())).isTrue();
+  }
+
   @Test
   void testPartitionMap() {
     Map<String, String> map = ImmutableMap.of("ka", "va", "kb", "vb", "kc", "vc");