Allow usage of a read-only Postgres replica (#1470)

* Allow usage of a read-only Postgres replica

This adds the Dagger provider code for both the regular and the BEAM
environments, which are similar but not quite the same.

In addition, this demonstrates usage of the replica DB in the
RdePipeline. I tested this on alpha with a modified version of the
RdePipeline that attempts to write some dummy values to the database and
it failed with the expected message that one cannot write to a replica.

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -110,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -131,7 +131,7 @@ public final class AsyncTaskEnqueuer {
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().stringify())
             .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))
@@ -148,7 +148,7 @@ public final class AsyncTaskEnqueuer {
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,
         TaskOptions.Builder.withMethod(Method.PULL)
-            .param(PARAM_HOST_KEY, hostKey.getOfyKey().getString())
+            .param(PARAM_HOST_KEY, hostKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString()));
   }
 

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -23,6 +23,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
+import google.registry.persistence.PersistenceModule.BeamReadOnlyReplicaJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.privileges.secretmanager.SecretManagerModule;
@@ -59,6 +60,13 @@ public interface RegistryPipelineComponent {
   @BeamBulkQueryJpaTm
   Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
 
+  /**
+   * A {@link JpaTransactionManager} that uses the Postgres read-only replica if configured (uses
+   * the standard DB otherwise).
+   */
+  @BeamReadOnlyReplicaJpaTm
+  Lazy<JpaTransactionManager> getReadOnlyReplicaJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -56,6 +56,10 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
       case BULK_QUERY:
         transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
         break;
+      case READ_ONLY_REPLICA:
+        transactionManagerLazy =
+            registryPipelineComponent.getReadOnlyReplicaJpaTransactionManager();
+        break;
       case REGULAR:
       default:
         transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -392,19 +392,26 @@ public final class RegistryConfig {
 
     @Provides
     @Config("cloudSqlJdbcUrl")
-    public static String providesCloudSqlJdbcUrl(RegistryConfigSettings config) {
+    public static String provideCloudSqlJdbcUrl(RegistryConfigSettings config) {
       return config.cloudSql.jdbcUrl;
     }
 
     @Provides
     @Config("cloudSqlInstanceConnectionName")
-    public static String providesCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
+    public static String provideCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
       return config.cloudSql.instanceConnectionName;
     }
 
+    @Provides
+    @Config("cloudSqlReplicaInstanceConnectionName")
+    public static Optional<String> provideCloudSqlReplicaInstanceConnectionName(
+        RegistryConfigSettings config) {
+      return Optional.ofNullable(config.cloudSql.replicaInstanceConnectionName);
+    }
+
     @Provides
     @Config("cloudSqlDbInstanceName")
-    public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
+    public static String provideCloudSqlDbInstance(RegistryConfigSettings config) {
       // Format of instanceConnectionName: project-id:region:instance-name
       int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
       return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -128,6 +128,7 @@ public class RegistryConfigSettings {
     // TODO(05012021): remove username field after it is removed from all yaml files.
     public String username;
     public String instanceConnectionName;
+    public String replicaInstanceConnectionName;
   }
 
   /** Configuration for Apache Beam (Cloud Dataflow). */

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -231,6 +231,7 @@ cloudSql:
   jdbcUrl: jdbc:postgresql://localhost
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
+  replicaInstanceConnectionName: null
 
 cloudDns:
   # Set both properties to null in Production.

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -27,6 +27,7 @@ import google.registry.model.EppResource;
 import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.replay.DatastoreOnlyEntity;
+import google.registry.persistence.VKey;
 
 /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */
 @ReportedOn
@@ -66,8 +67,9 @@ public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEn
     EppResourceIndex instance = instantiate(EppResourceIndex.class);
     instance.reference = resourceKey;
     instance.kind = resourceKey.getKind();
-    // TODO(b/207368050): figure out if this value has ever been used other than test cases
-    instance.id = resourceKey.getString(); // creates a web-safe key string
+    // creates a web-safe key string, this value is never used
+    // TODO(b/211785379): remove unused id
+    instance.id = VKey.from(resourceKey).stringify();
     instance.bucket = bucket;
     return instance;
   }

core/src/main/java/google/registry/persistence/PersistenceComponent.java

@@ -19,6 +19,7 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
+import google.registry.persistence.PersistenceModule.ReadOnlyReplicaJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
@@ -40,4 +41,7 @@ public interface PersistenceComponent {
 
   @AppEngineJpaTm
   JpaTransactionManager appEngineJpaTransactionManager();
+
+  @ReadOnlyReplicaJpaTm
+  JpaTransactionManager readOnlyReplicaJpaTransactionManager();
 }

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -122,8 +122,11 @@ public abstract class PersistenceModule {
       @Config("cloudSqlJdbcUrl") String jdbcUrl,
       @Config("cloudSqlInstanceConnectionName") String instanceConnectionName,
       @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs) {
-    return createPartialSqlConfigs(
-        jdbcUrl, instanceConnectionName, defaultConfigs, Optional.empty());
+    HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
+    overrides.put(Environment.URL, jdbcUrl);
+    overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory");
+    overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName);
+    return ImmutableMap.copyOf(overrides);
   }
 
   /**
@@ -184,22 +187,6 @@ public abstract class PersistenceModule {
     return ImmutableMap.copyOf(overrides);
   }
 
-  @VisibleForTesting
-  static ImmutableMap<String, String> createPartialSqlConfigs(
-      String jdbcUrl,
-      String instanceConnectionName,
-      ImmutableMap<String, String> defaultConfigs,
-      Optional<Provider<TransactionIsolationLevel>> isolationOverride) {
-    HashMap<String, String> overrides = Maps.newHashMap(defaultConfigs);
-    overrides.put(Environment.URL, jdbcUrl);
-    overrides.put(HIKARI_DS_SOCKET_FACTORY, "com.google.cloud.sql.postgres.SocketFactory");
-    overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName);
-    isolationOverride
-        .map(Provider::get)
-        .ifPresent(override -> overrides.put(Environment.ISOLATION, override.name()));
-    return ImmutableMap.copyOf(overrides);
-  }
-
   /**
    * Provides a {@link Supplier} of single-use JDBC {@link Connection connections} that can manage
    * the database DDL schema.
@@ -280,6 +267,36 @@ public abstract class PersistenceModule {
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
+  @Provides
+  @Singleton
+  @ReadOnlyReplicaJpaTm
+  static JpaTransactionManager provideReadOnlyReplicaJpaTm(
+      SqlCredentialStore credentialStore,
+      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
+      @Config("cloudSqlReplicaInstanceConnectionName")
+          Optional<String> replicaInstanceConnectionName,
+      Clock clock) {
+    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
+    setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides);
+    replicaInstanceConnectionName.ifPresent(
+        name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
+    return new JpaTransactionManagerImpl(create(overrides), clock);
+  }
+
+  @Provides
+  @Singleton
+  @BeamReadOnlyReplicaJpaTm
+  static JpaTransactionManager provideBeamReadOnlyReplicaJpaTm(
+      @BeamPipelineCloudSqlConfigs ImmutableMap<String, String> beamCloudSqlConfigs,
+      @Config("cloudSqlReplicaInstanceConnectionName")
+          Optional<String> replicaInstanceConnectionName,
+      Clock clock) {
+    HashMap<String, String> overrides = Maps.newHashMap(beamCloudSqlConfigs);
+    replicaInstanceConnectionName.ifPresent(
+        name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
+    return new JpaTransactionManagerImpl(create(overrides), clock);
+  }
+
   /** Constructs the {@link EntityManagerFactory} instance. */
   @VisibleForTesting
   static EntityManagerFactory create(
@@ -357,7 +374,12 @@ public abstract class PersistenceModule {
      * The {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities. Please
      * see {@link google.registry.model.bulkquery.BulkQueryEntities} for more information.
      */
-    BULK_QUERY
+    BULK_QUERY,
+    /**
+     * The {@link JpaTransactionManager} that uses the read-only Postgres replica if configured, or
+     * the standard DB if not.
+     */
+    READ_ONLY_REPLICA
   }
 
   /** Dagger qualifier for JDBC {@link Connection} with schema management privilege. */
@@ -383,6 +405,22 @@ public abstract class PersistenceModule {
   @Documented
   public @interface BeamBulkQueryJpaTm {}
 
+  /**
+   * Dagger qualifier for {@link JpaTransactionManager} used inside BEAM pipelines that uses the
+   * read-only Postgres replica if one is configured (otherwise it uses the standard DB).
+   */
+  @Qualifier
+  @Documented
+  public @interface BeamReadOnlyReplicaJpaTm {}
+
+  /**
+   * Dagger qualifier for {@link JpaTransactionManager} that uses the read-only Postgres replica if
+   * one is configured (otherwise it uses the standard DB).
+   */
+  @Qualifier
+  @Documented
+  public @interface ReadOnlyReplicaJpaTm {}
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for Nomulus tool. */
   @Qualifier
   @Documented

core/src/main/java/google/registry/persistence/VKey.java

@@ -144,7 +144,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    */
   public static <T> VKey<T> create(String keyString) {
     if (!keyString.startsWith(CLASS_TYPE + KV_SEPARATOR)) {
-      // to handle the existing ofy key string
+      // handle the existing ofy key string
       return fromWebsafeKey(keyString);
     } else {
       ImmutableMap<String, String> kvs =
@@ -307,6 +307,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
     if (maybeGetSqlKey().isPresent()) {
       key += DELIMITER + SQL_LOOKUP_KEY + KV_SEPARATOR + SerializeUtils.stringify(getSqlKey());
     }
+    // getString() method returns a Base64 encoded web safe of ofy key
     if (maybeGetOfyKey().isPresent()) {
       key += DELIMITER + OFY_LOOKUP_KEY + KV_SEPARATOR + getOfyKey().getString();
     }

core/src/main/java/google/registry/rde/RdeModule.java

@@ -14,7 +14,6 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static google.registry.request.RequestParameters.extractBooleanParameter;
 import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
@@ -22,7 +21,6 @@ import static google.registry.request.RequestParameters.extractRequiredDatetimeP
 import static google.registry.request.RequestParameters.extractSetOfDatetimeParameters;
 import static google.registry.request.RequestParameters.extractSetOfParameters;
 
-import com.google.appengine.api.taskqueue.Queue;
 import com.google.common.collect.ImmutableSet;
 import com.jcraft.jsch.SftpProgressMonitor;
 import dagger.Binds;
@@ -30,7 +28,6 @@ import dagger.Module;
 import dagger.Provides;
 import google.registry.request.Parameter;
 import java.util.Optional;
-import javax.inject.Named;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.DateTime;
 
@@ -110,12 +107,6 @@ public abstract class RdeModule {
     return extractOptionalParameter(req, PARAM_PREFIX);
   }
 
-  @Provides
-  @Named("rde-report")
-  static Queue provideQueueRdeReport() {
-    return getQueue("rde-report");
-  }
-
   @Binds
   abstract SftpProgressMonitor provideSftpProgressMonitor(
       LoggingSftpProgressMonitor loggingSftpProgressMonitor);

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -56,6 +56,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.request.Action;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
@@ -340,6 +341,9 @@ public final class RdeStagingAction implements Runnable {
                                           .encode(stagingKeyBytes))
                                   .put("registryEnvironment", RegistryEnvironment.get().name())
                                   .put("workerMachineType", machineType)
+                                  .put(
+                                      "jpaTransactionManagerType",
+                                      JpaTransactionManagerType.READ_ONLY_REPLICA.toString())
                                   // TODO (jianglai): Investigate turning off public IPs (for which
                                   // there is a quota) in order to increase the total number of
                                   // workers allowed (also under quota).

core/src/main/java/google/registry/tools/GenerateEscrowDepositCommand.java

@@ -14,7 +14,6 @@
 
 package google.registry.tools;
 
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.model.tld.Registries.assertTldsExist;
 import static google.registry.rde.RdeModule.PARAM_BEAM;
 import static google.registry.rde.RdeModule.PARAM_DIRECTORY;
@@ -23,23 +22,22 @@ import static google.registry.rde.RdeModule.PARAM_MANUAL;
 import static google.registry.rde.RdeModule.PARAM_MODE;
 import static google.registry.rde.RdeModule.PARAM_REVISION;
 import static google.registry.rde.RdeModule.PARAM_WATERMARKS;
+import static google.registry.rde.RdeModule.RDE_REPORT_QUEUE;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
 import com.beust.jcommander.Parameters;
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.rde.RdeMode;
 import google.registry.rde.RdeStagingAction;
+import google.registry.request.Action.Service;
 import google.registry.tools.params.DateTimeParameter;
 import google.registry.util.AppEngineServiceUtils;
+import google.registry.util.CloudTasksUtils;
 import java.util.List;
-import java.util.Optional;
 import java.util.stream.Collectors;
 import javax.inject.Inject;
-import javax.inject.Named;
 import org.joda.time.DateTime;
 
 /**
@@ -94,15 +92,7 @@ final class GenerateEscrowDepositCommand implements CommandWithRemoteApi {
 
   @Inject AppEngineServiceUtils appEngineServiceUtils;
 
-  @Inject
-  @Named("rde-report")
-  Queue queue;
-
-  // ETA is a required property for TaskOptions but we let the service to set it when submitting the
-  // task to the task queue. However, the local test service doesn't do that for us during the unit
-  // test, so we add this field here to let the unit test be able to inject the ETA to pass the
-  // test.
-  @VisibleForTesting Optional<Long> maybeEtaMillis = Optional.empty();
+  @Inject CloudTasksUtils cloudTasksUtils;
 
   @Override
   public void run() {
@@ -126,27 +116,25 @@ final class GenerateEscrowDepositCommand implements CommandWithRemoteApi {
       throw new ParameterException("Output subdirectory must not be empty");
     }
 
-    // Unlike many tool commands, this command is actually invoking an action on the backend module
-    // (because it's a mapreduce). So we invoke it in a different way.
-    String hostname = appEngineServiceUtils.getCurrentVersionHostname("backend");
-    TaskOptions opts =
-        withUrl(RdeStagingAction.PATH)
-            .header("Host", hostname)
-            .param(PARAM_MANUAL, String.valueOf(true))
-            .param(PARAM_MODE, mode.toString())
-            .param(PARAM_DIRECTORY, outdir)
-            .param(PARAM_LENIENT, Boolean.toString(lenient))
-            .param(PARAM_BEAM, Boolean.toString(beam))
-            .param(PARAM_TLDS, tlds.stream().collect(Collectors.joining(",")))
-            .param(
+    ImmutableMultimap.Builder<String, String> paramsBuilder =
+        new ImmutableMultimap.Builder<String, String>()
+            .put(PARAM_MANUAL, String.valueOf(true))
+            .put(PARAM_MODE, mode.toString())
+            .put(PARAM_DIRECTORY, outdir)
+            .put(PARAM_LENIENT, Boolean.toString(lenient))
+            .put(PARAM_BEAM, Boolean.toString(beam))
+            .put(PARAM_TLDS, tlds.stream().collect(Collectors.joining(",")))
+            .put(
                 PARAM_WATERMARKS,
                 watermarks.stream().map(DateTime::toString).collect(Collectors.joining(",")));
+
     if (revision != null) {
-      opts = opts.param(PARAM_REVISION, String.valueOf(revision));
+      paramsBuilder.put(PARAM_REVISION, String.valueOf(revision));
     }
-    if (maybeEtaMillis.isPresent()) {
-      opts = opts.etaMillis(maybeEtaMillis.get());
-    }
-    queue.add(opts);
+    cloudTasksUtils.enqueue(
+        RDE_REPORT_QUEUE,
+        CloudTasksUtils.createPostTask(
+            RdeStagingAction.PATH, Service.BACKEND.toString(), paramsBuilder.build()));
   }
+
 }

core/src/main/java/google/registry/tools/GetEppResourceCommand.java

@@ -55,7 +55,7 @@ abstract class GetEppResourceCommand implements CommandWithRemoteApi {
             ? String.format(
                 "%s\n\nWebsafe key: %s",
                 expand ? resource.get().toHydratedString() : resource.get(),
-                resource.get().createVKey().getOfyKey().getString())
+                resource.get().createVKey().stringify())
             : String.format("%s '%s' does not exist or is deleted\n", resourceType, uniqueId));
   }
 

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -19,6 +19,7 @@ import dagger.Component;
 import dagger.Lazy;
 import google.registry.batch.BatchModule;
 import google.registry.bigquery.BigqueryModule;
+import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule.LocalCredentialJson;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
@@ -64,6 +65,7 @@ import javax.inject.Singleton;
       BigqueryModule.class,
       ConfigModule.class,
       CloudDnsWriterModule.class,
+      CloudTasksUtilsModule.class,
       DatastoreAdminModule.class,
       DatastoreServiceModule.class,
       DummyKeyringModule.class,

core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json

@@ -11,6 +11,12 @@
         "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
       ]
     },
+    {
+      "name": "jpaTransactionManagerType",
+      "label": "The type of JPA transaction manager to use",
+      "helpText": "The standard SQL instance or a read-only replica may be used",
+      "regexes": ["^REGULAR|READ_ONLY_REPLICA$"]
+    },
     {
       "name": "pendings",
       "label": "The pendings deposits to generate.",

core/src/test/java/google/registry/batch/AsyncTaskEnqueuerTest.java

@@ -103,7 +103,7 @@ public class AsyncTaskEnqueuerTest {
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, contact.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .etaDelta(
                 standardDays(5).minus(standardSeconds(30)),
@@ -125,7 +125,7 @@ public class AsyncTaskEnqueuerTest {
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, contact.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, now.toString())
             .param(PARAM_RESAVE_TIMES, "2015-05-20T14:34:56.000Z,2015-05-21T15:34:56.000Z")
             .etaDelta(

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -18,6 +18,7 @@ import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.common.collect.MoreCollectors.onlyElement;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
 import static google.registry.batch.AsyncTaskMetrics.OperationResult.STALE;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
@@ -496,7 +497,7 @@ public class DeleteContactsAndHostsActionTest
         QUEUE_ASYNC_DELETE,
         new TaskMatcher()
             .etaDelta(standardHours(23), standardHours(25))
-            .param("resourceKey", contactNotSaved.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, contactNotSaved.createVKey().stringify())
             .param("requestingClientId", "TheRegistrar")
             .param("clientTransactionId", "fakeClientTrid")
             .param("serverTransactionId", "fakeServerTrid")
@@ -504,7 +505,7 @@ public class DeleteContactsAndHostsActionTest
             .param("requestedTime", timeBeforeRun.toString()),
         new TaskMatcher()
             .etaDelta(standardHours(23), standardHours(25))
-            .param("resourceKey", hostNotSaved.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, hostNotSaved.createVKey().stringify())
             .param("requestingClientId", "TheRegistrar")
             .param("clientTransactionId", "fakeClientTrid")
             .param("serverTransactionId", "fakeServerTrid")

core/src/test/java/google/registry/batch/RefreshDnsOnHostRenameActionTest.java

@@ -145,7 +145,7 @@ public class RefreshDnsOnHostRenameActionTest
     assertTasksEnqueued(
         QUEUE_ASYNC_HOST_RENAME,
         new TaskMatcher()
-            .param(PARAM_HOST_KEY, host.createVKey().getOfyKey().getString())
+            .param(PARAM_HOST_KEY, host.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, timeEnqueued.toString()));
     verify(action.asyncTaskMetrics).recordDnsRefreshBatchSize(1L);
     verifyNoMoreInteractions(action.asyncTaskMetrics);
@@ -237,7 +237,7 @@ public class RefreshDnsOnHostRenameActionTest
         QUEUE_ASYNC_HOST_RENAME,
         new TaskMatcher()
             .etaDelta(standardHours(23), standardHours(25))
-            .param("hostKey", host.createVKey().getOfyKey().getString()));
+            .param(PARAM_HOST_KEY, host.createVKey().stringify()));
     assertThat(acquireLock()).isPresent();
   }
 

core/src/test/java/google/registry/batch/ResaveEntityActionTest.java

@@ -150,7 +150,7 @@ public class ResaveEntityActionTest {
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, resavedDomain.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, resavedDomain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, requestedTime.toString())
             .etaDelta(
                 standardDays(5).minus(standardSeconds(30)),

core/src/test/java/google/registry/flows/ResourceFlowTestCase.java

@@ -16,6 +16,7 @@ package google.registry.flows;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
@@ -146,7 +147,7 @@ public abstract class ResourceFlowTestCase<F extends Flow, R extends EppResource
     TaskMatcher expected =
         new TaskMatcher()
             .etaDelta(Duration.standardSeconds(75), Duration.standardSeconds(105)) // expected: 90
-            .param("resourceKey", resource.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, resource.createVKey().stringify())
             .param("requestingClientId", requestingClientId)
             .param("serverTransactionId", trid.getServerTransactionId())
             .param("isSuperuser", Boolean.toString(isSuperuser))

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -321,7 +321,7 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, domain.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .param(PARAM_RESAVE_TIMES, clock.nowUtc().plusDays(5).toString())
             .etaDelta(when.minus(standardSeconds(30)), when.plus(standardSeconds(30))));

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -521,7 +521,7 @@ class DomainTransferRequestFlowTest
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, domain.createVKey().getOfyKey().getString())
+            .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .etaDelta(
                 registry.getAutomaticTransferLength().minus(standardSeconds(30)),

core/src/test/java/google/registry/flows/host/HostUpdateFlowTest.java

@@ -16,6 +16,7 @@ package google.registry.flows.host;
 
 import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.batch.AsyncTaskEnqueuer.PARAM_HOST_KEY;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -230,7 +231,7 @@ class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, HostResour
     assertTasksEnqueued(
         QUEUE_ASYNC_HOST_RENAME,
         new TaskMatcher()
-            .param("hostKey", renamedHost.createVKey().getOfyKey().getString())
+            .param(PARAM_HOST_KEY, renamedHost.createVKey().stringify())
             .param("requestedTime", clock.nowUtc().toString()));
   }
 

core/src/test/java/google/registry/persistence/VKeyTest.java

@@ -29,6 +29,7 @@ import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.common.ClassPathManager;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.host.HostResource;
 import google.registry.model.registrar.RegistrarContact;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
@@ -207,6 +208,29 @@ class VKeyTest {
         .isEqualTo(VKey.createSql(TestObject.class, "foo"));
   }
 
+  @Test
+  void testCreate_stringifiedVKey_resourceKeyFromTaskQueue() throws Exception {
+    VKey<HostResource> vkeyFromNewWebsafeKey =
+        VKey.create(
+            "kind:HostResource@sql:rO0ABXQADzZCQjJGNDc2LUdPT0dMRQ@ofy:ahdzfm"
+                + "RvbWFpbi1yZWdpc3RyeS1hbHBoYXIhCxIMSG9zdFJlc291cmNlIg82QkIyRjQ3Ni1HT09HTEUM");
+
+    assertThat(vkeyFromNewWebsafeKey.getSqlKey()).isEqualTo("6BB2F476-GOOGLE");
+    assertThat(vkeyFromNewWebsafeKey.getOfyKey().getString())
+        .isEqualTo(
+            "ahdzfmRvbWFpb"
+                + "i1yZWdpc3RyeS1hbHBoYXIhCxIMSG9zdFJlc291cmNlIg82QkIyRjQ3Ni1HT09HTEUM");
+
+    // the ofy portion of the new vkey string representation was the old vkey string representation
+    VKey<HostResource> vkeyFromOldWebsafeString =
+        VKey.fromWebsafeKey(
+            "ahdzfmRvbW"
+                + "Fpbi1yZWdpc3RyeS1hbHBoYXIhCxIMSG9zdFJlc291cmNlIg82QkIyRjQ3Ni1HT09HTEUM");
+
+    // the following is assertion is ensure backwork compatibility
+    assertThat(vkeyFromNewWebsafeKey).isEqualTo(vkeyFromOldWebsafeString);
+  }
+
   @Test
   void testCreate_stringifedVKey_ofyOnlyVKeyString() {
     assertThat(VKey.create("kind:TestObject@ofy:agR0ZXN0chMLEgpUZXN0T2JqZWN0IgNmb28M"))

core/src/test/java/google/registry/tools/GenerateEscrowDepositCommandTest.java

@@ -14,18 +14,16 @@
 
 package google.registry.tools;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.when;
 
 import com.beust.jcommander.ParameterException;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.util.AppEngineServiceUtils;
-import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -42,14 +40,15 @@ public class GenerateEscrowDepositCommandTest
 
   @Mock AppEngineServiceUtils appEngineServiceUtils;
 
+  CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
+
   @BeforeEach
   void beforeEach() {
     createTld("tld");
     createTld("anothertld");
     command = new GenerateEscrowDepositCommand();
     command.appEngineServiceUtils = appEngineServiceUtils;
-    command.queue = getQueue("rde-report");
-    command.maybeEtaMillis = Optional.of(fakeClock.nowUtc().getMillis());
+    command.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
     when(appEngineServiceUtils.getCurrentVersionHostname("backend"))
         .thenReturn("backend.test.localhost");
   }
@@ -197,11 +196,10 @@ public class GenerateEscrowDepositCommandTest
         "-r 42",
         "-o test");
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "rde-report",
         new TaskMatcher()
             .url("/_dr/task/rdeStaging")
-            .header("Host", "backend.test.localhost")
             .param("mode", "THIN")
             .param("lenient", "true")
             .param("watermarks", "2017-01-01T00:00:00.000Z")
@@ -221,11 +219,10 @@ public class GenerateEscrowDepositCommandTest
         "-r 42",
         "-o test");
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "rde-report",
         new TaskMatcher()
             .url("/_dr/task/rdeStaging")
-            .header("Host", "backend.test.localhost")
             .param("mode", "THIN")
             .param("beam", "true")
             .param("watermarks", "2017-01-01T00:00:00.000Z")
@@ -239,11 +236,10 @@ public class GenerateEscrowDepositCommandTest
   void testCommand_successWithDefaultValidationMode() throws Exception {
     runCommand("--tld=tld", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-r 42", "-o test");
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "rde-report",
         new TaskMatcher()
             .url("/_dr/task/rdeStaging")
-            .header("Host", "backend.test.localhost")
             .param("mode", "THIN")
             .param("lenient", "false")
             .param("watermarks", "2017-01-01T00:00:00.000Z")
@@ -257,11 +253,10 @@ public class GenerateEscrowDepositCommandTest
   void testCommand_successWithDefaultRevision() throws Exception {
     runCommand("--tld=tld", "--watermark=2017-01-01T00:00:00Z", "--mode=thin", "-o test");
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "rde-report",
         new TaskMatcher()
             .url("/_dr/task/rdeStaging")
-            .header("Host", "backend.test.localhost")
             .param("lenient", "false")
             .param("beam", "false")
             .param("mode", "THIN")
@@ -275,11 +270,10 @@ public class GenerateEscrowDepositCommandTest
   void testCommand_successWithDefaultMode() throws Exception {
     runCommand("--tld=tld", "--watermark=2017-01-01T00:00:00Z", "-r=42", "-o test");
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "rde-report",
         new TaskMatcher()
             .url("/_dr/task/rdeStaging")
-            .header("Host", "backend.test.localhost")
             .param("mode", "FULL")
             .param("lenient", "false")
             .param("beam", "false")
@@ -299,11 +293,10 @@ public class GenerateEscrowDepositCommandTest
         "-r 42",
         "-o test");
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "rde-report",
         new TaskMatcher()
             .url("/_dr/task/rdeStaging")
-            .header("Host", "backend.test.localhost")
             .param("mode", "THIN")
             .param("lenient", "false")
             .param("beam", "false")

core/src/test/java/google/registry/tools/GetContactCommandTest.java

@@ -42,7 +42,11 @@ class GetContactCommandTest extends CommandTestCase<GetContactCommand> {
     persistActiveContact("sh8013");
     runCommand("sh8013");
     assertInStdout("contactId=sh8013");
-    assertInStdout("Websafe key: agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjItUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:ContactResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjItUk9JRAw");
   }
 
   @Test
@@ -50,7 +54,11 @@ class GetContactCommandTest extends CommandTestCase<GetContactCommand> {
     persistActiveContact("sh8013");
     runCommand("sh8013", "--expand");
     assertInStdout("contactId=sh8013");
-    assertInStdout("Websafe key: agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjItUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:ContactResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjItUk9JRAw");
     assertNotInStdout("LiveRef");
   }
 
@@ -61,8 +69,16 @@ class GetContactCommandTest extends CommandTestCase<GetContactCommand> {
     runCommand("sh8013", "jd1234");
     assertInStdout("contactId=sh8013");
     assertInStdout("contactId=jd1234");
-    assertInStdout("Websafe key: agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjItUk9JRAw");
-    assertInStdout("Websafe key: agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjMtUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:ContactResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjItUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:ContactResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chsLEg9Db250YWN0UmVzb3VyY2UiBjItUk9JRAw");
   }
 
   @Test

core/src/test/java/google/registry/tools/GetDomainCommandTest.java

@@ -43,7 +43,11 @@ class GetDomainCommandTest extends CommandTestCase<GetDomainCommand> {
     runCommand("example.tld");
     assertInStdout("fullyQualifiedDomainName=example.tld");
     assertInStdout("contact=Key<?>(ContactResource(\"3-ROID\"))");
-    assertInStdout("Websafe key: agR0ZXN0chULEgpEb21haW5CYXNlIgUyLVRMRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:DomainBase"
+            + "@sql:rO0ABXQABTItVExE"
+            + "@ofy:agR0ZXN0chULEgpEb21haW5CYXNlIgUyLVRMRAw");
   }
 
   @Test
@@ -52,7 +56,11 @@ class GetDomainCommandTest extends CommandTestCase<GetDomainCommand> {
     runCommand("example.tld", "--expand");
     assertInStdout("fullyQualifiedDomainName=example.tld");
     assertInStdout("contactId=contact1234");
-    assertInStdout("Websafe key: agR0ZXN0chULEgpEb21haW5CYXNlIgUyLVRMRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:DomainBase"
+            + "@sql:rO0ABXQABTItVExE"
+            + "@ofy:agR0ZXN0chULEgpEb21haW5CYXNlIgUyLVRMRAw");
     assertNotInStdout("LiveRef");
   }
 
@@ -63,8 +71,16 @@ class GetDomainCommandTest extends CommandTestCase<GetDomainCommand> {
     runCommand("example.tld", "example2.tld");
     assertInStdout("fullyQualifiedDomainName=example.tld");
     assertInStdout("fullyQualifiedDomainName=example2.tld");
-    assertInStdout("Websafe key: agR0ZXN0chULEgpEb21haW5CYXNlIgUyLVRMRAw");
-    assertInStdout("Websafe key: agR0ZXN0chULEgpEb21haW5CYXNlIgU0LVRMRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:DomainBase"
+            + "@sql:rO0ABXQABTQtVExE"
+            + "@ofy:agR0ZXN0chULEgpEb21haW5CYXNlIgU0LVRMRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:DomainBase"
+            + "@sql:rO0ABXQABTQtVExE"
+            + "@ofy:agR0ZXN0chULEgpEb21haW5CYXNlIgU0LVRMRAw");
   }
 
   @Test

core/src/test/java/google/registry/tools/GetHostCommandTest.java

@@ -42,7 +42,11 @@ class GetHostCommandTest extends CommandTestCase<GetHostCommand> {
     persistActiveHost("ns1.example.tld");
     runCommand("ns1.example.tld");
     assertInStdout("fullyQualifiedHostName=ns1.example.tld");
-    assertInStdout("Websafe key: agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjItUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:HostResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjItUk9JRAw");
   }
 
   @Test
@@ -50,7 +54,11 @@ class GetHostCommandTest extends CommandTestCase<GetHostCommand> {
     persistActiveHost("ns1.example.tld");
     runCommand("ns1.example.tld", "--expand");
     assertInStdout("fullyQualifiedHostName=ns1.example.tld");
-    assertInStdout("Websafe key: agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjItUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:HostResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjItUk9JRAw");
     assertNotInStdout("LiveRef");
   }
 
@@ -61,8 +69,16 @@ class GetHostCommandTest extends CommandTestCase<GetHostCommand> {
     runCommand("ns1.example.tld", "ns2.example.tld");
     assertInStdout("fullyQualifiedHostName=ns1.example.tld");
     assertInStdout("fullyQualifiedHostName=ns2.example.tld");
-    assertInStdout("Websafe key: agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjItUk9JRAw");
-    assertInStdout("Websafe key: agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjMtUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:HostResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjItUk9JRAw");
+    assertInStdout(
+        "Websafe key: "
+            + "kind:HostResource"
+            + "@sql:rO0ABXQABjItUk9JRA"
+            + "@ofy:agR0ZXN0chgLEgxIb3N0UmVzb3VyY2UiBjItUk9JRAw");
   }
 
   @Test

core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java

@@ -52,6 +52,26 @@ class ResaveEntitiesCommandTest extends CommandTestCase<ResaveEntitiesCommand> {
         .containsExactlyElementsIn(auditedOfy().load().entities(contact1, contact2).values());
   }
 
+  @Test
+  void testSuccess_createsCommitLogs_withNewWebsafeKey() throws Exception {
+    ContactResource contact1 = persistActiveContact("contact1");
+    ContactResource contact2 = persistActiveContact("contact2");
+    deleteEntitiesOfTypes(CommitLogManifest.class, CommitLogMutation.class);
+    assertThat(auditedOfy().load().type(CommitLogManifest.class).keys()).isEmpty();
+    assertThat(auditedOfy().load().type(CommitLogMutation.class).keys()).isEmpty();
+    runCommandForced(contact1.createVKey().stringify(), contact2.createVKey().stringify());
+
+    assertThat(auditedOfy().load().type(CommitLogManifest.class).keys()).hasSize(1);
+    Iterable<ImmutableObject> savedEntities =
+        transform(
+            auditedOfy().load().type(CommitLogMutation.class).list(),
+            mutation -> auditedOfy().load().fromEntity(mutation.getEntity()));
+    // Reload the contacts before asserting, since their update times will have changed.
+    auditedOfy().clearSessionCache();
+    assertThat(savedEntities)
+        .containsExactlyElementsIn(auditedOfy().load().entities(contact1, contact2).values());
+  }
+
   @SafeVarargs
   private static void deleteEntitiesOfTypes(Class<? extends ImmutableObject>... types) {
     for (Class<? extends ImmutableObject> type : types) {

release/cloudbuild-delete.yaml

@@ -19,11 +19,21 @@
 # expanded in the copies sent to Spinnaker, we preserve the brackets around
 # them for safe pattern matching during release.
 # See https://github.com/spinnaker/spinnaker/issues/3028 for more information.
-steps:
-# Delete unused GAE versions.
+#
 # GAE has a limit of ~250 versions per-project, including unused versions. We
 # therefore need to periodically delete old versions. This GCB job finds all
 # stopped versions and delete all but the last 3 (in case we need to rollback).
+steps:
+# Pull the credential for nomulus tool.
+- name: 'gcr.io/$PROJECT_ID/builder:latest'
+  entrypoint: /bin/bash
+  args:
+  - -c
+  - |
+    set -e
+    gcloud secrets versions access latest \
+      --secret nomulus-tool-cloudbuild-credential > tool-credential.json
+# Delete unused GAE versions.
 - name: 'gcr.io/$PROJECT_ID/builder:latest'
   entrypoint: /bin/bash
   args:
@@ -36,6 +46,8 @@ steps:
       project_id="domain-registry-${_ENV}"
     fi
 
+    gcloud auth activate-service-account --key-file=tool-credential.json
+
     for service in default pubapi backend tools
     do
       for version in $(gcloud app versions list \