Copy DTRs into PersistentSet field if applicable in HistoryEntry (#1456)

In replay (and possibly in other cases) we're getting an exception:

Caused by: org.hibernate.HibernateException: A collection with cascade="all-delete-orphan" was no longer referenced by the owning entity instance: google.registry.model.domain.DomainHistory.internalDomainTransactionRecords

The main cause of this, according to research (StackOverflow :P) is that
when Hibernate is calling the setters for these sets of children it's
losing the connection to the previously-managed child entity (which it
needs, in order to know how to delete orphans). Thus, the solution is to
maintain the same instance of the persistent set and just add/remove
to/from it as necessary.

This is complicated by the fact that sometimes the setter is given the
persistent set (the one we want to keep) and sometimes (?) it isn't. We
will need to try this out to be sure.

build.gradle

@@ -46,7 +46,7 @@ plugins {
   id 'com.diffplug.gradle.spotless' version '3.25.0'
 
   id 'jacoco'
-  id 'com.dorongold.task-tree' version '1.5'
+  id 'com.dorongold.task-tree' version '2.1.0'
 }
 
 node {
@@ -196,9 +196,41 @@ allprojects {
   }
 }
 
+rootProject.ext {
+  pyver = { exe ->
+    try {
+      ext.execInBash(
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
+    } catch (org.gradle.process.internal.ExecException e) {
+      return -1;
+    }
+  }
+
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
+    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
+    // Debian so it seems like that should be available anywhere).
+    def MIN_PY_VER = 0x3070300
+    if (pyver('python') >= MIN_PY_VER) {
+      return 'python'
+    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
+      return '/usr/bin/python3'
+    } else {
+      throw new GradleException("No usable Python version found (build " +
+                                "requires at least python 3.7.3)");
+    }
+  }
+}
+
 task runPresubmits(type: Exec) {
-  executable '/usr/bin/python3'
+
   args('config/presubmits.py')
+
+  doFirst {
+    executable getPythonExecutable()
+  }
 }
 
 def javadocSource = []
@@ -412,9 +444,10 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
@@ -422,18 +455,23 @@ rootProject.ext {
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
-    def checkResult = invokeJavaDiffFormatScript("check")
-    if (checkResult == 'true') {
-      throw new IllegalStateException(
-          "Some Java files need to be reformatted. You may use the "
-              + "'javaIncrementalFormatDryRun' task to review\n "
-              + "the changes, or the 'javaIncrementalFormatApply' task "
-              + "to reformat.")
-    } else if (checkResult != 'false') {
-      throw new RuntimeException(
-          "Failed to invoke format check script:\n" + checkResult)
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
+      def checkResult = invokeJavaDiffFormatScript("check")
+      if (checkResult == 'true') {
+        throw new IllegalStateException(
+            "Some Java files need to be reformatted. You may use the "
+                + "'javaIncrementalFormatDryRun' task to review\n "
+                + "the changes, or the 'javaIncrementalFormatApply' task "
+                + "to reformat.")
+      } else if (checkResult != 'false') {
+        throw new RuntimeException(
+            "Failed to invoke format check script:\n" + checkResult)
+      }
+      println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
     }
-    println("Incremental Java format check ok.")
   }
 }
 
@@ -457,15 +495,14 @@ task javaIncrementalFormatApply {
 task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
-  // Exclude the misbehaving generated-by-Soy Java files
-  exclude "**/*SoyInfo.java"
   destinationDir = file("${buildDir}/docs/javadoc")
   options.encoding = "UTF-8"
   // In a lot of places we don't write @return so suppress warnings about that.
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
   options.tags = ["type:a:Generic Type",
-                  "error:a:Expected Error"]
+                  "error:a:Expected Error",
+                  "invariant:a:Guaranteed Property"]
 }
 
 tasks.build.dependsOn(tasks.javadoc)
@@ -483,3 +520,15 @@ task coreDev {
 }
 
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
+
+// disable javadoc in subprojects, these will break because they don't have
+// the correct classpath (see above).
+gradle.taskGraph.whenReady { graph ->
+  graph.getAllTasks().each { task ->
+    def subprojectJavadoc = (task.path =~ /:.+:javadoc/)
+    if (subprojectJavadoc) {
+        println "Skipping ${task.path} for javadoc (only root javadoc works)"
+        task.enabled = false
+    }
+  }
+}

common/src/testing/java/google/registry/testing/SystemInfo.java

@@ -39,7 +39,7 @@ public final class SystemInfo {
                     pid.getOutputStream().close();
                     pid.waitFor();
                   } catch (IOException e) {
-                    logger.atWarning().withCause(e).log("%s command not available", cmd);
+                    logger.atWarning().withCause(e).log("%s command not available.", cmd);
                     return false;
                   }
                   return true;

config/nom_build.py

@@ -140,6 +140,8 @@ PROPERTIES = [
              'a BEAM pipeline to image. Setting this property to empty string '
              'will disable image generation.',
              '/usr/bin/dot'),
+    Property('pipeline',
+             'The name of the Beam pipeline being staged.')
 ]
 
 GRADLE_FLAGS = [

config/presubmits.py

@@ -17,9 +17,11 @@ These aren't built in to the static code analysis tools we use (e.g. Checkstyle,
 Error Prone) so we must write them manually.
 """
 
+import json
 import os
 from typing import List, Tuple
 import sys
+import textwrap
 import re
 
 # We should never analyze any generated files
@@ -28,6 +30,13 @@ UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/"
 FORBIDDEN = 1
 REQUIRED = 2
 
+# The list of expected json packages and their licenses.
+# These should be one of the allowed licenses in:
+# config/dependency-license/allowed_licenses.json
+EXPECTED_JS_PACKAGES = [
+    'google-closure-library',   # Owned by Google, Apache 2.0
+]
+
 
 class PresubmitCheck:
 
@@ -110,9 +119,10 @@ PRESUBMITS = {
       "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
+    # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
-        "java", {}):
+        "java", {"DatabaseSnapshotTest.java"}):
       "PostgreSQLContainer instantiation must specify docker tag.",
 
     # Various Soy linting checks
@@ -177,52 +187,6 @@ PRESUBMITS = {
         {"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
     ):
         "JavaScript files should not include console logging.",
-    # SQL injection protection rule for java source file:
-    # The sql template passed to createQuery/createNativeQuery methods must be
-    # a variable name in UPPER_CASE_UNDERSCORE format, i.e., a static final
-    # String variable. This forces the use of parameter-binding on all queries
-    # that take parameters.
-    # The rule would forbid invocation of createQuery(Criteria). However, this
-    # can be handled by adding a helper method in an exempted class to make
-    # the calls.
-    # TODO(b/179158393): enable the 'ConstantName' Java style check to ensure
-    # that non-final variables do not use the UPPER_CASE_UNDERSCORE format.
-    PresubmitCheck(
-        # Line 1: the method names we check and the opening parenthesis, which
-        #    marks the beginning of the first parameter
-        # Line 2: The first parameter is a match if is NOT any of the following:
-        #    - final variable name: \s*([A-Z_]+
-        #    - string literal: "([^"]|\\")*"
-        #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
-        # Line 3: , or the closing parenthesis, marking the end of the first
-        #    parameter
-        r'.*\.(query|createQuery|createNativeQuery)\('
-        r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
-        r'(,|\s*\))))',
-        "java",
-        # ActivityReportingQueryBuilder deals with Dremel queries
-        {"src/test", "ActivityReportingQueryBuilder.java",
-         # This class contains helper method to make queries in Beam.
-         "RegistryJpaIO.java",
-         # TODO(b/179158393): Remove everything below, which should be done
-         # using Criteria
-         "ForeignKeyIndex.java",
-         "HistoryEntryDao.java",
-         "JpaTransactionManager.java",
-         "JpaTransactionManagerImpl.java",
-         # CriteriaQueryBuilder is a false positive
-         "CriteriaQueryBuilder.java",
-         "RdapDomainSearchAction.java",
-         "RdapNameserverSearchAction.java",
-         "RdapSearchActionBase.java",
-         "RegistryQuery",
-         },
-    ):
-        "The first String parameter to EntityManager.create(Native)Query "
-        "methods must be one of the following:\n"
-        "  - A String literal\n"
-        "  - Concatenation of String literals only\n"
-        "  - The name of a static final String variable"
 }
 
 # Note that this regex only works for one kind of Flyway file.  If we want to
@@ -310,6 +274,26 @@ def verify_flyway_index():
   return not success
 
 
+def verify_javascript_deps():
+  """Verifies that we haven't introduced any new javascript dependencies."""
+  with open('package.json') as f:
+    package = json.load(f)
+
+  deps = list(package['dependencies'].keys())
+  if deps != EXPECTED_JS_PACKAGES:
+    print('Unexpected javascript dependencies.  Was expecting '
+          '%s, got %s.' % (EXPECTED_JS_PACKAGES, deps))
+    print(textwrap.dedent("""
+        * If the new dependencies are intentional, please verify that the
+        * license is one of the allowed licenses (see
+        * config/dependency-license/allowed_licenses.json) and add an entry
+        * for the package (with the license in a comment) to the
+        * EXPECTED_JS_PACKAGES variable in config/presubmits.py.
+        """))
+    return True
+  return False
+
+
 def get_files():
   for root, dirnames, filenames in os.walk("."):
     for filename in filenames:
@@ -317,6 +301,7 @@ def get_files():
 
 
 if __name__ == "__main__":
+  print('python version is %s' % sys.version)
   failed = False
   for file in get_files():
     error_messages = []
@@ -333,5 +318,8 @@ if __name__ == "__main__":
   # when we put it here it fails fast before all of the tests are run.
   failed |= verify_flyway_index()
 
+  # Make sure we haven't introduced any javascript dependencies.
+  failed |= verify_javascript_deps()
+
   if failed:
     sys.exit(1)

core/build.gradle

@@ -44,7 +44,6 @@ def outcastTestPatterns = [
     "google/registry/flows/domain/DomainCreateFlowTest.*",
     "google/registry/flows/domain/DomainUpdateFlowTest.*",
     "google/registry/tools/CreateDomainCommandTest.*",
-    "google/registry/tools/server/CreatePremiumListActionTest.*",
 ]
 
 // Tests that fail when running Gradle in a docker container, e. g. when
@@ -70,8 +69,6 @@ def dockerIncompatibleTestPatterns = [
 // Nomulus classes, e.g., threads and objects retained by frameworks.
 // TODO(weiminyu): identify cause and fix offending tests.
 def fragileTestPatterns = [
-    // Problem seems to lie with AppEngine TaskQueue for test.
-    "google/registry/cron/TldFanoutActionTest.*",
     // Test Datastore inexplicably aborts transaction.
     "google/registry/model/tmch/ClaimsListShardTest.*",
     // Changes cache timeouts and for some reason appears to have contention
@@ -185,6 +182,7 @@ dependencies {
   compile deps['com.google.monitoring-client:metrics']
   compile deps['com.google.monitoring-client:stackdriver']
   compile deps['com.google.api-client:google-api-client-java6']
+  compile deps['com.google.api.grpc:proto-google-cloud-tasks-v2']
   compile deps['com.google.apis:google-api-services-admin-directory']
   compile deps['com.google.apis:google-api-services-appengine']
   compile deps['com.google.apis:google-api-services-bigquery']
@@ -217,11 +215,13 @@ dependencies {
   compile deps['com.google.flogger:flogger']
   runtime deps['com.google.flogger:flogger-system-backend']
   compile deps['com.google.guava:guava']
+  compile deps['com.google.protobuf:protobuf-java']
   gradleLint.ignore('unused-dependency') {
     compile deps['com.google.gwt:gwt-user']
   }
   compile deps['com.google.cloud:google-cloud-core']
   compile deps['com.google.cloud:google-cloud-storage']
+  compile deps['com.google.cloud:google-cloud-tasks']
   compile deps['com.google.http-client:google-http-client']
   compile deps['com.google.http-client:google-http-client-appengine']
   compile deps['com.google.http-client:google-http-client-jackson2']
@@ -457,6 +457,22 @@ task soyToJava {
               "--srcs", "${soyFiles.join(',')}",
               "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
     }
+
+    // Replace the "@link" tags after the "@deprecated" tags in the generated
+    // files.  The soy compiler doesn't generate imports for these, causing
+    // us to get warnings when we generate javadocs.
+    // TODO(b/200296387): To be fair, the deprecations are accurate: we're
+    // using the old "SoyInfo" classes instead of the new "Templates" files.
+    // When we convert to the new classes, this hack can go away.
+    def outputs = fileTree(outputDirectory) {
+      include '**/*.java'
+    }
+
+    outputs.each { file ->
+      exec {
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
+      }
+    }
   }
 
   doLast {
@@ -687,33 +703,19 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
+createToolTask(
+    'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
 
 createToolTask(
-        'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
 
-project.tasks.create('initSqlPipeline', JavaExec) {
-  main = 'google.registry.beam.initsql.InitSqlPipeline'
 
-  doFirst {
-    getToolArgsList().ifPresent {
-      args it
-    }
+createToolTask(
+    'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
 
-    def isDirectRunner =
-        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
-    // The dependency containing DirectRunner is intentionally excluded from the
-    // production binary, so that it won't be chosen by mistake: we definitely do
-    // not want to use it for the real jobs, yet DirectRunner is the default if
-    // the user forgets to override it.
-    // DirectRunner is required for tests and is already on testRuntimeClasspath.
-    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
-    // of defining a new configuration just for the DirectRunner dependency.
-    classpath =
-        isDirectRunner
-            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
-            : sourceSets.main.runtimeClasspath
-  }
-}
+createToolTask(
+    'createSyntheticHistoryEntries',
+    'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
 // Caller must provide projectId, GCP region, runner, and the kinds to delete
 // (comma-separated kind names or '*' for all). E.g.:
@@ -756,50 +758,57 @@ createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
 // This packages more code and dependency than necessary. However, without
 // restructuring the source tree it is difficult to generate leaner jars.
 createUberJar(
-    'beam_pipeline_common',
+    'beamPipelineCommon',
     'beam_pipeline_common',
     '')
 
-// Create beam staging task if environment is alpha or crash.
-// All other environments use formally released pipelines through CloudBuild.
+// Create beam staging task if the environment is alpha. Production, sandbox and
+// qa use formally released pipelines through CloudBuild, whereas crash and
+// alpha use the pipelines staged on alpha deployment project.
 //
 // User should install gcloud and login to GCP before invoking this tasks.
-if (environment in ['alpha', 'crash']) {
+if (environment == 'alpha') {
   def pipelines = [
-      [
-          mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
-          metaData: 'google/registry/beam/init_sql_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
-          metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
-          metaData: 'google/registry/beam/spec11_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
-          metaData: 'google/registry/beam/invoicing_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.rde.RdePipeline',
-          metaData: 'google/registry/beam/rde_pipeline_metadata.json'
-      ],
+      initSql            :
+          [
+              mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
+              metaData : 'google/registry/beam/init_sql_pipeline_metadata.json'
+          ],
+      bulkDeleteDatastore:
+          [
+              mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
+              metaData : 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
+          ],
+      spec11             :
+          [
+              mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
+              metaData : 'google/registry/beam/spec11_pipeline_metadata.json'
+          ],
+      invoicing          :
+          [
+              mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
+              metaData : 'google/registry/beam/invoicing_pipeline_metadata.json'
+          ],
+      rde                :
+          [
+              mainClass: 'google.registry.beam.rde.RdePipeline',
+              metaData : 'google/registry/beam/rde_pipeline_metadata.json'
+          ],
   ]
-  project.tasks.create("stage_beam_pipelines") {
+  project.tasks.create("stageBeamPipelines") {
     doLast {
       pipelines.each {
-        def mainClass = it['mainClass']
-        def metaData = it['metaData']
-        def pipelineName = CaseFormat.UPPER_CAMEL.to(
-                CaseFormat.LOWER_UNDERSCORE,
-                mainClass.substring(mainClass.lastIndexOf('.') + 1))
-        def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
-        def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
-        def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath
+        if (rootProject.pipeline == ''|| rootProject.pipeline == it.key) {
+          def mainClass = it.value['mainClass']
+          def metaData = it.value['metaData']
+          def pipelineName = CaseFormat.UPPER_CAMEL.to(
+              CaseFormat.LOWER_UNDERSCORE,
+              mainClass.substring(mainClass.lastIndexOf('.') + 1))
+          def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
+          def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
+          def uberJarName = tasks.beamPipelineCommon.outputs.files.asPath
 
-        def command = "\
+          def command = "\
             gcloud dataflow flex-template build \
             gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
             --image-gcr-path ${imageName}:live \
@@ -809,10 +818,11 @@ if (environment in ['alpha', 'crash']) {
             --jar ${uberJarName} \
             --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
             --project ${gcpProject}".toString()
-        rootProject.ext.execInBash(command, '/tmp')
+          rootProject.ext.execInBash(command, '/tmp')
+        }
       }
     }
-  }.dependsOn(tasks.beam_pipeline_common)
+  }.dependsOn(tasks.beamPipelineCommon)
 }
 
 // A jar with classes and resources from main sourceSet, excluding internal
@@ -1087,6 +1097,10 @@ test {
   // TODO(weiminyu): Remove dependency on sqlIntegrationTest
 }.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
 
+// When we override tests, we also break the cleanTest command.
+cleanTest.dependsOn(cleanFragileTest, cleanOutcastTest, cleanStandardTest,
+                    cleanRegistryToolIntegrationTest, cleanSqlIntegrationTest)
+
 project.build.dependsOn devtool
 project.build.dependsOn buildToolImage
 project.build.dependsOn ':stage'

core/gradle/dependency-locks/compile.lockfile

@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -83,10 +86,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -104,11 +107,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -119,10 +123,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -134,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -151,17 +155,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -259,7 +263,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -82,10 +85,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -103,11 +106,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
@@ -117,10 +121,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -133,7 +137,7 @@ com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -149,17 +153,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -252,7 +256,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4

core/gradle/dependency-locks/default.lockfile

@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -87,10 +90,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -109,11 +112,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -124,10 +128,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -159,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -273,7 +277,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -87,10 +90,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -109,11 +112,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -124,10 +128,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -159,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -272,7 +276,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -83,10 +86,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -104,11 +107,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -119,10 +123,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -134,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -151,17 +155,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -259,7 +263,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -82,10 +85,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -103,11 +106,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
@@ -117,10 +121,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -133,7 +137,7 @@ com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -149,17 +153,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -253,7 +257,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -87,10 +90,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -108,11 +111,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -123,10 +127,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -138,8 +142,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -158,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -271,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -87,10 +90,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -108,11 +111,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -123,10 +127,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -138,8 +142,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -158,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -271,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3

core/gradle/dependency-locks/runtime.lockfile

@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -87,10 +90,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -108,11 +111,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -123,10 +127,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -138,8 +142,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -158,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -271,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -87,10 +90,10 @@ com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -109,11 +112,12 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
 com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
@@ -124,10 +128,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,8 +143,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -159,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -272,7 +276,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3

core/gradle/dependency-locks/testCompile.lockfile

@@ -54,10 +54,13 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
 com.google.api.grpc:proto-google-common-protos:2.3.2
 com.google.api.grpc:proto-google-iam-v1:1.0.14
 com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.83.0
 com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
@@ -106,6 +109,7 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
@@ -160,17 +164,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
 io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -53,10 +53,13 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
 com.google.api.grpc:proto-google-common-protos:2.3.2
 com.google.api.grpc:proto-google-iam-v1:1.0.14
 com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.83.0
 com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
@@ -105,6 +108,7 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
@@ -158,17 +162,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
 io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final

core/gradle/dependency-locks/testRuntime.lockfile

@@ -58,10 +58,13 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
 com.google.api.grpc:proto-google-common-protos:2.3.2
 com.google.api.grpc:proto-google-iam-v1:1.0.14
 com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.83.0
 com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
@@ -111,6 +114,7 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
@@ -169,17 +173,17 @@ guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
 io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -58,10 +58,13 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
 com.google.api.grpc:proto-google-common-protos:2.3.2
 com.google.api.grpc:proto-google-iam-v1:1.0.14
 com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
+com.google.api:gax-grpc:1.66.0
 com.google.api:gax-httpjson:0.83.0
 com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
@@ -111,6 +114,7 @@ com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.7
 com.google.common.html.types:types:1.0.6
@@ -169,17 +173,17 @@ guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
 io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -41,11 +41,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnlyWithoutBackup().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -14,21 +14,21 @@
 
 package google.registry.backup;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -56,7 +56,8 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Inject Clock clock;
   @Inject CommitLogCheckpointStrategy strategy;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject CommitLogCheckpointAction() {}
 
   @Override
@@ -73,16 +74,20 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveWithoutBackup()
+                  .saveIgnoringReadOnlyWithoutBackup()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.
-              taskQueueUtils.enqueue(
-                  getQueue(QUEUE_NAME),
-                  withUrl(ExportCommitLogDiffAction.PATH)
-                      .param(LOWER_CHECKPOINT_TIME_PARAM, lastWrittenTime.toString())
-                      .param(
-                          UPPER_CHECKPOINT_TIME_PARAM, checkpoint.getCheckpointTime().toString()));
+              cloudTasksUtils.enqueue(
+                  QUEUE_NAME,
+                  CloudTasksUtils.createPostTask(
+                      ExportCommitLogDiffAction.PATH,
+                      Service.BACKEND.toString(),
+                      ImmutableMultimap.of(
+                          LOWER_CHECKPOINT_TIME_PARAM,
+                          lastWrittenTime.toString(),
+                          UPPER_CHECKPOINT_TIME_PARAM,
+                          checkpoint.getCheckpointTime().toString())));
             });
   }
 }

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -55,10 +55,9 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+  static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
       InputStream inputStream) {
-    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
-        InputStream input = new BufferedInputStream(inputStream)) {
+    try (InputStream input = new BufferedInputStream(inputStream)) {
       Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
       checkState(commitLogs.hasNext());
       checkState(commitLogs.next() instanceof CommitLogCheckpoint);
@@ -105,7 +104,7 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+  static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
     return loadEntitiesByTransaction(inputStream).stream()
         .flatMap(ImmutableList::stream)
         .collect(toImmutableList());

core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java

@@ -93,7 +93,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
   public void run() {
     DateTime deletionThreshold = clock.nowUtc().minus(maxAge);
     logger.atInfo().log(
-        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s",
+        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s.",
         deletionThreshold);
 
     mrRunner
@@ -208,7 +208,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       getContext().incrementCounter("EPP resources missing pre-threshold revision (SEE LOGS)");
       logger.atSevere().log(
           "EPP resource missing old enough revision: "
-              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s",
+              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s.",
           Key.create(eppResource),
           eppResource.getCreationTime(),
           eppResource.getRevisions().size(),

core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java

@@ -100,7 +100,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
 
     // Load the keys of all the manifests to include in this diff.
     List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
-    logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
+    logger.atInfo().log("Found %d manifests to export.", sortedKeys.size());
     // Open an output channel to GCS, wrapped in a stream for convenience.
     try (OutputStream gcsStream =
         gcsUtils.openOutputStream(
@@ -124,7 +124,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
       for (int i = 0; i < keyChunks.size(); i++) {
         // Force the async load to finish.
         Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
-        logger.atInfo().log("Loaded %d manifests", chunkValues.size());
+        logger.atInfo().log("Loaded %d manifests.", chunkValues.size());
         // Since there is no hard bound on how much data this might be, take care not to let the
         // Objectify session cache fill up and potentially run out of memory. This is the only safe
         // point to do this since at this point there is no async load in progress.
@@ -134,12 +134,12 @@ public final class ExportCommitLogDiffAction implements Runnable {
           nextChunkToExport = auditedOfy().load().keys(keyChunks.get(i + 1));
         }
         exportChunk(gcsStream, chunkValues);
-        logger.atInfo().log("Exported %d manifests", chunkValues.size());
+        logger.atInfo().log("Exported %d manifests.", chunkValues.size());
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    logger.atInfo().log("Exported %d manifests in total", sortedKeys.size());
+    logger.atInfo().log("Exported %d total manifests.", sortedKeys.size());
   }
 
   /**

core/src/main/java/google/registry/backup/GcsDiffFileLister.java

@@ -94,14 +94,14 @@ class GcsDiffFileLister {
           logger.atInfo().log(
               "Gap discovered in sequence terminating at %s, missing file: %s",
               sequence.lastKey(), filename);
-          logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+          logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
           return false;
         }
       }
       sequence.put(checkpointTime, blobInfo);
       checkpointTime = getLowerBoundTime(blobInfo);
     }
-    logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+    logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
     return true;
   }
 
@@ -140,7 +140,7 @@ class GcsDiffFileLister {
         }
       }
       if (upperBoundTimesToBlobInfo.isEmpty()) {
-        logger.atInfo().log("No files found");
+        logger.atInfo().log("No files found.");
         return ImmutableList.of();
       }
 
@@ -185,7 +185,7 @@ class GcsDiffFileLister {
 
     logger.atInfo().log(
         "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
-    logger.atInfo().log("Found %d files to restore", sequence.size());
+    logger.atInfo().log("Found %d files to restore.", sequence.size());
     return ImmutableList.copyOf(sequence.values());
   }
 

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
+import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
 import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
@@ -111,7 +112,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
@@ -139,7 +140,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 
@@ -222,8 +223,11 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       // Load and process the Datastore transactions one at a time
       ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
           CommitLogImports.loadEntitiesByTransaction(input);
-      allTransactions.forEach(
-          transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
+      try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+          UpdateAutoTimestamp.disableAutoUpdate()) {
+        allTransactions.forEach(
+            transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
+      }
       // if we succeeded, set the last-seen time
       DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
       jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
@@ -260,7 +264,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             .ifPresent(
                 sqlEntity -> {
                   sqlEntity.beforeSqlSaveOnReplay();
-                  jpaTm().put(sqlEntity);
+                  jpaTm().putIgnoringReadOnlyWithoutBackup(sqlEntity);
                 });
       } else {
         // this should never happen, but we shouldn't fail on it
@@ -269,7 +273,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             ofyPojo.getClass());
       }
     } catch (Throwable t) {
-      logger.atSevere().log("Error when replaying object %s", ofyPojo);
+      logger.atSevere().log("Error when replaying object %s.", ofyPojo);
       throw t;
     }
   }
@@ -293,10 +297,10 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
           && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
           && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
         ReplaySpecializer.beforeSqlDelete(entityVKey);
-        jpaTm().delete(entityVKey);
+        jpaTm().deleteIgnoringReadOnlyWithoutBackup(entityVKey);
       }
     } catch (Throwable t) {
-      logger.atSevere().log("Error when deleting key %s", entityVKey);
+      logger.atSevere().log("Error when deleting key %s.", entityVKey);
       throw t;
     }
   }

core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java

@@ -103,13 +103,13 @@ public class RestoreCommitLogsAction implements Runnable {
         !FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
         "DO NOT RUN IN PRODUCTION OR SANDBOX.");
     if (dryRun) {
-      logger.atInfo().log("Running in dryRun mode");
+      logger.atInfo().log("Running in dry-run mode.");
     }
     String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
     logger.atInfo().log("Restoring from %s.", gcsBucket);
     List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
     if (diffFiles.isEmpty()) {
-      logger.atInfo().log("Nothing to restore");
+      logger.atInfo().log("Nothing to restore.");
       return;
     }
     Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
@@ -143,7 +143,7 @@ public class RestoreCommitLogsAction implements Runnable {
                                 .build()),
                 Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
             .collect(toImmutableList()));
-    logger.atInfo().log("Restore complete");
+    logger.atInfo().log("Restore complete.");
   }
 
   /**

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -105,29 +105,29 @@ public abstract class VersionedEntity implements Serializable {
    * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
    * information.
    */
-  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+  static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
     long commitTimeMillis = manifest.getCommitTime().getMillis();
     return manifest.getDeletions().stream()
         .map(com.googlecode.objectify.Key::getRaw)
-        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+        .map(key -> newBuilder().commitTimeMills(commitTimeMillis).key(key).build());
   }
 
   /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
-  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+  static VersionedEntity fromMutation(CommitLogMutation mutation) {
     return from(
         com.googlecode.objectify.Key.create(mutation).getParent().getId(),
         mutation.getEntityProtoBytes());
   }
 
   public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
-    return builder()
+    return newBuilder()
         .entityProtoBytes(entityProtoBytes)
         .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
         .commitTimeMills(commitTimeMillis)
         .build();
   }
 
-  static Builder builder() {
+  private static Builder newBuilder() {
     return new AutoValue_VersionedEntity.Builder();
   }
 
@@ -142,7 +142,7 @@ public abstract class VersionedEntity implements Serializable {
 
     public abstract VersionedEntity build();
 
-    public Builder entityProtoBytes(byte[] bytes) {
+    Builder entityProtoBytes(byte[] bytes) {
       return entityProtoBytes(new ImmutableBytes(bytes));
     }
   }

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,10 +24,8 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
@@ -84,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -96,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -114,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -126,18 +122,17 @@ public final class AsyncTaskEnqueuer {
   public void enqueueAsyncDelete(
       EppResource resourceToDelete,
       DateTime now,
-      String requestingClientId,
+      String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingClientId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
-            .param(PARAM_REQUESTING_CLIENT_ID, requestingClientId)
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().getOfyKey().getString())
+            .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))
             .param(PARAM_REQUESTED_TIME, now.toString());

core/src/main/java/google/registry/batch/BatchModule.java

@@ -24,6 +24,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.request.RequestParameters.extractIntParameter;
 import static google.registry.request.RequestParameters.extractLongParameter;
 import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
+import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter;
 import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
@@ -78,6 +79,7 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
+  // TODO(b/207363014): figure out if this needs to be modified for vkey string replacement
   static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
     return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
   }
@@ -106,6 +108,13 @@ public class BatchModule {
     return extractIntParameter(req, RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM);
   }
 
+  @Provides
+  @Parameter(ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME)
+  static Optional<DateTime> provideCursorTime(HttpServletRequest req) {
+    return extractOptionalDatetimeParameter(
+        req, ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME);
+  }
+
   @Provides
   @Named(QUEUE_ASYNC_ACTIONS)
   static Queue provideAsyncActionsPushQueue() {

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -311,7 +311,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
     @Override
     public void reduce(final DeletionRequest deletionRequest, ReducerInput<Boolean> values) {
       final boolean hasNoActiveReferences = !Iterators.contains(values, true);
-      logger.atInfo().log("Processing async deletion request for %s", deletionRequest.key());
+      logger.atInfo().log("Processing async deletion request for %s.", deletionRequest.key());
       DeletionResult result =
           tm()
               .transactNew(
@@ -343,15 +343,15 @@ public class DeleteContactsAndHostsAction implements Runnable {
       }
       // Contacts and external hosts have a direct client id. For subordinate hosts it needs to be
       // read off of the superordinate domain.
-      String resourceClientId = resource.getPersistedCurrentSponsorClientId();
+      String resourceRegistrarId = resource.getPersistedCurrentSponsorRegistrarId();
       if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
-        resourceClientId =
+        resourceRegistrarId =
             tm().loadByKey(((HostResource) resource).getSuperordinateDomain())
                 .cloneProjectedAtTime(now)
-                .getCurrentSponsorClientId();
+                .getCurrentSponsorRegistrarId();
       }
       boolean requestedByCurrentOwner =
-          resourceClientId.equals(deletionRequest.requestingClientId());
+          resourceRegistrarId.equals(deletionRequest.requestingClientId());
 
       boolean deleteAllowed =
           hasNoActiveReferences && (requestedByCurrentOwner || deletionRequest.isSuperuser());
@@ -371,14 +371,14 @@ public class DeleteContactsAndHostsAction implements Runnable {
 
       HistoryEntry historyEntry =
           HistoryEntry.createBuilderForResource(resource)
-              .setClientId(deletionRequest.requestingClientId())
+              .setRegistrarId(deletionRequest.requestingClientId())
               .setModificationTime(now)
               .setType(getHistoryEntryType(resource, deleteAllowed))
               .build();
 
       PollMessage.OneTime pollMessage =
           new PollMessage.OneTime.Builder()
-              .setClientId(deletionRequest.requestingClientId())
+              .setRegistrarId(deletionRequest.requestingClientId())
               .setMsg(pollMessageText)
               .setParent(historyEntry)
               .setEventTime(now)
@@ -523,18 +523,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(
@@ -605,12 +606,12 @@ public class DeleteContactsAndHostsAction implements Runnable {
   static boolean doesResourceStateAllowDeletion(EppResource resource, DateTime now) {
     Key<EppResource> key = Key.create(resource);
     if (isDeleted(resource, now)) {
-      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted", key);
+      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted.", key);
       return false;
     }
     if (!resource.getStatusValues().contains(PENDING_DELETE)) {
       logger.atWarning().log(
-          "Cannot asynchronously delete %s because it is not in PENDING_DELETE", key);
+          "Cannot asynchronously delete %s because it is not in PENDING_DELETE.", key);
       return false;
     }
     return true;

core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java

@@ -167,7 +167,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
   /** Runs the actual domain delete flow and returns whether the deletion was successful. */
   private boolean runDomainDeleteFlow(DomainBase domain) {
-    logger.atInfo().log("Attempting to delete domain %s", domain.getDomainName());
+    logger.atInfo().log("Attempting to delete domain '%s'.", domain.getDomainName());
     // Create a new transaction that the flow's execution will be enlisted in that loads the domain
     // transactionally. This way we can ensure that nothing else has modified the domain in question
     // in the intervening period since the query above found it.
@@ -203,7 +203,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
     if (eppOutput.isPresent()) {
       if (eppOutput.get().isSuccess()) {
-        logger.atInfo().log("Successfully deleted domain %s", domain.getDomainName());
+        logger.atInfo().log("Successfully deleted domain '%s'.", domain.getDomainName());
       } else {
         logger.atSevere().log(
             "Failed to delete domain %s; EPP response:\n\n%s",

core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java

@@ -135,14 +135,14 @@ public class DeleteLoadTestDataAction implements Runnable {
   }
 
   private void deleteContact(ContactResource contact) {
-    if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorClientId())) {
+    if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorRegistrarId())) {
       return;
     }
     // We cannot remove contacts from domains in the general case, so we cannot delete contacts
     // that are linked to domains (since it would break the foreign keys)
     if (EppResourceUtils.isLinked(contact.createVKey(), clock.nowUtc())) {
       logger.atWarning().log(
-          "Cannot delete contact with repo ID %s since it is referenced from a domain",
+          "Cannot delete contact with repo ID %s since it is referenced from a domain.",
           contact.getRepoId());
       return;
     }
@@ -150,7 +150,7 @@ public class DeleteLoadTestDataAction implements Runnable {
   }
 
   private void deleteHost(HostResource host) {
-    if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorClientId())) {
+    if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorRegistrarId())) {
       return;
     }
     VKey<HostResource> hostVKey = host.createVKey();
@@ -177,7 +177,7 @@ public class DeleteLoadTestDataAction implements Runnable {
         HistoryEntryDao.loadHistoryObjectsForResource(eppResource.createVKey());
     if (isDryRun) {
       logger.atInfo().log(
-          "Would delete repo ID %s along with %d history objects",
+          "Would delete repo ID %s along with %d history objects.",
           eppResource.getRepoId(), historyObjects.size());
     } else {
       historyObjects.forEach(tm()::delete);
@@ -198,7 +198,7 @@ public class DeleteLoadTestDataAction implements Runnable {
 
     @Override
     public final void map(EppResource resource) {
-      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorClientId())) {
+      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorRegistrarId())) {
         deleteResource(resource);
         getContext()
             .incrementCounter(

core/src/main/java/google/registry/batch/DeleteProberDataAction.java

@@ -23,6 +23,7 @@ import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexD
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
 import static google.registry.model.tld.Registries.getTldsOfType;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
@@ -42,6 +43,7 @@ import google.registry.config.RegistryEnvironment;
 import google.registry.dns.DnsQueue;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
+import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.EppResourceUtils;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainHistory;
@@ -54,15 +56,18 @@ import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.inject.Inject;
+import org.hibernate.CacheMode;
+import org.hibernate.ScrollMode;
+import org.hibernate.ScrollableResults;
+import org.hibernate.query.Query;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /**
- * Deletes all prober DomainBases and their subordinate history entries, poll messages, and
- * billing events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
- *
- * <p>See: https://www.youtube.com/watch?v=xuuv0syoHnM
+ * Deletes all prober DomainBases and their subordinate history entries, poll messages, and billing
+ * events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
  */
 @Action(
     service = Action.Service.BACKEND,
@@ -73,10 +78,51 @@ public class DeleteProberDataAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
+  /**
+   * The maximum amount of time we allow a prober domain to be in use.
+   *
+   * <p>In practice, the prober's connection will time out well before this duration. This includes
+   * a decent buffer.
+   */
+  private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
+
+  /**
+   * The minimum amount of time we want a domain to be "soft deleted".
+   *
+   * <p>The domain has to remain soft deleted for at least enough time for the DNS task to run and
+   * remove it from DNS itself. This is probably on the order of minutes.
+   */
+  private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
+
+  private static final DnsQueue dnsQueue = DnsQueue.create();
+
+  // Domains to delete must:
+  // 1. Be in one of the prober TLDs
+  // 2. Not be a nic domain
+  // 3. Have no subordinate hosts
+  // 4. Not still be used (within an hour of creation time)
+  // 5. Either be active (creationTime <= now < deletionTime) or have been deleted a while ago (this
+  //    prevents accidental double-map with the same key from immediately deleting active domains)
+  //
+  // Note: creationTime must be compared to a Java object (CreateAutoTimestamp) but deletionTime can
+  // be compared directly to the SQL timestamp (it's a DateTime)
+  private static final String DOMAIN_QUERY_STRING =
+      "FROM Domain d WHERE d.tld IN :tlds AND d.fullyQualifiedDomainName NOT LIKE 'nic.%' AND"
+          + " (d.subordinateHosts IS EMPTY OR d.subordinateHosts IS NULL) AND d.creationTime <"
+          + " :creationTimeCutoff AND ((d.creationTime <= :nowAutoTimestamp AND d.deletionTime >"
+          + " current_timestamp()) OR d.deletionTime < :nowMinusSoftDeleteDelay) ORDER BY d.repoId";
+
+  /** Number of domains to retrieve and delete per SQL transaction. */
+  private static final int BATCH_SIZE = 1000;
+
   @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
   /** List of TLDs to work on. If empty - will work on all TLDs that end with .test. */
   @Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds;
-  @Inject @Config("registryAdminClientId") String registryAdminClientId;
+
+  @Inject
+  @Config("registryAdminClientId")
+  String registryAdminRegistrarId;
+
   @Inject MapreduceRunner mrRunner;
   @Inject Response response;
   @Inject DeleteProberDataAction() {}
@@ -84,25 +130,14 @@ public class DeleteProberDataAction implements Runnable {
   @Override
   public void run() {
     checkState(
-        !Strings.isNullOrEmpty(registryAdminClientId),
+        !Strings.isNullOrEmpty(registryAdminRegistrarId),
         "Registry admin client ID must be configured for prober data deletion to work");
-    mrRunner
-        .setJobName("Delete prober data")
-        .setModuleName("backend")
-        .runMapOnly(
-            new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId),
-            ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
-        .sendLinkToMapreduceConsole(response);
-  }
-
-  private ImmutableSet<String> getProberRoidSuffixes() {
     checkArgument(
         !PRODUCTION.equals(RegistryEnvironment.get())
             || tlds.stream().allMatch(tld -> tld.endsWith(".test")),
         "On production, can only work on TLDs that end with .test");
     ImmutableSet<String> deletableTlds =
-        getTldsOfType(TldType.TEST)
-            .stream()
+        getTldsOfType(TldType.TEST).stream()
             .filter(tld -> tlds.isEmpty() ? tld.endsWith(".test") : tlds.contains(tld))
             .collect(toImmutableSet());
     checkArgument(
@@ -110,10 +145,161 @@ public class DeleteProberDataAction implements Runnable {
         "If tlds are given, they must all exist and be TEST tlds. Given: %s, not found: %s",
         tlds,
         Sets.difference(tlds, deletableTlds));
-    return deletableTlds
-        .stream()
-        .map(tld -> Registry.get(tld).getRoidSuffix())
-        .collect(toImmutableSet());
+    ImmutableSet<String> proberRoidSuffixes =
+        deletableTlds.stream()
+            .map(tld -> Registry.get(tld).getRoidSuffix())
+            .collect(toImmutableSet());
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Delete prober data")
+          .setModuleName("backend")
+          .runMapOnly(
+              new DeleteProberDataMapper(proberRoidSuffixes, isDryRun, registryAdminRegistrarId),
+              ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      runSqlJob(deletableTlds);
+    }
+  }
+
+  private void runSqlJob(ImmutableSet<String> deletableTlds) {
+    AtomicInteger softDeletedDomains = new AtomicInteger();
+    AtomicInteger hardDeletedDomains = new AtomicInteger();
+    jpaTm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have soft-deleted" : "Soft-deleted", softDeletedDomains.get());
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have hard-deleted" : "Hard-deleted", hardDeletedDomains.get());
+  }
+
+  private void processDomains(
+      ImmutableSet<String> deletableTlds,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    DateTime now = tm().getTransactionTime();
+    // Scroll through domains, soft-deleting as necessary (very few will be soft-deleted) and
+    // keeping track of which domains to hard-delete (there can be many, so we batch them up)
+    ScrollableResults scrollableResult =
+        jpaTm()
+            .query(DOMAIN_QUERY_STRING, DomainBase.class)
+            .setParameter("tlds", deletableTlds)
+            .setParameter(
+                "creationTimeCutoff", CreateAutoTimestamp.create(now.minus(DOMAIN_USED_DURATION)))
+            .setParameter("nowMinusSoftDeleteDelay", now.minus(SOFT_DELETE_DELAY))
+            .setParameter("nowAutoTimestamp", CreateAutoTimestamp.create(now))
+            .unwrap(Query.class)
+            .setCacheMode(CacheMode.IGNORE)
+            .scroll(ScrollMode.FORWARD_ONLY);
+    ImmutableList.Builder<String> domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+    ImmutableList.Builder<String> hostNamesToHardDelete = new ImmutableList.Builder<>();
+    for (int i = 1; scrollableResult.next(); i = (i + 1) % BATCH_SIZE) {
+      DomainBase domain = (DomainBase) scrollableResult.get(0);
+      processDomain(
+          domain,
+          domainRepoIdsToHardDelete,
+          hostNamesToHardDelete,
+          softDeletedDomains,
+          hardDeletedDomains);
+      // Batch the deletion and DB flush + session clearing so we don't OOM
+      if (i == 0) {
+        hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+        domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+        hostNamesToHardDelete = new ImmutableList.Builder<>();
+        jpaTm().getEntityManager().flush();
+        jpaTm().getEntityManager().clear();
+      }
+    }
+    // process the remainder
+    hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+  }
+
+  private void processDomain(
+      DomainBase domain,
+      ImmutableList.Builder<String> domainRepoIdsToHardDelete,
+      ImmutableList.Builder<String> hostNamesToHardDelete,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    // If the domain is still active, that means that the prober encountered a failure and did not
+    // successfully soft-delete the domain (thus leaving its DNS entry published). We soft-delete
+    // it now so that the DNS entry can be handled. The domain will then be hard-deleted the next
+    // time the job is run.
+    if (EppResourceUtils.isActive(domain, tm().getTransactionTime())) {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would soft-delete the active domain: %s (%s).",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue);
+      }
+      softDeletedDomains.incrementAndGet();
+    } else {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would hard-delete the non-active domain: %s (%s) and its dependents.",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        domainRepoIdsToHardDelete.add(domain.getRepoId());
+        hostNamesToHardDelete.addAll(domain.getSubordinateHosts());
+      }
+      hardDeletedDomains.incrementAndGet();
+    }
+  }
+
+  private void hardDeleteDomainsAndHosts(
+      ImmutableList<String> domainRepoIds, ImmutableList<String> hostNames) {
+    jpaTm()
+        .query("DELETE FROM Host WHERE fullyQualifiedHostName IN :hostNames")
+        .setParameter("hostNames", hostNames)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingRecurrence WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM DomainHistory WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM Domain WHERE repoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+  }
+
+  // Take a DNS queue + admin registrar id as input so that it can be called from the mapper as well
+  private static void softDeleteDomain(
+      DomainBase domain, String registryAdminRegistrarId, DnsQueue localDnsQueue) {
+    DomainBase deletedDomain =
+        domain.asBuilder().setDeletionTime(tm().getTransactionTime()).setStatusValues(null).build();
+    DomainHistory historyEntry =
+        new DomainHistory.Builder()
+            .setDomain(domain)
+            .setType(DOMAIN_DELETE)
+            .setModificationTime(tm().getTransactionTime())
+            .setBySuperuser(true)
+            .setReason("Deletion of prober data")
+            .setRegistrarId(registryAdminRegistrarId)
+            .build();
+    // Note that we don't bother handling grace periods, billing events, pending transfers, poll
+    // messages, or auto-renews because those will all be hard-deleted the next time the job runs
+    // anyway.
+    tm().putAllWithoutBackup(ImmutableList.of(deletedDomain, historyEntry));
+    // updating foreign keys is a no-op in SQL
+    updateForeignKeyIndexDeletionTime(deletedDomain);
+    localDnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
   }
 
   /** Provides the map method that runs for each existing DomainBase entity. */
@@ -122,32 +308,17 @@ public class DeleteProberDataAction implements Runnable {
     private static final DnsQueue dnsQueue = DnsQueue.create();
     private static final long serialVersionUID = -7724537393697576369L;
 
-    /**
-     * The maximum amount of time we allow a prober domain to be in use.
-     *
-     * In practice, the prober's connection will time out well before this duration. This includes a
-     * decent buffer.
-     *
-     */
-    private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
-
-    /**
-     * The minimum amount of time we want a domain to be "soft deleted".
-     *
-     * The domain has to remain soft deleted for at least enough time for the DNS task to run and
-     * remove it from DNS itself. This is probably on the order of minutes.
-     */
-    private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
-
     private final ImmutableSet<String> proberRoidSuffixes;
     private final Boolean isDryRun;
-    private final String registryAdminClientId;
+    private final String registryAdminRegistrarId;
 
     public DeleteProberDataMapper(
-        ImmutableSet<String> proberRoidSuffixes, Boolean isDryRun, String registryAdminClientId) {
+        ImmutableSet<String> proberRoidSuffixes,
+        Boolean isDryRun,
+        String registryAdminRegistrarId) {
       this.proberRoidSuffixes = proberRoidSuffixes;
       this.isDryRun = isDryRun;
-      this.registryAdminClientId = registryAdminClientId;
+      this.registryAdminRegistrarId = registryAdminRegistrarId;
     }
 
     @Override
@@ -160,7 +331,7 @@ public class DeleteProberDataAction implements Runnable {
           getContext().incrementCounter("skipped, non-prober data");
         }
       } catch (Throwable t) {
-        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s", key);
+        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s.", key);
         getContext().incrementCounter(String.format("error, kind %s", key.getKind()));
       }
     }
@@ -201,9 +372,9 @@ public class DeleteProberDataAction implements Runnable {
       if (EppResourceUtils.isActive(domain, now)) {
         if (isDryRun) {
           logger.atInfo().log(
-              "Would soft-delete the active domain: %s (%s)", domainName, domainKey);
+              "Would soft-delete the active domain: %s (%s).", domainName, domainKey);
         } else {
-          softDeleteDomain(domain);
+          tm().transact(() -> softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue));
         }
         getContext().incrementCounter("domains soft-deleted");
         return;
@@ -223,8 +394,7 @@ public class DeleteProberDataAction implements Runnable {
           tm().transact(
                   () -> {
                     // This ancestor query selects all descendant HistoryEntries, BillingEvents,
-                    // PollMessages,
-                    // and TLD-specific entities, as well as the domain itself.
+                    // PollMessages, and TLD-specific entities, as well as the domain itself.
                     List<Key<Object>> domainAndDependentKeys =
                         auditedOfy().load().ancestor(domainKey).keys().list();
                     ImmutableSet<Key<?>> allKeys =
@@ -243,32 +413,5 @@ public class DeleteProberDataAction implements Runnable {
       getContext().incrementCounter("domains hard-deleted");
       getContext().incrementCounter("total entities hard-deleted", entitiesDeleted);
     }
-
-    private void softDeleteDomain(final DomainBase domain) {
-      tm().transactNew(
-              () -> {
-                DomainBase deletedDomain =
-                    domain
-                        .asBuilder()
-                        .setDeletionTime(tm().getTransactionTime())
-                        .setStatusValues(null)
-                        .build();
-                DomainHistory historyEntry =
-                    new DomainHistory.Builder()
-                        .setDomain(domain)
-                        .setType(DOMAIN_DELETE)
-                        .setModificationTime(tm().getTransactionTime())
-                        .setBySuperuser(true)
-                        .setReason("Deletion of prober data")
-                        .setClientId(registryAdminClientId)
-                        .build();
-                // Note that we don't bother handling grace periods, billing events, pending
-                // transfers, poll messages, or auto-renews because these will all be hard-deleted
-                // the next time the mapreduce runs anyway.
-                tm().putAll(deletedDomain, historyEntry);
-                updateForeignKeyIndexDeletionTime(deletedDomain);
-                dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
-              });
-    }
   }
 }

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -148,9 +148,9 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
               .reduce(0, Integer::sum);
 
       if (!isDryRun) {
-        logger.atInfo().log("Saved OneTime billing events", numBillingEventsSaved);
+        logger.atInfo().log("Saved OneTime billing events.", numBillingEventsSaved);
       } else {
-        logger.atInfo().log("Generated OneTime billing events (dry run)", numBillingEventsSaved);
+        logger.atInfo().log("Generated OneTime billing events (dry run).", numBillingEventsSaved);
       }
       logger.atInfo().log(
           "Recurring event expansion %s complete for billing event range [%s, %s).",
@@ -324,7 +324,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       DomainHistory historyEntry =
           new DomainHistory.Builder()
               .setBySuperuser(false)
-              .setClientId(recurring.getClientId())
+              .setRegistrarId(recurring.getRegistrarId())
               .setModificationTime(tm().getTransactionTime())
               .setDomain(tm().loadByKey(domainKey))
               .setPeriod(Period.create(1, YEARS))
@@ -354,7 +354,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       syntheticOneTimesBuilder.add(
           new OneTime.Builder()
               .setBillingTime(billingTime)
-              .setClientId(recurring.getClientId())
+              .setRegistrarId(recurring.getRegistrarId())
               .setCost(renewCost)
               .setEventTime(eventTime)
               .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -173,7 +173,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
                     retrier.callWithRetry(
                         () -> dnsQueue.addDomainRefreshTask(domainName),
                         TransientFailureException.class);
-                    logger.atInfo().log("Enqueued DNS refresh for domain %s.", domainName);
+                    logger.atInfo().log("Enqueued DNS refresh for domain '%s'.", domainName);
                   });
           deleteTasksWithRetry(
               refreshRequests,
@@ -353,8 +353,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
       VKey<HostResource> hostKey =
-          VKey.fromWebsafeKey(
-              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
           tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
               .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -203,11 +203,11 @@ public class RelockDomainAction implements Runnable {
         "Domain %s has a pending transfer.",
         domainName);
     checkArgument(
-        domain.getCurrentSponsorClientId().equals(oldLock.getRegistrarId()),
+        domain.getCurrentSponsorRegistrarId().equals(oldLock.getRegistrarId()),
         "Domain %s has been transferred from registrar %s to registrar %s since the unlock.",
         domainName,
         oldLock.getRegistrarId(),
-        domain.getCurrentSponsorClientId());
+        domain.getCurrentSponsorRegistrarId());
   }
 
   private void handleNonRetryableFailure(RegistryLock oldLock, Throwable t) {
@@ -293,7 +293,7 @@ public class RelockDomainAction implements Runnable {
 
   private ImmutableSet<InternetAddress> getEmailRecipients(String registrarId) {
     Registrar registrar =
-        Registrar.loadByClientIdCached(registrarId)
+        Registrar.loadByRegistrarIdCached(registrarId)
             .orElseThrow(
                 () ->
                     new IllegalStateException(String.format("Unknown registrar %s", registrarId)));
@@ -313,7 +313,7 @@ public class RelockDomainAction implements Runnable {
         builder.add(new InternetAddress(registryLockEmailAddress));
       } catch (AddressException e) {
         // This shouldn't stop any other emails going out, so swallow it
-        logger.atWarning().log("Invalid email address %s", registryLockEmailAddress);
+        logger.atWarning().log("Invalid email address '%s'.", registryLockEmailAddress);
       }
     }
     return builder.build();

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -76,13 +76,15 @@ public class ResaveEntityAction implements Runnable {
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
     tm().transact(
             () -> {
+              // TODO(/207363014): figure out if this should modified for vkey string replacement
               ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
               tm().put(
                       (entity instanceof EppResource)
                           ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
                           : entity);
               if (!resaveTimes.isEmpty()) {
-                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.from(resourceKey), requestedTime, resaveTimes);
               }
             });
     response.setPayload("Entity re-saved.");

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -0,0 +1,353 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.joda.time.DateTimeZone.UTC;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.flows.certs.CertificateChecker;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registrar.RegistrarContact.Type;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.EmailMessage;
+import google.registry.util.SendEmailService;
+import java.util.Date;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.mail.internet.AddressException;
+import javax.mail.internet.InternetAddress;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/** An action that sends notification emails to registrars whose certificates are expiring soon. */
+@Action(
+    service = Action.Service.BACKEND,
+    path = SendExpiringCertificateNotificationEmailAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class SendExpiringCertificateNotificationEmailAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
+  /**
+   * Used as an offset when storing the last notification email sent date.
+   *
+   * <p>This is used to handle edges cases when the update happens in between the day switch. For
+   * instance,if the job starts at 2:00 am every day and it finishes at 2:03 of the same day, then
+   * next day at 2am, the date difference will be less than a day, which will lead to the date
+   * difference between two successive email sent date being the expected email interval days + 1;
+   */
+  protected static final Duration UPDATE_TIME_OFFSET = Duration.standardMinutes(10);
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd");
+
+  private final CertificateChecker certificateChecker;
+  private final String expirationWarningEmailBodyText;
+  private final SendEmailService sendEmailService;
+  private final String expirationWarningEmailSubjectText;
+  private final InternetAddress gSuiteOutgoingEmailAddress;
+  private final Response response;
+
+  @Inject
+  public SendExpiringCertificateNotificationEmailAction(
+      @Config("expirationWarningEmailBodyText") String expirationWarningEmailBodyText,
+      @Config("expirationWarningEmailSubjectText") String expirationWarningEmailSubjectText,
+      @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
+      SendEmailService sendEmailService,
+      CertificateChecker certificateChecker,
+      Response response) {
+    this.certificateChecker = certificateChecker;
+    this.expirationWarningEmailSubjectText = expirationWarningEmailSubjectText;
+    this.sendEmailService = sendEmailService;
+    this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress;
+    this.expirationWarningEmailBodyText = expirationWarningEmailBodyText;
+    this.response = response;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int numEmailsSent = sendNotificationEmails();
+      String message =
+          String.format(
+              "Done. Sent %d expiring certificate notification emails in total.", numEmailsSent);
+      logger.atInfo().log(message);
+      response.setStatus(SC_OK);
+      response.setPayload(message);
+    } catch (Exception e) {
+      logger.atWarning().withCause(e).log(
+          "Exception thrown when sending expiring certificate notification emails.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(String.format("Exception thrown with cause: %s", e));
+    }
+  }
+
+  /**
+   * Returns a list of registrars that should receive expiring notification emails.
+   *
+   * <p>There are two certificates that should be considered (the main certificate and failOver
+   * certificate). The registrars should receive notifications if one of the certificate checks
+   * returns true.
+   */
+  @VisibleForTesting
+  ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
+    logger.atInfo().log(
+        "Getting a list of registrars that should receive expiring notification emails.");
+    return Streams.stream(Registrar.loadAllCached())
+        .map(
+            registrar ->
+                RegistrarInfo.create(
+                    registrar,
+                    registrar.getClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringCertNotificationSentDate(),
+                            registrar.getClientCertificate().get()),
+                    registrar.getFailoverClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringFailoverCertNotificationSentDate(),
+                            registrar.getFailoverClientCertificate().get())))
+        .filter(
+            registrarInfo ->
+                registrarInfo.isCertExpiring() || registrarInfo.isFailOverCertExpiring())
+        .collect(toImmutableList());
+  }
+
+  /**
+   * Sends a notification email to the registrar regarding the expiring certificate and returns true
+   * if it's sent successfully.
+   */
+  @VisibleForTesting
+  boolean sendNotificationEmail(
+      Registrar registrar,
+      DateTime lastExpiringCertNotificationSentDate,
+      CertificateType certificateType,
+      Optional<String> certificate) {
+    if (!certificate.isPresent()
+        || !certificateChecker.shouldReceiveExpiringNotification(
+            lastExpiringCertNotificationSentDate, certificate.get())) {
+      return false;
+    }
+    try {
+      ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      ImmutableSet<InternetAddress> ccs = getEmailAddresses(registrar, Type.ADMIN);
+      Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
+      logger.atInfo().log(
+          " %s SSL certificate of registrar '%s' will expire on %s.",
+          certificateType.getDisplayName(),
+          registrar.getRegistrarName(),
+          expirationDate.toString());
+      if (recipients.isEmpty() && ccs.isEmpty()) {
+        logger.atWarning().log(
+            "Registrar %s contains no TECH nor ADMIN email addresses to receive notification"
+                + " email.",
+            registrar.getRegistrarName());
+        return false;
+      }
+      sendEmailService.sendEmail(
+          EmailMessage.newBuilder()
+              .setFrom(gSuiteOutgoingEmailAddress)
+              .setSubject(expirationWarningEmailSubjectText)
+              .setBody(
+                  getEmailBody(
+                      registrar.getRegistrarName(),
+                      certificateType,
+                      expirationDate,
+                      registrar.getRegistrarId()))
+              .setRecipients(recipients)
+              .setCcs(ccs)
+              .build());
+      /*
+       * A duration time offset is used here to ensure that date comparison between two
+       * successive dates is always greater than 1 day. This date is set as last updated date,
+       * for applicable certificate.
+       */
+      updateLastNotificationSentDate(
+          registrar,
+          DateTime.now(UTC).minusMinutes((int) UPDATE_TIME_OFFSET.getStandardMinutes()),
+          certificateType);
+      return true;
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to send expiring certificate notification email to registrar %s.",
+              registrar.getRegistrarName()));
+    }
+  }
+
+  /** Updates the last notification sent date in database. */
+  @VisibleForTesting
+  void updateLastNotificationSentDate(
+      Registrar registrar, DateTime now, CertificateType certificateType) {
+    try {
+      tm().transact(
+              () -> {
+                Registrar.Builder newRegistrar = tm().loadByEntity(registrar).asBuilder();
+                switch (certificateType) {
+                  case PRIMARY:
+                    newRegistrar.setLastExpiringCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  case FAILOVER:
+                    newRegistrar.setLastExpiringFailoverCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  default:
+                    throw new IllegalArgumentException(
+                        String.format(
+                            "Unsupported certificate type: %s being passed in when updating "
+                                + "the last notification sent date to registrar %s.",
+                            certificateType.toString(), registrar.getRegistrarName()));
+                }
+              });
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to update the last notification sent date to Registrar %s for the %s "
+                  + "certificate.",
+              registrar.getRegistrarName(), certificateType.getDisplayName()));
+    }
+  }
+
+  /** Sends notification emails to registrars with expiring certificates. */
+  @VisibleForTesting
+  int sendNotificationEmails() {
+    int numEmailsSent = 0;
+    for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
+      Registrar registrar = registrarInfo.registrar();
+      if (registrarInfo.isCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringCertNotificationSentDate(),
+              CertificateType.PRIMARY,
+              registrar.getClientCertificate())) {
+        numEmailsSent++;
+      }
+      if (registrarInfo.isFailOverCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringFailoverCertNotificationSentDate(),
+              CertificateType.FAILOVER,
+              registrar.getFailoverClientCertificate())) {
+        numEmailsSent++;
+      }
+    }
+    return numEmailsSent;
+  }
+
+  /**
+   * Returns a list of email addresses of the registrar that should receive a notification email.
+   */
+  @VisibleForTesting
+  ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
+    ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
+    ImmutableSet.Builder<InternetAddress> recipientEmails = new ImmutableSet.Builder<>();
+    for (RegistrarContact contact : contacts) {
+      try {
+        recipientEmails.add(new InternetAddress(contact.getEmailAddress()));
+      } catch (AddressException e) {
+        logger.atWarning().withCause(e).log(
+            "Registrar Contact email address %s of Registrar %s is invalid; skipping.",
+            contact.getEmailAddress(), registrar.getRegistrarName());
+      }
+    }
+    return recipientEmails.build();
+  }
+
+  /**
+   * Generates email content by taking registrar name, certificate type and expiration date as
+   * parameters.
+   */
+  @VisibleForTesting
+  @SuppressWarnings("lgtm[java/dereferenced-value-may-be-null]")
+  String getEmailBody(
+      String registrarName, CertificateType type, Date expirationDate, String registrarId) {
+    checkArgumentNotNull(expirationDate, "Expiration date cannot be null");
+    checkArgumentNotNull(type, "Certificate type cannot be null");
+    checkArgumentNotNull(registrarId, "Registrar Id cannot be null");
+    return String.format(
+        expirationWarningEmailBodyText,
+        registrarName,
+        type.getDisplayName(),
+        DATE_FORMATTER.print(new DateTime(expirationDate)),
+        registrarId);
+  }
+
+  /**
+   * Certificate types for X509Certificate.
+   *
+   * <p><b>Note:</b> These types are only used to indicate the type of expiring certificate in
+   * notification emails.
+   */
+  protected enum CertificateType {
+    PRIMARY("primary"),
+    FAILOVER("fail-over");
+
+    private final String displayName;
+
+    CertificateType(String displayName) {
+      this.displayName = displayName;
+    }
+
+    public String getDisplayName() {
+      return displayName;
+    }
+  }
+
+  @AutoValue
+  public abstract static class RegistrarInfo {
+
+    static RegistrarInfo create(
+        Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
+      return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(
+          registrar, isCertExpiring, isFailOverCertExpiring);
+    }
+
+    public abstract Registrar registrar();
+
+    public abstract boolean isCertExpiring();
+
+    public abstract boolean isFailOverCertExpiring();
+  }
+}

core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java

@@ -0,0 +1,138 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.contact.ContactHistory;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * An action that wipes out Personal Identifiable Information (PII) fields of {@link ContactHistory}
+ * entities.
+ *
+ * <p>ContactHistory entities should be retained in the database for only certain amount of time.
+ * This periodic wipe out action only applies to SQL.
+ */
+@Action(
+    service = Service.BACKEND,
+    path = WipeOutContactHistoryPiiAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeOutContactHistoryPiiAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/wipeOutContactHistoryPii";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private final Clock clock;
+  private final Response response;
+  private final int minMonthsBeforeWipeOut;
+  private final int wipeOutQueryBatchSize;
+
+  @Inject
+  public WipeOutContactHistoryPiiAction(
+      Clock clock,
+      @Config("minMonthsBeforeWipeOut") int minMonthsBeforeWipeOut,
+      @Config("wipeOutQueryBatchSize") int wipeOutQueryBatchSize,
+      Response response) {
+    this.clock = clock;
+    this.response = response;
+    this.minMonthsBeforeWipeOut = minMonthsBeforeWipeOut;
+    this.wipeOutQueryBatchSize = wipeOutQueryBatchSize;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int totalNumOfWipedEntities = 0;
+      DateTime wipeOutTime = clock.nowUtc().minusMonths(minMonthsBeforeWipeOut);
+      logger.atInfo().log(
+          "About to wipe out all PII of contact history entities prior to %s.", wipeOutTime);
+
+      int numOfWipedEntities = 0;
+      do {
+        numOfWipedEntities =
+            jpaTm()
+                .transact(
+                    () ->
+                        wipeOutContactHistoryData(
+                            getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime)));
+        totalNumOfWipedEntities += numOfWipedEntities;
+      } while (numOfWipedEntities > 0);
+      String msg =
+          String.format(
+              "Done. Wiped out PII of %d ContactHistory entities in total.",
+              totalNumOfWipedEntities);
+      logger.atInfo().log(msg);
+      response.setPayload(msg);
+      response.setStatus(SC_OK);
+
+    } catch (Exception e) {
+      logger.atSevere().withCause(e).log(
+          "Exception thrown during the process of wiping out contact history PII.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(
+          String.format(
+              "Exception thrown during the process of wiping out contact history PII with cause"
+                  + ": %s",
+              e));
+    }
+  }
+
+  /**
+   * Returns a stream of up to {@link #wipeOutQueryBatchSize} {@link ContactHistory} entities
+   * containing PII that are prior to @param wipeOutTime.
+   */
+  @VisibleForTesting
+  Stream<ContactHistory> getNextContactHistoryEntitiesWithPiiBatch(DateTime wipeOutTime) {
+    // email is one of the required fields in EPP, meaning it's initially not null.
+    // Therefore, checking if it's null is one way to avoid processing contact history entities
+    // that have been processed previously. Refer to RFC 5733 for more information.
+    return jpaTm()
+        .query(
+            "FROM ContactHistory WHERE modificationTime < :wipeOutTime " + "AND email IS NOT NULL",
+            ContactHistory.class)
+        .setParameter("wipeOutTime", wipeOutTime)
+        .setMaxResults(wipeOutQueryBatchSize)
+        .getResultStream();
+  }
+
+  /** Wipes out the PII of each of the {@link ContactHistory} entities in the stream. */
+  @VisibleForTesting
+  int wipeOutContactHistoryData(Stream<ContactHistory> contactHistoryEntities) {
+    AtomicInteger numOfEntities = new AtomicInteger(0);
+    contactHistoryEntities.forEach(
+        contactHistoryEntity -> {
+          jpaTm().update(contactHistoryEntity.asBuilder().wipeOutPii().build());
+          numOfEntities.incrementAndGet();
+        });
+    logger.atInfo().log(
+        "Wiped out all PII fields of %d ContactHistory entities.", numOfEntities.get());
+    return numOfEntities.get();
+  }
+}

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -92,7 +92,12 @@ public class WipeoutDatastoreAction implements Runnable {
               .setJobName(createJobName("bulk-delete-datastore-", clock))
               .setContainerSpecGcsPath(
                   String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+              .setParameters(
+                  ImmutableMap.of(
+                      "kindsToDelete",
+                      "*",
+                      "registryEnvironment",
+                      RegistryEnvironment.get().name()));
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -0,0 +1,88 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import java.util.List;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+
+/**
+ * A database snapshot shareable by concurrent queries from multiple database clients. A snapshot is
+ * uniquely identified by its {@link #getSnapshotId snapshotId}, and must stay open until all
+ * concurrent queries to this snapshot have attached to it by calling {@link
+ * google.registry.persistence.transaction.JpaTransactionManager#setDatabaseSnapshot}. However, it
+ * can be closed before those queries complete.
+ *
+ * <p>This feature is <em>Postgresql-only</em>.
+ *
+ * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to avoid
+ * exhausting predicate locks at the SERIALIZABLE level.
+ */
+// TODO(b/193662898): vendor-independent support for richer transaction semantics.
+public class DatabaseSnapshot implements AutoCloseable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private String snapshotId;
+  private EntityManager entityManager;
+  private EntityTransaction transaction;
+
+  private DatabaseSnapshot() {}
+
+  public String getSnapshotId() {
+    checkState(entityManager != null, "Snapshot not opened yet.");
+    checkState(entityManager.isOpen(), "Snapshot already closed.");
+    return snapshotId;
+  }
+
+  private DatabaseSnapshot open() {
+    entityManager = jpaTm().getStandaloneEntityManager();
+    transaction = entityManager.getTransaction();
+    transaction.setRollbackOnly();
+    transaction.begin();
+
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+
+    List<?> snapshotIds =
+        entityManager.createNativeQuery("SELECT pg_export_snapshot();").getResultList();
+    checkState(snapshotIds.size() == 1, "Unexpected number of snapshots: %s", snapshotIds.size());
+    snapshotId = (String) snapshotIds.get(0);
+    return this;
+  }
+
+  @Override
+  public void close() {
+    if (transaction != null && transaction.isActive()) {
+      try {
+        transaction.rollback();
+      } catch (Exception e) {
+        logger.atWarning().withCause(e).log("Failed to close a Database Snapshot");
+      }
+    }
+    if (entityManager != null && entityManager.isOpen()) {
+      entityManager.close();
+    }
+  }
+
+  public static DatabaseSnapshot createSnapshot() {
+    return new DatabaseSnapshot().open();
+  }
+}

core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java

@@ -17,7 +17,6 @@ package google.registry.beam.common;
 import static com.google.common.base.Verify.verify;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.model.contact.ContactResource;
 import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -59,18 +58,16 @@ public class JpaDemoPipeline implements Serializable {
                   public void processElement() {
                     // AppEngineEnvironment is needed as long as JPA entity classes still depends
                     // on Objectify.
-                    try (AppEngineEnvironment allowOfyEntity = new AppEngineEnvironment()) {
-                      int result =
-                          (Integer)
-                              jpaTm()
-                                  .transact(
-                                      () ->
-                                          jpaTm()
-                                              .getEntityManager()
-                                              .createNativeQuery("select 1;")
-                                              .getSingleResult());
-                      verify(result == 1, "Expecting 1, got %s.", result);
-                    }
+                    int result =
+                        (Integer)
+                            jpaTm()
+                                .transact(
+                                    () ->
+                                        jpaTm()
+                                            .getEntityManager()
+                                            .createNativeQuery("select 1;")
+                                            .getSingleResult());
+                    verify(result == 1, "Expecting 1, got %s.", result);
                     counter.inc();
                   }
                 }));

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -20,9 +20,9 @@ import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
-import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
 import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -138,6 +138,9 @@ public final class RegistryJpaIO {
 
     abstract Coder<T> coder();
 
+    @Nullable
+    abstract String snapshotId();
+
     abstract Builder<R, T> toBuilder();
 
     @Override
@@ -145,7 +148,9 @@ public final class RegistryJpaIO {
     public PCollection<T> expand(PBegin input) {
       return input
           .apply("Starting " + name(), Create.of((Void) null))
-          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
+          .apply(
+              "Run query for " + name(),
+              ParDo.of(new QueryRunner<>(query(), resultMapper(), snapshotId())))
           .setCoder(coder())
           .apply("Reshuffle", Reshuffle.viaRandomKey());
     }
@@ -162,6 +167,18 @@ public final class RegistryJpaIO {
       return toBuilder().coder(coder).build();
     }
 
+    /**
+     * Specifies the database snapshot to use for this query.
+     *
+     * <p>This feature is <em>Postgresql-only</em>. User is responsible for keeping the snapshot
+     * available until all JVM workers have started using it by calling {@link
+     * JpaTransactionManager#setDatabaseSnapshot}.
+     */
+    // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+    public Read<R, T> withSnapshot(@Nullable String snapshotId) {
+      return toBuilder().snapshotId(snapshotId).build();
+    }
+
     static <R, T> Builder<R, T> builder() {
       return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
           .name(DEFAULT_NAME)
@@ -179,6 +196,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<R, T> coder(Coder coder);
 
+      abstract Builder<R, T> snapshotId(@Nullable String sharedSnapshotId);
+
       abstract Read<R, T> build();
 
       Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
@@ -201,22 +220,28 @@ public final class RegistryJpaIO {
     static class QueryRunner<R, T> extends DoFn<Void, T> {
       private final RegistryQuery<R> query;
       private final SerializableFunction<R, T> resultMapper;
+      // java.util.Optional is not serializable. Use of Guava Optional is discouraged.
+      @Nullable private final String snapshotId;
 
-      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+      QueryRunner(
+          RegistryQuery<R> query,
+          SerializableFunction<R, T> resultMapper,
+          @Nullable String snapshotId) {
         this.query = query;
         this.resultMapper = resultMapper;
+        this.snapshotId = snapshotId;
       }
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
-        // AppEngineEnvironment is need for handling VKeys, which involve Ofy keys. Unlike
-        // SqlBatchWriter, it is unnecessary to initialize ObjectifyService in this class.
-        try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-          // TODO(b/187210388): JpaTransactionManager should support non-transactional query.
-          jpaTm()
-              .transactNoRetry(
-                  () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
-        }
+        jpaTm()
+            .transactNoRetry(
+                () -> {
+                  if (snapshotId != null) {
+                    jpaTm().setDatabaseSnapshot(snapshotId);
+                  }
+                  query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
+                });
       }
     }
   }
@@ -274,6 +299,12 @@ public final class RegistryJpaIO {
 
     public abstract SerializableFunction<T, Object> jpaConverter();
 
+    /**
+     * Signal to the writer that the {@link UpdateAutoTimestamp} property should be allowed to
+     * manipulate its value before persistence. The default value is {@code true}.
+     */
+    abstract boolean withUpdateAutoTimestamp();
+
     public Write<T> withName(String name) {
       return toBuilder().name(name).build();
     }
@@ -294,6 +325,10 @@ public final class RegistryJpaIO {
       return toBuilder().jpaConverter(jpaConverter).build();
     }
 
+    public Write<T> disableUpdateAutoTimestamp() {
+      return toBuilder().withUpdateAutoTimestamp(false).build();
+    }
+
     abstract Builder<T> toBuilder();
 
     @Override
@@ -310,7 +345,7 @@ public final class RegistryJpaIO {
               GroupIntoBatches.<Integer, T>ofSize(batchSize()).withShardedKey())
           .apply(
               "Write in batch for " + name(),
-              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter())));
+              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter(), withUpdateAutoTimestamp())));
     }
 
     static <T> Builder<T> builder() {
@@ -318,7 +353,8 @@ public final class RegistryJpaIO {
           .name(DEFAULT_NAME)
           .batchSize(DEFAULT_BATCH_SIZE)
           .shards(DEFAULT_SHARDS)
-          .jpaConverter(x -> x);
+          .jpaConverter(x -> x)
+          .withUpdateAutoTimestamp(true);
     }
 
     @AutoValue.Builder
@@ -332,6 +368,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<T> jpaConverter(SerializableFunction<T, Object> jpaConverter);
 
+      abstract Builder<T> withUpdateAutoTimestamp(boolean withUpdateAutoTimestamp);
+
       abstract Write<T> build();
     }
   }
@@ -340,37 +378,38 @@ public final class RegistryJpaIO {
   private static class SqlBatchWriter<T> extends DoFn<KV<ShardedKey<Integer>, Iterable<T>>, Void> {
     private final Counter counter;
     private final SerializableFunction<T, Object> jpaConverter;
+    private final boolean withAutoTimestamp;
 
-    SqlBatchWriter(String type, SerializableFunction<T, Object> jpaConverter) {
+    SqlBatchWriter(
+        String type, SerializableFunction<T, Object> jpaConverter, boolean withAutoTimestamp) {
       counter = Metrics.counter("SQL_WRITE", type);
       this.jpaConverter = jpaConverter;
-    }
-
-    @Setup
-    public void setup() {
-      // AppEngineEnvironment is needed as long as Objectify keys are still involved in the handling
-      // of SQL entities (e.g., in VKeys). ObjectifyService needs to be initialized when conversion
-      // between Ofy entity and Datastore entity is needed.
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ObjectifyService.initOfy();
-      }
+      this.withAutoTimestamp = withAutoTimestamp;
     }
 
     @ProcessElement
     public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ImmutableList<Object> entities =
-            Streams.stream(kv.getValue())
-                .map(this.jpaConverter::apply)
-                // TODO(b/177340730): post migration delete the line below.
-                .filter(Objects::nonNull)
-                .collect(ImmutableList.toImmutableList());
-        try {
-          jpaTm().transact(() -> jpaTm().putAll(entities));
-          counter.inc(entities.size());
-        } catch (RuntimeException e) {
-          processSingly(entities);
-        }
+      if (withAutoTimestamp) {
+        actuallyProcessElement(kv);
+        return;
+      }
+      try (DisableAutoUpdateResource disable = UpdateAutoTimestamp.disableAutoUpdate()) {
+        actuallyProcessElement(kv);
+      }
+    }
+
+    private void actuallyProcessElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
+      ImmutableList<Object> entities =
+          Streams.stream(kv.getValue())
+              .map(this.jpaConverter::apply)
+              // TODO(b/177340730): post migration delete the line below.
+              .filter(Objects::nonNull)
+              .collect(ImmutableList.toImmutableList());
+      try {
+        jpaTm().transact(() -> jpaTm().putAll(entities));
+        counter.inc(entities.size());
+      } catch (RuntimeException e) {
+        processSingly(entities);
       }
     }
 

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -21,6 +21,7 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -45,9 +46,19 @@ public interface RegistryPipelineComponent {
   @Config("projectId")
   String getProjectId();
 
+  /** Returns the regular {@link JpaTransactionManager} for general use. */
   @BeamJpaTm
   Lazy<JpaTransactionManager> getJpaTransactionManager();
 
+  /**
+   * Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
+   * ({@link google.registry.model.domain.DomainBase} and {@link
+   * google.registry.model.domain.DomainHistory}). Please refer to {@link
+   * google.registry.model.bulkquery.BulkQueryEntities} for more information.
+   */
+  @BeamBulkQueryJpaTm
+  Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -34,7 +35,6 @@ import org.apache.beam.sdk.options.Description;
 public interface RegistryPipelineOptions extends GcpOptions {
 
   @Description("The Registry environment.")
-  @Nullable
   RegistryEnvironment getRegistryEnvironment();
 
   void setRegistryEnvironment(RegistryEnvironment environment);
@@ -45,6 +45,12 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setIsolationOverride(TransactionIsolationLevel isolationOverride);
 
+  @Description("The JPA Transaction Manager to use.")
+  @Default.Enum(value = "REGULAR")
+  JpaTransactionManagerType getJpaTransactionManagerType();
+
+  void setJpaTransactionManagerType(JpaTransactionManagerType jpaTransactionManagerType);
+
   @Description("The number of entities to write to the SQL database in one operation.")
   @Default.Integer(20)
   int getSqlWriteBatchSize();

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -20,6 +20,8 @@ import com.google.auto.service.AutoService;
 import com.google.common.flogger.FluentLogger;
 import dagger.Lazy;
 import google.registry.config.RegistryEnvironment;
+import google.registry.config.SystemPropertySetter;
+import google.registry.model.AppEngineEnvironment;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import org.apache.beam.sdk.harness.JvmInitializer;
@@ -35,18 +37,36 @@ import org.apache.beam.sdk.options.PipelineOptions;
 @AutoService(JvmInitializer.class)
 public class RegistryPipelineWorkerInitializer implements JvmInitializer {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  public static final String PROPERTY = "google.registry.beam";
 
   @Override
   public void beforeProcessing(PipelineOptions options) {
     RegistryPipelineOptions registryOptions = options.as(RegistryPipelineOptions.class);
     RegistryEnvironment environment = registryOptions.getRegistryEnvironment();
     if (environment == null || environment.equals(RegistryEnvironment.UNITTEST)) {
-      return;
+      throw new RuntimeException(
+          "A registry environment must be specified in the pipeline options.");
     }
-    logger.atInfo().log("Setting up RegistryEnvironment: %s", environment);
+    logger.atInfo().log("Setting up RegistryEnvironment %s.", environment);
     environment.setup();
-    Lazy<JpaTransactionManager> transactionManagerLazy =
-        toRegistryPipelineComponent(registryOptions).getJpaTransactionManager();
+    RegistryPipelineComponent registryPipelineComponent =
+        toRegistryPipelineComponent(registryOptions);
+    Lazy<JpaTransactionManager> transactionManagerLazy;
+    switch (registryOptions.getJpaTransactionManagerType()) {
+      case BULK_QUERY:
+        transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
+        break;
+      case REGULAR:
+      default:
+        transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
+    }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
+    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // loads all ofy entities.
+    new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
+        .setEnvironmentForAllThreads();
+    // Set the system property so that we can call IdService.allocateId() without access to
+    // datastore.
+    SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
   }
 }

core/src/main/java/google/registry/beam/common/RegistryQuery.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import google.registry.persistence.transaction.JpaTransactionManager;
 import java.io.Serializable;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -28,6 +29,15 @@ import javax.persistence.criteria.CriteriaQuery;
 
 /** Interface for query instances used by {@link RegistryJpaIO.Read}. */
 public interface RegistryQuery<T> extends Serializable {
+
+  /**
+   * Number of JPA entities to fetch in each batch during a query.
+   *
+   * <p>With Hibernate, for result streaming to work, a query's fetchSize property must be set to a
+   * non-zero value.
+   */
+  int QUERY_FETCH_SIZE = 1000;
+
   Stream<T> stream();
 
   interface CriteriaQuerySupplier<T> extends Supplier<CriteriaQuery<T>>, Serializable {}
@@ -49,6 +59,7 @@ public interface RegistryQuery<T> extends Serializable {
       if (parameters != null) {
         parameters.forEach(query::setParameter);
       }
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
       @SuppressWarnings("unchecked")
       Stream<T> resultStream = query.getResultStream();
       return nativeQuery ? resultStream : resultStream.map(e -> detach(entityManager, e));
@@ -64,11 +75,14 @@ public interface RegistryQuery<T> extends Serializable {
   static <T> RegistryQuery<T> createQuery(
       String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
     return () -> {
-      TypedQuery<T> query = jpaTm().query(jpql, clazz);
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(jpql, clazz);
       if (parameters != null) {
         parameters.forEach(query::setParameter);
       }
-      return query.getResultStream();
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
     };
   }
 
@@ -82,7 +96,13 @@ public interface RegistryQuery<T> extends Serializable {
    * @param <T> Type of each row in the result set.
    */
   static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
-    return () -> jpaTm().query(criteriaQuery.get()).getResultStream();
+    return () -> {
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(criteriaQuery.get());
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
+    };
   }
 
   /**
@@ -108,7 +128,10 @@ public interface RegistryQuery<T> extends Serializable {
       return;
     }
     try {
-      entityManager.detach(object);
+      // TODO(b/193662898): choose detach() or clear() based on the type of transaction.
+      // For context, EntityManager.detach() does not remove all metadata about loaded entities.
+      // See b/193925312 or https://hibernate.atlassian.net/browse/HHH-14735 for details.
+      entityManager.clear();
     } catch (IllegalArgumentException e) {
       // Not an entity. Do nothing.
     }

core/src/main/java/google/registry/beam/comparedb/DatastoreSnapshots.java

@@ -0,0 +1,169 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.initsql.Transforms.createTagForKind;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.Transforms;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.joda.time.DateTime;
+
+/** Utilities for loading Datastore snapshots. */
+public final class DatastoreSnapshots {
+
+  private DatastoreSnapshots() {}
+
+  /**
+   * Datastore kinds eligible for validation. This set must be consistent with {@link
+   * SqlSnapshots#ALL_SQL_ENTITIES}.
+   */
+  @VisibleForTesting
+  static final ImmutableSet<Class<?>> ALL_DATASTORE_KINDS =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          HistoryEntry.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class);
+
+  /**
+   * Returns the Datastore snapshot right before {@code commitLogToTime} for the user specified
+   * {@code kinds}. The resulting snapshot has all changes that happened before {@code
+   * commitLogToTime}, and none at or after {@code commitLogToTime}.
+   *
+   * <p>If {@code HistoryEntry} is included in {@code kinds}, the result will contain {@code
+   * PCollections} for the child entities, {@code DomainHistory}, {@code ContactHistory}, and {@code
+   * HostHistory}.
+   */
+  static PCollectionTuple loadDatastoreSnapshotByKind(
+      Pipeline pipeline,
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<Class<?>> kinds) {
+    PCollectionTuple snapshot =
+        pipeline.apply(
+            "Load Datastore snapshot.",
+            Transforms.loadDatastoreSnapshot(
+                exportDir,
+                commitLogDir,
+                commitLogFromTime,
+                commitLogToTime,
+                kinds.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet())));
+
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<?> kind : kinds) {
+      PCollection<VersionedEntity> perKindSnapshot =
+          snapshot.get(createTagForKind(Key.getKind(kind)));
+      if (SqlEntity.class.isAssignableFrom(kind)) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag((Class<? extends SqlEntity>) kind),
+                datastoreEntityToPojo(perKindSnapshot, kind.getSimpleName()));
+        continue;
+      }
+      Verify.verify(kind == HistoryEntry.class, "Unexpected Non-SqlEntity class: %s", kind);
+      PCollectionTuple historyEntriesByType = splitHistoryEntry(perKindSnapshot);
+      for (Map.Entry<TupleTag<?>, PCollection<?>> entry :
+          historyEntriesByType.getAll().entrySet()) {
+        perTypeSnapshots = perTypeSnapshots.and(entry.getKey().getId(), entry.getValue());
+      }
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Splits a {@link PCollection} of {@link HistoryEntry HistoryEntries} into three collections of
+   * its child entities by type.
+   */
+  static PCollectionTuple splitHistoryEntry(PCollection<VersionedEntity> historyEntries) {
+    return historyEntries.apply(
+        "Split HistoryEntry by Resource Type",
+        ParDo.of(
+                new DoFn<VersionedEntity, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity historyEntry, MultiOutputReceiver out) {
+                    Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(historyEntry))
+                        .ifPresent(
+                            sqlEntity ->
+                                out.get(createSqlEntityTupleTag(sqlEntity.getClass()))
+                                    .output(sqlEntity));
+                  }
+                })
+            .withOutputTags(
+                createSqlEntityTupleTag(DomainHistory.class),
+                TupleTagList.of(createSqlEntityTupleTag(ContactHistory.class))
+                    .and(createSqlEntityTupleTag(HostHistory.class))));
+  }
+
+  /**
+   * Transforms a {@link PCollection} of {@link VersionedEntity VersionedEntities} to Ofy Java
+   * objects.
+   */
+  static PCollection<SqlEntity> datastoreEntityToPojo(
+      PCollection<VersionedEntity> entities, String desc) {
+    return entities.apply(
+        "Datastore Entity to Pojo " + desc,
+        ParDo.of(
+            new DoFn<VersionedEntity, SqlEntity>() {
+              @ProcessElement
+              public void processElement(
+                  @Element VersionedEntity entity, OutputReceiver<SqlEntity> out) {
+                Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(entity))
+                    .ifPresent(out::output);
+              }
+            }));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -0,0 +1,145 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import dagger.Component;
+import google.registry.config.CloudTasksUtilsModule;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.util.Clock;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.joda.time.DateTime;
+import org.joda.time.Instant;
+import org.joda.time.Interval;
+
+/** Finds the necessary information for loading the most recent Datastore snapshot. */
+public class LatestDatastoreSnapshotFinder {
+  private final String projectId;
+  private final GcsUtils gcsUtils;
+  private final Clock clock;
+
+  @Inject
+  LatestDatastoreSnapshotFinder(
+      @Config("projectId") String projectId, GcsUtils gcsUtils, Clock clock) {
+    this.projectId = projectId;
+    this.gcsUtils = gcsUtils;
+    this.clock = clock;
+  }
+
+  /**
+   * Finds information of the most recent Datastore snapshot, including the GCS folder of the
+   * exported data files and the start and stop times of the export. The folder of the CommitLogs is
+   * also included in the return.
+   */
+  public DatastoreSnapshotInfo getSnapshotInfo() {
+    String bucketName = RegistryConfig.getDatastoreBackupsBucket().substring("gs://".length());
+    /**
+     * Find the bucket-relative path to the overall metadata file of the last Datastore export.
+     * Since Datastore export is saved daily, we may need to look back to yesterday. If found, the
+     * return value is like
+     * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
+     */
+    Optional<String> metaFilePathOptional = findMostRecentExportMetadataFile(bucketName, 2);
+    if (!metaFilePathOptional.isPresent()) {
+      throw new NoSuchElementException("No exports found over the past 2 days.");
+    }
+    String metaFilePath = metaFilePathOptional.get();
+    String metaFileFolder = metaFilePath.substring(0, metaFilePath.indexOf('/'));
+    Instant exportStartTime = Instant.parse(metaFileFolder.replace('_', '.') + 'Z');
+    BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath));
+    Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+    return DatastoreSnapshotInfo.create(
+        String.format("gs://%s/%s", bucketName, metaFileFolder),
+        getCommitLogDir(),
+        new Interval(exportStartTime, exportEndTime));
+  }
+
+  public String getCommitLogDir() {
+    return "gs://" + projectId + "-commits";
+  }
+
+  /**
+   * Finds the bucket-relative path of the overall export metadata file, in the given bucket,
+   * searching back up to {@code lookBackDays} days, including today.
+   *
+   * <p>The overall export metadata file is the last file created during a Datastore export. All
+   * data has been exported by the creation time of this file. The name of this file, like that of
+   * all files in the same export, begins with the timestamp when the export starts.
+   *
+   * <p>An example return value: {@code
+   * 2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata}.
+   */
+  private Optional<String> findMostRecentExportMetadataFile(String bucketName, int lookBackDays) {
+    DateTime today = clock.nowUtc();
+    for (int day = 0; day < lookBackDays; day++) {
+      String dateString = today.minusDays(day).toString("yyyy-MM-dd");
+      try {
+        Optional<String> metaFilePath =
+            gcsUtils.listFolderObjects(bucketName, dateString).stream()
+                .filter(s -> s.endsWith("overall_export_metadata"))
+                .map(s -> dateString + s)
+                .sorted(Comparator.<String>naturalOrder().reversed())
+                .findFirst();
+        if (metaFilePath.isPresent()) {
+          return metaFilePath;
+        }
+      } catch (IOException ioe) {
+        throw new RuntimeException(ioe);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /** Holds information about a Datastore snapshot. */
+  @AutoValue
+  abstract static class DatastoreSnapshotInfo {
+    abstract String exportDir();
+
+    abstract String commitLogDir();
+
+    abstract Interval exportInterval();
+
+    static DatastoreSnapshotInfo create(
+        String exportDir, String commitLogDir, Interval exportOperationInterval) {
+      return new AutoValue_LatestDatastoreSnapshotFinder_DatastoreSnapshotInfo(
+          exportDir, commitLogDir, exportOperationInterval);
+    }
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
+  interface LatestDatastoreSnapshotFinderFinderComponent {
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotInfoFinder();
+  }
+}

core/src/main/java/google/registry/beam/comparedb/SqlSnapshots.java

@@ -0,0 +1,384 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.comparedb.ValidateSqlUtils.getMedianIdForHistoryTable;
+
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Streams;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.model.bulkquery.DomainBaseLite;
+import google.registry.model.bulkquery.DomainHistoryHost;
+import google.registry.model.bulkquery.DomainHistoryLite;
+import google.registry.model.bulkquery.DomainHost;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+
+/**
+ * Utilities for loading SQL snapshots.
+ *
+ * <p>For {@link DomainBase} and {@link DomainHistory}, this class assumes the presence of the
+ * {@link google.registry.persistence.PersistenceModule.JpaTransactionManagerType#BULK_QUERY
+ * bulk-query-capable JpaTransactionManager}, and takes advantage of it for higher throughput.
+ *
+ * <p>For now this class is meant for use during the database migration period only. Therefore, it
+ * contains optimizations specifically for the production database at the current size, e.g.,
+ * parallel queries for select tables.
+ */
+public final class SqlSnapshots {
+
+  private SqlSnapshots() {}
+
+  /**
+   * SQL entity types that are eligible for validation. This set must be consistent with {@link
+   * DatastoreSnapshots#ALL_DATASTORE_KINDS}.
+   */
+  static final ImmutableSet<Class<? extends SqlEntity>> ALL_SQL_ENTITIES =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class,
+          ContactHistory.class,
+          HostHistory.class,
+          DomainHistory.class);
+
+  /**
+   * Loads a SQL snapshot for the given {@code sqlEntityTypes}.
+   *
+   * <p>If {@code snapshotId} is present, all queries use the specified database snapshot,
+   * guaranteeing a consistent result.
+   */
+  public static PCollectionTuple loadCloudSqlSnapshotByType(
+      Pipeline pipeline,
+      ImmutableSet<Class<? extends SqlEntity>> sqlEntityTypes,
+      Optional<String> snapshotId) {
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<? extends SqlEntity> clazz : sqlEntityTypes) {
+      if (clazz == DomainBase.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainBase.class),
+                loadAndAssembleDomainBase(pipeline, snapshotId));
+        continue;
+      }
+      if (clazz == DomainHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainHistory.class),
+                loadAndAssembleDomainHistory(pipeline, snapshotId));
+        continue;
+      }
+      if (clazz == ContactHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(ContactHistory.class),
+                loadContactHistory(pipeline, snapshotId));
+        continue;
+      }
+      perTypeSnapshots =
+          perTypeSnapshots.and(
+              createSqlEntityTupleTag(clazz),
+              pipeline.apply(
+                  "SQL Load " + clazz.getSimpleName(),
+                  RegistryJpaIO.read(
+                          () -> CriteriaQueryBuilder.create(clazz).build(), SqlEntity.class::cast)
+                      .withSnapshot(snapshotId.orElse(null))));
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Bulk-loads parts of {@link DomainBase} and assembles them in the pipeline.
+   *
+   * @see BulkQueryEntities
+   */
+  public static PCollection<SqlEntity> loadAndAssembleDomainBase(
+      Pipeline pipeline, Optional<String> snapshotId) {
+    PCollection<KV<String, Serializable>> baseObjects =
+        readAllAndAssignKey(pipeline, DomainBaseLite.class, DomainBaseLite::getRepoId, snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(pipeline, GracePeriod.class, GracePeriod::getDomainRepoId, snapshotId);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DelegationSignerData.class,
+            DelegationSignerData::getDomainRepoId,
+            snapshotId);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(pipeline, DomainHost.class, DomainHost::getDomainRepoId, snapshotId);
+
+    return PCollectionList.of(
+            ImmutableList.of(baseObjects, gracePeriods, delegationSigners, domainHosts))
+        .apply("SQL Merge DomainBase parts", Flatten.pCollections())
+        .apply("Group by Domain Parts by RepoId", GroupByKey.create())
+        .apply(
+            "Assemble DomainBase",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainBaseLite> baseObjects =
+                        partsByType.getAllOf(DomainBaseLite.class);
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainBaseLite object per repoId: " + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainBase(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(GracePeriod.class),
+                            partsByType.getAllOf(DelegationSignerData.class),
+                            partsByType.getAllOf(DomainHost.class).stream()
+                                .map(DomainHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet())));
+                  }
+                }));
+  }
+
+  /**
+   * Loads all {@link ContactHistory} entities from the database.
+   *
+   * <p>This method uses two queries to load data in parallel. This is a performance optimization
+   * specifically for the production database.
+   */
+  static PCollection<SqlEntity> loadContactHistory(Pipeline pipeline, Optional<String> snapshotId) {
+    long medianId =
+        getMedianIdForHistoryTable("ContactHistory")
+            .orElseThrow(
+                () -> new IllegalStateException("Not a valid database: no ContactHistory."));
+    PCollection<SqlEntity> part1 =
+        pipeline.apply(
+            "SQL Load ContactHistory first half",
+            RegistryJpaIO.read(
+                    String.format("select c from ContactHistory c where id <= %s", medianId),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    PCollection<SqlEntity> part2 =
+        pipeline.apply(
+            "SQL Load ContactHistory second half",
+            RegistryJpaIO.read(
+                    String.format("select c from ContactHistory c where id > %s", medianId),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    return PCollectionList.of(part1)
+        .and(part2)
+        .apply("Combine ContactHistory parts", Flatten.pCollections());
+  }
+
+  /**
+   * Bulk-loads all parts of {@link DomainHistory} and assembles them in the pipeline.
+   *
+   * <p>This method uses two queries to load {@link DomainBaseLite} in parallel. This is a
+   * performance optimization specifically for the production database.
+   *
+   * @see BulkQueryEntities
+   */
+  static PCollection<SqlEntity> loadAndAssembleDomainHistory(
+      Pipeline pipeline, Optional<String> snapshotId) {
+    long medianId =
+        getMedianIdForHistoryTable("DomainHistory")
+            .orElseThrow(
+                () -> new IllegalStateException("Not a valid database: no DomainHistory."));
+    PCollection<KV<String, Serializable>> baseObjectsPart1 =
+        queryAndAssignKey(
+            pipeline,
+            "first half",
+            String.format("select c from DomainHistory c where id <= %s", medianId),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> baseObjectsPart2 =
+        queryAndAssignKey(
+            pipeline,
+            "second half",
+            String.format("select c from DomainHistory c where id > %s", medianId),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriodHistory.class,
+            compose(GracePeriodHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DomainDsDataHistory.class,
+            compose(DomainDsDataHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline,
+            DomainHistoryHost.class,
+            compose(DomainHistoryHost::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> transactionRecords =
+        readAllAndAssignKey(
+            pipeline,
+            DomainTransactionRecord.class,
+            compose(DomainTransactionRecord::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+
+    return PCollectionList.of(
+            ImmutableList.of(
+                baseObjectsPart1,
+                baseObjectsPart2,
+                gracePeriods,
+                delegationSigners,
+                domainHosts,
+                transactionRecords))
+        .apply("Merge DomainHistory parts", Flatten.pCollections())
+        .apply("Group by DomainHistory Parts by DomainHistoryId string", GroupByKey.create())
+        .apply(
+            "Assemble DomainHistory",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainHistoryLite> baseObjects =
+                        partsByType.getAllOf(DomainHistoryLite.class);
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainHistoryLite object per domainHistoryId: "
+                            + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainHistory(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(DomainDsDataHistory.class),
+                            partsByType.getAllOf(DomainHistoryHost.class).stream()
+                                .map(DomainHistoryHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet()),
+                            partsByType.getAllOf(GracePeriodHistory.class),
+                            partsByType.getAllOf(DomainTransactionRecord.class)));
+                  }
+                }));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> readAllAndAssignKey(
+      Pipeline pipeline,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName(),
+            RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(type).build())
+                .withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName(),
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> queryAndAssignKey(
+      Pipeline pipeline,
+      String diffrentiator,
+      String jplQuery,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName() + " " + diffrentiator,
+            RegistryJpaIO.read(jplQuery, false, type::cast).withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName() + " " + diffrentiator,
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  // TODO(b/205988530): don't use beam serializablefunction, make one that extends Java's Function.
+  private static <R, I, T> SerializableFunction<R, T> compose(
+      SerializableFunction<R, I> f1, SerializableFunction<I, T> f2) {
+    return r -> f2.apply(f1.apply(r));
+  }
+
+  /** Container that receives mixed-typed data and groups them by {@link Class}. */
+  static class TypedClassifier {
+    private final ImmutableSetMultimap<Class<?>, Object> classifiedEntities;
+
+    TypedClassifier(Iterable<Serializable> inputs) {
+      this.classifiedEntities =
+          Streams.stream(inputs)
+              .collect(ImmutableSetMultimap.toImmutableSetMultimap(Object::getClass, x -> x));
+    }
+
+    <T> ImmutableSet<T> getAllOf(Class<T> clazz) {
+      return classifiedEntities.get(clazz).stream()
+          .map(clazz::cast)
+          .collect(ImmutableSet.toImmutableSet());
+    }
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -0,0 +1,159 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+
+/**
+ * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
+ * (secondary storage).
+ */
+public class ValidateSqlPipeline {
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final int COMMIT_LOG_MARGIN_MINUTES = 10;
+
+  private final ValidateSqlPipelineOptions options;
+  private final DatastoreSnapshotInfo mostRecentExport;
+
+  public ValidateSqlPipeline(
+      ValidateSqlPipelineOptions options, DatastoreSnapshotInfo mostRecentExport) {
+    this.options = options;
+    this.mostRecentExport = mostRecentExport;
+  }
+
+  void run() {
+    run(Pipeline.create(options));
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    // TODO(weiminyu): Acquire the commit log replay lock when the lock release bug is fixed.
+    DateTime latestCommitLogTime =
+        TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+    Preconditions.checkState(
+        latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
+        "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
+    try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+      setupPipeline(pipeline, Optional.of(databaseSnapshot.getSnapshotId()), latestCommitLogTime);
+      State state = pipeline.run().waitUntilFinish();
+      if (!State.DONE.equals(state)) {
+        throw new IllegalStateException("Unexpected pipeline state: " + state);
+      }
+    }
+  }
+
+  void setupPipeline(
+      Pipeline pipeline, Optional<String> sqlSnapshotId, DateTime latestCommitLogTime) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minusMinutes(COMMIT_LOG_MARGIN_MINUTES),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      PCollectionList.of(datastoreSnapshot.get(tag))
+          .and(cloudSqlSnapshot.get(tag))
+          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
+          .apply(
+              "Assign primary key to merged " + clazz.getSimpleName(),
+              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
+          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  public static void main(String[] args) {
+    ValidateSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    DatastoreSnapshotInfo mostRecentExport =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder()
+            .getSnapshotInfo();
+
+    new ValidateSqlPipeline(options, mostRecentExport).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -0,0 +1,20 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+
+/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
+public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -0,0 +1,270 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.contact.ContactBase;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.eppcommon.AuthInfo;
+import google.registry.model.host.HostHistory;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import java.lang.reflect.Field;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Helpers for use by {@link ValidateSqlPipeline}. */
+final class ValidateSqlUtils {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ValidateSqlUtils() {}
+
+  /**
+   * Query template for finding the median value of the {@code history_revision_id} column in one of
+   * the History tables.
+   *
+   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * history tables. Although the {@code repo_id} column is the leading column in the primary keys
+   * of these tables, in practice and with production data, division by {@code history_revision_id}
+   * works slightly faster for unknown reasons.
+   */
+  private static final String MEDIAN_ID_QUERY_TEMPLATE =
+      "SELECT history_revision_id FROM (                                                        "
+          + "  SELECT"
+          + "    ROW_NUMBER() OVER (ORDER BY history_revision_id ASC) AS rownumber,"
+          + "    history_revision_id"
+          + "  FROM \"%TABLE%\""
+          + ") AS foo\n"
+          + "WHERE rownumber in (select count(*) / 2 + 1 from \"%TABLE%\")";
+
+  static Optional<Long> getMedianIdForHistoryTable(String tableName) {
+    Preconditions.checkArgument(
+        tableName.endsWith("History"), "Table must be one of the History tables.");
+    String sqlText = MEDIAN_ID_QUERY_TEMPLATE.replace("%TABLE%", tableName);
+    List results =
+        jpaTm()
+            .transact(() -> jpaTm().getEntityManager().createNativeQuery(sqlText).getResultList());
+    verify(results.size() < 2, "MidPoint query should have at most one result.");
+    if (results.isEmpty()) {
+      return Optional.empty();
+    }
+    return Optional.of(((BigInteger) results.get(0)).longValue());
+  }
+
+  static TupleTag<SqlEntity> createSqlEntityTupleTag(Class<? extends SqlEntity> actualType) {
+    return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
+  }
+
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+    private final HashMap<String, Counter> totalCounters = new HashMap<>();
+    private final HashMap<String, Counter> missingCounters = new HashMap<>();
+    private final HashMap<String, Counter> unequalCounters = new HashMap<>();
+    private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
+
+    private volatile boolean logPrinted = false;
+
+    private String getCounterKey(Class<?> clazz) {
+      return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
+    }
+
+    private synchronized void ensureCounterExists(String counterKey) {
+      if (totalCounters.containsKey(counterKey)) {
+        return;
+      }
+      totalCounters.put(counterKey, Metrics.counter("CompareDB", "Total Compared: " + counterKey));
+      missingCounters.put(
+          counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
+      unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
+      badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
+    }
+
+    /**
+     * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
+     * This will be removed when we start exporting such entities to GCS.
+     */
+    void logDiff(String key, Object entry0, Object entry1) {
+      if (logPrinted) {
+        return;
+      }
+      logPrinted = true;
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      StringBuilder sb = new StringBuilder();
+      fields0.forEach(
+          (field, value) -> {
+            if (fields1.containsKey(field)) {
+              if (!Objects.equals(value, fields1.get(field))) {
+                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
+              }
+            } else {
+              sb.append(field + "Not found in entity 2\n");
+            }
+          });
+      fields1.forEach(
+          (field, value) -> {
+            if (!fields0.containsKey(field)) {
+              sb.append(field + "Not found in entity 1\n");
+            }
+          });
+      logger.atWarning().log(key + "  " + sb.toString());
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+      ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
+
+      verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
+      verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
+
+      String counterKey = getCounterKey(entities.get(0).getClass());
+      ensureCounterExists(counterKey);
+      totalCounters.get(counterKey).inc();
+
+      if (entities.size() == 1) {
+        missingCounters.get(counterKey).inc();
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
+        }
+        return;
+      }
+      SqlEntity entity0;
+      SqlEntity entity1;
+
+      try {
+        entity0 = normalizeEntity(entities.get(0));
+        entity1 = normalizeEntity(entities.get(1));
+      } catch (Exception e) {
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          badEntityCounters.get(counterKey).inc();
+        }
+        return;
+      }
+
+      if (!Objects.equals(entity0, entity1)) {
+        unequalCounters.get(counterKey).inc();
+        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+      }
+    }
+  }
+
+  static SqlEntity normalizeEntity(SqlEntity sqlEntity) {
+    if (sqlEntity instanceof EppResource) {
+      return normalizeEppResource(sqlEntity);
+    }
+    if (sqlEntity instanceof HistoryEntry) {
+      return (SqlEntity) normalizeHistoryEntry((HistoryEntry) sqlEntity);
+    }
+    return sqlEntity;
+  }
+
+  /**
+   * Normalizes an {@link EppResource} instance for comparison.
+   *
+   * <p>This method may modify the input object using reflection instead of making a copy with
+   * {@code eppResource.asBuilder().build()}, because when {@code eppResource} is a {@link
+   * google.registry.model.domain.DomainBase}, the {@code build} method accesses the Database, which
+   * we want to avoid.
+   */
+  static SqlEntity normalizeEppResource(SqlEntity eppResource) {
+    try {
+      Field authField =
+          eppResource instanceof DomainContent
+              ? DomainContent.class.getDeclaredField("authInfo")
+              : eppResource instanceof ContactBase
+                  ? ContactBase.class.getDeclaredField("authInfo")
+                  : null;
+      if (authField != null) {
+        authField.setAccessible(true);
+        AuthInfo authInfo = (AuthInfo) authField.get(eppResource);
+        // When AuthInfo is missing, the authInfo field is null if the object is loaded from
+        // Datastore, or a PasswordAuth with null properties if loaded from SQL. In the second case
+        // we set the authInfo field to null.
+        if (authInfo != null
+            && authInfo.getPw() != null
+            && authInfo.getPw().getRepoId() == null
+            && authInfo.getPw().getValue() == null) {
+          authField.set(eppResource, null);
+        }
+      }
+
+      Field field = EppResource.class.getDeclaredField("revisions");
+      field.setAccessible(true);
+      field.set(eppResource, null);
+      return eppResource;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Normalizes a {@link HistoryEntry} for comparison.
+   *
+   * <p>This method modifies the input using reflection because relevant builder methods performs
+   * unwanted checks and changes.
+   */
+  static HistoryEntry normalizeHistoryEntry(HistoryEntry historyEntry) {
+    // History objects from Datastore do not have details of their EppResource objects
+    // (domainContent, contactBase, hostBase).
+    try {
+      if (historyEntry instanceof DomainHistory) {
+        Field domainContent = DomainHistory.class.getDeclaredField("domainContent");
+        domainContent.setAccessible(true);
+        domainContent.set(historyEntry, null);
+        Field domainTransactionRecords =
+            HistoryEntry.class.getDeclaredField("domainTransactionRecords");
+        domainTransactionRecords.setAccessible(true);
+        Set<?> domainTransactionRecordsValue = (Set<?>) domainTransactionRecords.get(historyEntry);
+        if (domainTransactionRecordsValue != null && domainTransactionRecordsValue.isEmpty()) {
+          domainTransactionRecords.set(historyEntry, null);
+        }
+      } else if (historyEntry instanceof ContactHistory) {
+        Field contactBase = ContactHistory.class.getDeclaredField("contactBase");
+        contactBase.setAccessible(true);
+        contactBase.set(historyEntry, null);
+      } else if (historyEntry instanceof HostHistory) {
+        Field hostBase = HostHistory.class.getDeclaredField("hostBase");
+        hostBase.setAccessible(true);
+        hostBase.set(historyEntry, null);
+      }
+      return historyEntry;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java

@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.datastore.v1.Entity;
+import google.registry.config.RegistryEnvironment;
 import java.util.Iterator;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
@@ -308,6 +309,11 @@ public class BulkDeleteDatastorePipeline {
 
   public interface BulkDeletePipelineOptions extends GcpOptions {
 
+    @Description("The Registry environment.")
+    RegistryEnvironment getRegistryEnvironment();
+
+    void setRegistryEnvironment(RegistryEnvironment environment);
+
     @Description(
         "The Datastore KINDs to be deleted. The format may be:\n"
             + "\t- The list of kinds to be deleted as a comma-separated string, or\n"

core/src/main/java/google/registry/beam/datastore/DatastoreV1.java

@@ -169,14 +169,15 @@ public class DatastoreV1 {
       int numSplits;
       try {
         long estimatedSizeBytes = getEstimatedSizeBytes(datastore, query, namespace);
-        logger.atInfo().log("Estimated size bytes for the query is: %s", estimatedSizeBytes);
+        logger.atInfo().log("Estimated size for the query is %d bytes.", estimatedSizeBytes);
         numSplits =
             (int)
                 Math.min(
                     NUM_QUERY_SPLITS_MAX,
                     Math.round(((double) estimatedSizeBytes) / DEFAULT_BUNDLE_SIZE_BYTES));
       } catch (Exception e) {
-        logger.atWarning().log("Failed the fetch estimatedSizeBytes for query: %s", query, e);
+        logger.atWarning().withCause(e).log(
+            "Failed the fetch estimatedSizeBytes for query: %s", query);
         // Fallback in case estimated size is unavailable.
         numSplits = NUM_QUERY_SPLITS_MIN;
       }
@@ -215,7 +216,7 @@ public class DatastoreV1 {
     private static Entity getLatestTableStats(
         String ourKind, @Nullable String namespace, Datastore datastore) throws DatastoreException {
       long latestTimestamp = queryLatestStatisticsTimestamp(datastore, namespace);
-      logger.atInfo().log("Latest stats timestamp for kind %s is %s", ourKind, latestTimestamp);
+      logger.atInfo().log("Latest stats timestamp for kind %s is %s.", ourKind, latestTimestamp);
 
       Query.Builder queryBuilder = Query.newBuilder();
       if (Strings.isNullOrEmpty(namespace)) {
@@ -234,7 +235,7 @@ public class DatastoreV1 {
       long now = System.currentTimeMillis();
       RunQueryResponse response = datastore.runQuery(request);
       logger.atFine().log(
-          "Query for per-kind statistics took %sms", System.currentTimeMillis() - now);
+          "Query for per-kind statistics took %d ms.", System.currentTimeMillis() - now);
 
       QueryResultBatch batch = response.getBatch();
       if (batch.getEntityResultsCount() == 0) {
@@ -330,7 +331,7 @@ public class DatastoreV1 {
           logger.atWarning().log(
               "Failed to translate Gql query '%s': %s", gqlQueryWithZeroLimit, e.getMessage());
           logger.atWarning().log(
-              "User query might have a limit already set, so trying without zero limit");
+              "User query might have a limit already set, so trying without zero limit.");
           // Retry without the zero limit.
           return translateGqlQuery(gql, datastore, namespace);
         } else {
@@ -514,10 +515,10 @@ public class DatastoreV1 {
       @ProcessElement
       public void processElement(ProcessContext c) throws Exception {
         String gqlQuery = c.element();
-        logger.atInfo().log("User query: '%s'", gqlQuery);
+        logger.atInfo().log("User query: '%s'.", gqlQuery);
         Query query =
             translateGqlQueryWithLimitCheck(gqlQuery, datastore, v1Options.getNamespace());
-        logger.atInfo().log("User gql query translated to Query(%s)", query);
+        logger.atInfo().log("User gql query translated to Query(%s).", query);
         c.output(query);
       }
     }
@@ -573,7 +574,7 @@ public class DatastoreV1 {
           estimatedNumSplits = numSplits;
         }
 
-        logger.atInfo().log("Splitting the query into %s splits", estimatedNumSplits);
+        logger.atInfo().log("Splitting the query into %d splits.", estimatedNumSplits);
         List<Query> querySplits;
         try {
           querySplits =
@@ -647,7 +648,7 @@ public class DatastoreV1 {
               throw exception;
             }
             if (!BackOffUtils.next(sleeper, backoff)) {
-              logger.atSevere().log("Aborting after %s retries.", MAX_RETRIES);
+              logger.atSevere().log("Aborting after %d retries.", MAX_RETRIES);
               throw exception;
             }
           }

core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java

@@ -20,11 +20,11 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.backup.VersionedEntity;
 import google.registry.beam.common.RegistryJpaIO;
 import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken;
@@ -62,6 +62,7 @@ import org.joda.time.DateTime;
  * <ol>
  *   <li>{@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set
  *       up in the SQL database.
+ *   <li>{@link Cursor}: Logically can depend on {@code Registry}, but without foreign key.
  *   <li>{@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet.
  *   <li>{@link ContactResource}: references {@code Registrar}
  *   <li>{@link RegistrarContact}: references {@code Registrar}.
@@ -101,7 +102,11 @@ public class InitSqlPipeline implements Serializable {
    */
   private static final ImmutableList<Class<?>> PHASE_ONE_ORDERED =
       ImmutableList.of(
-          Registry.class, Registrar.class, ContactResource.class, RegistrarContact.class);
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class);
 
   /**
    * Datastore kinds to be written to the SQL database after the cleansed version of {@link
@@ -219,13 +224,12 @@ public class InitSqlPipeline implements Serializable {
             .withName(transformId)
             .withBatchSize(options.getSqlWriteBatchSize())
             .withShards(options.getSqlWriteShards())
-            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity));
+            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity)
+            .disableUpdateAutoTimestamp());
   }
 
   private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
-    try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-      return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
-    }
+    return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
   }
 
   public static void main(String[] args) {

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -22,6 +22,7 @@ import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.util.Comparator.comparing;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -100,8 +101,9 @@ public final class Transforms {
   }
 
   /**
-   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
-   * commitLogToTime} for caller specified {@code kinds}.
+   * Composite {@link PTransform transform} that loads the Datastore snapshot right before {@code
+   * commitLogToTime} for caller specified {@code kinds}. The resulting snapshot has all changes
+   * that happened before {@code commitLogToTime}, and none at or after {@code commitLogToTime}.
    *
    * <p>Caller must provide the location of a Datastore export that started AFTER {@code
    * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
@@ -261,8 +263,8 @@ public final class Transforms {
 
   // Production data repair configs go below. See b/185954992.
 
-  // Prober domains in bad state, without associated contacts, hosts, billings, and history.
-  // They can be safely ignored.
+  // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized
+  // history. They can be safely ignored.
   private static final ImmutableSet<String> IGNORED_DOMAINS =
       ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT");
 
@@ -277,7 +279,7 @@ public final class Transforms {
 
   // Prober contacts referencing phantom registrars. They and their associated history entries can
   // be safely ignored.
-  private static final ImmutableSet IGNORED_CONTACTS =
+  private static final ImmutableSet<String> IGNORED_CONTACTS =
       ImmutableSet.of(
           "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE");
 
@@ -298,7 +300,7 @@ public final class Transforms {
       return !IGNORED_HOSTS.contains(roid);
     }
     if (entity.getKind().equals("HistoryEntry")) {
-      // Remove production bad data: History of the contacts to be ignored:
+      // Remove production bad data: Histories of ignored EPP resources:
       com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent();
       if (parentKey.getKind().equals("ContactResource")) {
         String contactRoid = parentKey.getName();
@@ -308,6 +310,10 @@ public final class Transforms {
         String hostRoid = parentKey.getName();
         return !IGNORED_HOSTS.contains(hostRoid);
       }
+      if (parentKey.getKind().equals("DomainBase")) {
+        String domainRoid = parentKey.getName();
+        return !IGNORED_DOMAINS.contains(domainRoid);
+      }
     }
     // End of production-specific checks.
 
@@ -320,7 +326,8 @@ public final class Transforms {
     return true;
   }
 
-  private static Entity repairBadData(Entity entity) {
+  @VisibleForTesting
+  static Entity repairBadData(Entity entity) {
     if (entity.getKind().equals("Cancellation")
         && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) {
       // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the
@@ -328,6 +335,15 @@ public final class Transforms {
       // instead of append. See b/185954992.
       entity.setUnindexedProperty("reason", Reason.RENEW.name());
       entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name()));
+    } else if (entity.getKind().equals("DomainBase")) {
+      // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this.
+      entity.setIndexedProperty(
+          "fullyQualifiedDomainName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedDomainName")));
+    } else if (entity.getKind().equals("HostResource")) {
+      entity.setIndexedProperty(
+          "fullyQualifiedHostName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedHostName")));
     }
     return entity;
   }
@@ -348,7 +364,7 @@ public final class Transforms {
    *     to make Optional work with BEAM)
    */
   @Nullable
-  public static Object convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
+  public static SqlEntity convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
     return dsEntity
         .getEntity()
         .filter(Transforms::isMigratable)
@@ -365,7 +381,8 @@ public final class Transforms {
    * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
    * given {@link Iterable}.
    */
-  static PTransform<PBegin, PCollection<String>> toStringPCollection(Iterable<String> strings) {
+  private static PTransform<PBegin, PCollection<String>> toStringPCollection(
+      Iterable<String> strings) {
     return Create.of(strings).withCoder(StringUtf8Coder.of());
   }
 
@@ -373,7 +390,7 @@ public final class Transforms {
    * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
    * caller-provided {@code transformer}.
    */
-  static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+  private static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
       DoFn<ReadableFile, VersionedEntity> transformer) {
     return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
       @Override
@@ -389,7 +406,7 @@ public final class Transforms {
     private final DateTime fromTime;
     private final DateTime toTime;
 
-    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+    FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
       checkNotNull(fromTime, "fromTime");
       checkNotNull(toTime, "toTime");
       checkArgument(

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -57,7 +57,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=invoicing}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
@@ -125,7 +126,7 @@ public class InvoicingPipeline implements Serializable {
         oneTime.getId(),
         DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
         DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
-        registrar.getClientId(),
+        registrar.getRegistrarId(),
         registrar.getBillingIdentifier().toString(),
         registrar.getPoNumber().orElse(""),
         DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -19,10 +19,13 @@ import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static google.registry.rde.RdeModule.BRDA_QUEUE;
+import static google.registry.rde.RdeModule.RDE_UPLOAD_QUEUE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.auto.value.AutoValue;
 import com.google.cloud.storage.BlobId;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.PgpHelper;
@@ -31,14 +34,20 @@ import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
 import google.registry.model.tld.Registry;
+import google.registry.rde.BrdaCopyAction;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.Ghostryde;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.RdeCounter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.rde.RdeModule;
 import google.registry.rde.RdeResourceType;
+import google.registry.rde.RdeUploadAction;
 import google.registry.rde.RdeUtil;
+import google.registry.request.Action.Service;
+import google.registry.request.RequestParameters;
 import google.registry.tldconfig.idn.IdnTableEnum;
+import google.registry.util.CloudTasksUtils;
 import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
 import google.registry.xml.ValidationMode;
@@ -66,8 +75,12 @@ public class RdeIO {
   abstract static class Write
       extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
 
+    private static final long serialVersionUID = 3334807737227087760L;
+
     abstract GcsUtils gcsUtils();
 
+    abstract CloudTasksUtils cloudTasksUtils();
+
     abstract String rdeBucket();
 
     // It's OK to return a primitive array because we are only using it to construct the
@@ -83,7 +96,9 @@ public class RdeIO {
 
     @AutoValue.Builder
     abstract static class Builder {
-      abstract Builder setGcsUtils(GcsUtils gcsUtils);
+      abstract Builder setGcsUtils(GcsUtils value);
+
+      abstract Builder setCloudTasksUtils(CloudTasksUtils value);
 
       abstract Builder setRdeBucket(String value);
 
@@ -100,7 +115,9 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()));
+          .apply(
+              "Update cursor and enqueue next action",
+              ParDo.of(new CursorUpdater(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -109,6 +126,7 @@ public class RdeIO {
       extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5496375923068400382L;
 
     private final GcsUtils gcsUtils;
     private final String rdeBucket;
@@ -155,7 +173,7 @@ public class RdeIO {
         checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
         prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
       } else {
-        prefix = prefix + "/" + basename;
+        prefix = prefix + '/' + basename;
       }
       BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
       // This file will contain the byte length (ASCII) of the raw unencrypted XML.
@@ -172,7 +190,7 @@ public class RdeIO {
 
       // Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
 
-      logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
+      logger.atInfo().log("Writing files '%s' and '%s'.", xmlFilename, xmlLengthFilename);
       try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
           OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
           OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
@@ -219,7 +237,7 @@ public class RdeIO {
       //
       // This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
       if (mode == RdeMode.FULL) {
-        logger.atInfo().log("Writing %s", reportFilename);
+        logger.atInfo().log("Writing file '%s'.", reportFilename);
         try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
             OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
           counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
@@ -229,7 +247,7 @@ public class RdeIO {
       }
       // Now that we're done, output roll the cursor forward.
       if (key.manual()) {
-        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
+        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task.");
       } else {
         outputReceiver.output(KV.of(key, revision));
       }
@@ -237,10 +255,19 @@ public class RdeIO {
   }
 
   private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5822176227753327224L;
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private CursorUpdater(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
 
     @ProcessElement
-    public void processElement(@Element KV<PendingDeposit, Integer> input) {
+    public void processElement(
+        @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -265,8 +292,33 @@ public class RdeIO {
                     key);
                 tm().put(Cursor.create(key.cursor(), newPosition, registry));
                 logger.atInfo().log(
-                    "Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
+                    "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                if (key.mode() == RdeMode.FULL) {
+                  cloudTasksUtils.enqueue(
+                      RDE_UPLOAD_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          RdeUploadAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                } else {
+                  cloudTasksUtils.enqueue(
+                      BRDA_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          BrdaCopyAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_WATERMARK,
+                              key.watermark().toString(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                }
               });
     }
   }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -14,36 +14,53 @@
 
 package google.registry.beam.rde;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.beam.rde.RdePipeline.TupleTags.DOMAIN_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.EXTERNAL_HOST_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.HOST_TO_PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_CONTACTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
+import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
+import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
-import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.util.CloudTasksUtils;
+import google.registry.util.UtilsModule;
 import google.registry.xml.ValidationMode;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -51,73 +68,158 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.persistence.Entity;
+import javax.persistence.IdClass;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.DateTime;
 
 /**
  * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=rde}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
+ * <p>This pipeline only works for pending deposits with the same watermark, the {@link
+ * google.registry.rde.RdeStagingAction} will batch such pending deposits together and launch
+ * multiple pipelines if multiple watermarks exist.
+ *
+ * <p>The pipeline is broadly divided into two parts -- creating the {@link DepositFragment}s, and
+ * processing them.
+ *
+ * <h1>Creating {@link DepositFragment}</h1>
+ *
+ * <h2>{@link Registrar}</h2>
+ *
+ * Non-test registrar entities are loaded from Cloud SQL and marshalled into deposit fragments. They
+ * are <b>NOT</b> rewound to the watermark.
+ *
+ * <h2>{@link EppResource}</h2>
+ *
+ * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
+ * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
+ * revision ID) from the SQL query.
+ *
+ * <h3>{@link DomainBase}</h3>
+ *
+ * After the most recent (live) domain resources are loaded from the corresponding history objects,
+ * we marshall them to deposit fragments and emit the (pending deposit: deposit fragment) pairs for
+ * further processing. We also find all the contacts and hosts referenced by a given domain and emit
+ * pairs of (contact/host repo ID: pending deposit) for all RDE pending deposits for further
+ * processing.
+ *
+ * <h3>{@link ContactResource}</h3>
+ *
+ * We first join most recent contact histories, represented by (contact repo ID: contact history
+ * revision ID) pairs, with referenced contacts, represented by (contact repo ID: pending deposit)
+ * pairs, on the contact repo ID, to remove unreferenced contact histories. Contact resources are
+ * then loaded from the remaining referenced contact histories, and marshalled into (pending
+ * deposit: deposit fragment) pairs.
+ *
+ * <h3>{@link HostResource}</h3>
+ *
+ * Similar to {@link ContactResource}, we join the most recent host history with referenced hosts to
+ * find most recent referenced hosts. For external hosts we do the same treatment as we did on
+ * contacts and obtain the (pending deposit: deposit fragment) pairs. For subordinate hosts, we need
+ * to find the superordinate domain in order to properly handle pending transfer in the deposit as
+ * well. So we first find the superordinate domain repo ID from the host and join the (superordinate
+ * domain repo ID: (subordinate host repo ID: (pending deposit: revision ID))) pair with the (domain
+ * repo ID: revision ID) pair obtained from the domain history query in order to map the host at
+ * watermark to the domain at watermark. We then proceed to create the (pending deposit: deposit
+ * fragment) pair for subordinate hosts using the added domain information.
+ *
+ * <h1>Processing {@link DepositFragment}</h1>
+ *
+ * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
+ * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
+ * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
+ * pending deposit is then rolled forward, and the next action is enqueued. The latter two
+ * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
+ *
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
 @Singleton
 public class RdePipeline implements Serializable {
 
+  private static final long serialVersionUID = -4866795928854754666L;
   private final transient RdePipelineOptions options;
   private final ValidationMode mode;
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final ImmutableSet<PendingDeposit> pendingDeposits;
+  private final DateTime watermark;
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
+  private final CloudTasksUtils cloudTasksUtils;
+  private final RdeMarshaller marshaller;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
   private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
       Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
 
-  private static final String EPP_RESOURCE_QUERY =
-      "SELECT id FROM %entity% "
-          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
-
-  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
-    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
-        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
-  }
+  // The field name of the EPP resource embedded in its corresponding history entry.
+  private static final ImmutableMap<Class<? extends HistoryEntry>, String> EPP_RESOURCE_FIELD_NAME =
+      ImmutableMap.of(
+          DomainHistory.class,
+          "domainContent",
+          ContactHistory.class,
+          "contactBase",
+          HostHistory.class,
+          "hostBase");
 
   @Inject
-  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
+  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
-    this.pendings = decodePendings(options.getPendings());
-    this.rdeBucket = options.getGcsBucket();
+    this.pendingDeposits = decodePendingDeposits(options.getPendings());
+    ImmutableSet<DateTime> potentialWatermarks =
+        pendingDeposits.stream()
+            .map(PendingDeposit::watermark)
+            .distinct()
+            .collect(toImmutableSet());
+    checkArgument(
+        potentialWatermarks.size() == 1,
+        String.format(
+            "RDE pipeline should only work on pending deposits "
+                + "with the same watermark, but %d were given: %s",
+            potentialWatermarks.size(), potentialWatermarks));
+    this.watermark = potentialWatermarks.asList().get(0);
+    this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
+    this.marshaller = new RdeMarshaller(mode);
   }
 
   PipelineResult run() {
@@ -129,152 +231,452 @@ public class RdePipeline implements Serializable {
   }
 
   PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
-    return PCollectionList.of(processRegistrars(pipeline))
-        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
-        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
-        .and(processNonRegistrarEntities(pipeline, HostResource.class))
-        .apply(Flatten.pCollections())
+    PCollection<KV<PendingDeposit, DepositFragment>> registrarFragments =
+        processRegistrars(pipeline);
+
+    PCollection<KV<String, Long>> domainHistories =
+        getMostRecentHistoryEntries(pipeline, DomainHistory.class);
+
+    PCollection<KV<String, Long>> contactHistories =
+        getMostRecentHistoryEntries(pipeline, ContactHistory.class);
+
+    PCollection<KV<String, Long>> hostHistories =
+        getMostRecentHistoryEntries(pipeline, HostHistory.class);
+
+    PCollectionTuple processedDomainHistories = processDomainHistories(domainHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> domainFragments =
+        processedDomainHistories.get(DOMAIN_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> contactFragments =
+        processContactHistories(
+            processedDomainHistories.get(REFERENCED_CONTACTS), contactHistories);
+
+    PCollectionTuple processedHosts =
+        processHostHistories(processedDomainHistories.get(REFERENCED_HOSTS), hostHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> externalHostFragments =
+        processedHosts.get(EXTERNAL_HOST_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> subordinateHostFragments =
+        processSubordinateHosts(processedHosts.get(SUPERORDINATE_DOMAINS), domainHistories);
+
+    return PCollectionList.of(registrarFragments)
+        .and(domainFragments)
+        .and(contactFragments)
+        .and(externalHostFragments)
+        .and(subordinateHostFragments)
+        .apply(
+            "Combine PendingDeposit:DepositFragment pairs from all entities",
+            Flatten.pCollections())
         .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply("Group by PendingDeposit", GroupByKey.create());
+        .apply("Group DepositFragment by PendingDeposit", GroupByKey.create());
   }
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
     input.apply(
-        "Write to GCS and update cursors",
+        "Write to GCS, update cursors, and enqueue upload tasks",
         RdeIO.Write.builder()
             .setRdeBucket(rdeBucket)
             .setGcsUtils(gcsUtils)
+            .setCloudTasksUtils(cloudTasksUtils)
             .setValidationMode(mode)
             .setStagingKeyBytes(stagingKeyBytes)
             .build());
   }
 
-  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+  private PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    // Note that the namespace in the metric is not being used by Stackdriver, it just has to be
+    // non-empty.
+    // See:
+    // https://stackoverflow.com/questions/48530496/google-dataflow-custom-metrics-not-showing-on-stackdriver
+    Counter includedRegistrarCounter = Metrics.counter("RDE", "IncludedRegistrar");
+    Counter registrarFragmentCounter = Metrics.counter("RDE", "RegistrarFragment");
     return pipeline
         .apply(
-            "Read all production Registrar entities",
+            "Read all production Registrars",
             RegistryJpaIO.read(
                 "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
                 ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
                 String.class,
-                // TODO: consider adding coders for entities and pass them directly instead of using
-                // VKeys.
                 id -> VKey.createSql(Registrar.class, id)))
         .apply(
-            "Marshal Registrar into DepositFragment",
+            "Marshall Registrar into DepositFragment",
             FlatMapElements.into(
-                    TypeDescriptors.kvs(
+                    kvs(
                         TypeDescriptor.of(PendingDeposit.class),
                         TypeDescriptor.of(DepositFragment.class)))
                 .via(
                     (VKey<Registrar> key) -> {
+                      includedRegistrarCounter.inc();
                       Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
-                      DepositFragment fragment =
-                          new RdeMarshaller(mode).marshalRegistrar(registrar);
-                      return pendings.values().stream()
-                          .map(pending -> KV.of(pending, fragment))
-                          .collect(toImmutableSet());
+                      DepositFragment fragment = marshaller.marshalRegistrar(registrar);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          pendingDeposits.stream()
+                              .map(pending -> KV.of(pending, fragment))
+                              .collect(toImmutableSet());
+                      registrarFragmentCounter.inc(fragments.size());
+                      return fragments;
                     }));
   }
 
-  @SuppressWarnings("deprecation") // Reshuffle is still recommended by Dataflow.
-  <T extends EppResource>
-      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
-          Pipeline pipeline, Class<T> clazz) {
-    return createInputs(pipeline, clazz)
-        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
+  /**
+   * Load the most recent history entry before the watermark for a given history entry type.
+   *
+   * <p>Note that deleted and non-production resources are not included.
+   *
+   * @return A KV pair of (repoId, revisionId), used to reconstruct the composite key for the
+   *     history entry.
+   */
+  private <T extends HistoryEntry> PCollection<KV<String, Long>> getMostRecentHistoryEntries(
+      Pipeline pipeline, Class<T> historyClass) {
+    String repoIdFieldName = HistoryEntryDao.REPO_ID_FIELD_NAMES.get(historyClass);
+    String resourceFieldName = EPP_RESOURCE_FIELD_NAME.get(historyClass);
+    return pipeline
         .apply(
-            "Reshuffle KV<PendingDeposit, DepositFragment> of "
-                + clazz.getSimpleName()
-                + " to prevent fusion",
-            Reshuffle.of());
+            String.format("Load most recent %s", historyClass.getSimpleName()),
+            RegistryJpaIO.read(
+                ("SELECT %repoIdField%, id FROM %entity% WHERE (%repoIdField%, modificationTime)"
+                     + " IN (SELECT %repoIdField%, MAX(modificationTime) FROM %entity% WHERE"
+                     + " modificationTime <= :watermark GROUP BY %repoIdField%) AND"
+                     + " %resourceField%.deletionTime > :watermark AND"
+                     + " COALESCE(%resourceField%.creationClientId, '') NOT LIKE 'prober-%' AND"
+                     + " COALESCE(%resourceField%.currentSponsorClientId, '') NOT LIKE 'prober-%'"
+                     + " AND COALESCE(%resourceField%.lastEppUpdateClientId, '') NOT LIKE"
+                     + " 'prober-%' "
+                        + (historyClass == DomainHistory.class
+                            ? "AND %resourceField%.tld IN "
+                                + "(SELECT id FROM Tld WHERE tldType = 'REAL')"
+                            : ""))
+                    .replace("%entity%", historyClass.getSimpleName())
+                    .replace("%repoIdField%", repoIdFieldName)
+                    .replace("%resourceField%", resourceFieldName),
+                ImmutableMap.of("watermark", watermark),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1])))
+        .setCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));
   }
 
-  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
-    return pipeline.apply(
-        "Read all production " + clazz.getSimpleName() + " entities",
-        RegistryJpaIO.read(
-            createEppResourceQuery(clazz),
-            clazz.equals(DomainBase.class)
-                ? ImmutableMap.of("tlds", pendings.keySet())
-                : ImmutableMap.of(),
-            String.class,
-            // TODO: consider adding coders for entities and pass them directly instead of using
-            // VKeys.
-            x -> VKey.create(clazz, x)));
+  private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
+      Class<T> historyEntryClazz, String repoId, long revisionId) {
+    try {
+      Class<?> idClazz = historyEntryClazz.getAnnotation(IdClass.class).value();
+      Serializable idObject =
+          (Serializable)
+              idClazz.getConstructor(String.class, long.class).newInstance(repoId, revisionId);
+      return jpaTm()
+          .transact(() -> jpaTm().loadByKey(VKey.createSql(historyEntryClazz, idObject)))
+          .getResourceAtPointInTime()
+          .map(resource -> resource.cloneProjectedAtTime(watermark))
+          .get();
+    } catch (NoSuchMethodException
+        | InvocationTargetException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot load resource from %s with repoId %s and revisionId %s",
+              historyEntryClazz.getSimpleName(), repoId, revisionId),
+          e);
+    }
   }
 
-  <T extends EppResource>
-      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
-    return FlatMapElements.into(
-            TypeDescriptors.kvs(
-                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
-        .via(
-            (VKey<T> key) -> {
-              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
-              // The set of all TLDs to which this resource should be emitted.
-              ImmutableSet<String> tlds =
-                  clazz.equals(DomainBase.class)
-                      ? ImmutableSet.of(((DomainBase) resource).getTld())
-                      : pendings.keySet();
-              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
-              ImmutableSet<DateTime> dates =
-                  tlds.stream()
-                      .map(pendings::get)
-                      .flatMap(ImmutableSet::stream)
-                      .map(PendingDeposit::watermark)
-                      .collect(toImmutableSet());
-              // Launch asynchronous fetches of point-in-time representations of resource.
-              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
-                  ImmutableMap.copyOf(
-                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
-              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
-              // the result.
-              RdeFragmenter fragmenter =
-                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
-              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
-              for (String tld : tlds) {
-                for (PendingDeposit pending : pendings.get(tld)) {
-                  // Hosts and contacts don't get included in BRDA deposits.
-                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
-                    continue;
+  /**
+   * Remove unreferenced resources by joining the (repoId, pendingDeposit) pair with the (repoId,
+   * revisionId) on the repoId.
+   *
+   * <p>The (repoId, pendingDeposit) pairs denote resources (contact, host) that are referenced from
+   * a domain, that are to be included in the corresponding pending deposit.
+   *
+   * <p>The (repoId, revisionId) paris come from the most recent history entry query, which can be
+   * used to load the embedded resources themselves.
+   *
+   * @return a pair of (repoId, ([pendingDeposit], [revisionId])) where neither the pendingDeposit
+   *     nor the revisionId list is empty.
+   */
+  private static PCollection<KV<String, CoGbkResult>> removeUnreferencedResource(
+      PCollection<KV<String, PendingDeposit>> referencedResources,
+      PCollection<KV<String, Long>> historyEntries,
+      Class<? extends EppResource> resourceClazz) {
+    String resourceName = resourceClazz.getSimpleName();
+    Class<? extends HistoryEntry> historyEntryClazz =
+        RESOURCE_TYPES_TO_HISTORY_TYPES.get(resourceClazz);
+    String historyEntryName = historyEntryClazz.getSimpleName();
+    Counter referencedResourceCounter = Metrics.counter("RDE", "Referenced" + resourceName);
+    return KeyedPCollectionTuple.of(PENDING_DEPOSIT, referencedResources)
+        .and(REVISION_ID, historyEntries)
+        .apply(
+            String.format(
+                "Join PendingDeposit with %s revision ID on %s", historyEntryName, resourceName),
+            CoGroupByKey.create())
+        .apply(
+            String.format("Remove unreferenced %s", resourceName),
+            Filter.by(
+                (KV<String, CoGbkResult> kv) -> {
+                  boolean toInclude =
+                      // If a resource does not have corresponding pending deposit, it is not
+                      // referenced and should not be included.
+                      kv.getValue().getAll(PENDING_DEPOSIT).iterator().hasNext()
+                          // If a resource does not have revision id (this should not happen, as
+                          // every referenced resource must be valid at watermark time, therefore
+                          // be embedded in a history entry valid at watermark time, otherwise
+                          // the domain cannot reference it), there is no way for us to find the
+                          // history entry and load the embedded resource. So we ignore the resource
+                          // to keep the downstream process simple.
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedResourceCounter.inc();
                   }
-                  Optional<DepositFragment> fragment =
-                      fragmenter.marshal(pending.watermark(), pending.mode());
-                  fragment.ifPresent(
-                      depositFragment -> results.add(KV.of(pending, depositFragment)));
-                }
-              }
-              return results;
-            });
+                  return toInclude;
+                }));
+  }
+
+  private PCollectionTuple processDomainHistories(PCollection<KV<String, Long>> domainHistories) {
+    Counter activeDomainCounter = Metrics.counter("RDE", "ActiveDomainBase");
+    Counter domainFragmentCounter = Metrics.counter("RDE", "DomainFragment");
+    Counter referencedContactCounter = Metrics.counter("RDE", "ReferencedContactResource");
+    Counter referencedHostCounter = Metrics.counter("RDE", "ReferencedHostResource");
+    return domainHistories.apply(
+        "Map DomainHistory to DepositFragment "
+            + "and emit referenced ContactResource and HostResource",
+        ParDo.of(
+                new DoFn<KV<String, Long>, KV<PendingDeposit, DepositFragment>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Long> kv, MultiOutputReceiver receiver) {
+                    activeDomainCounter.inc();
+                    DomainBase domain =
+                        (DomainBase)
+                            loadResourceByHistoryEntryId(
+                                DomainHistory.class, kv.getKey(), kv.getValue());
+                    pendingDeposits.stream()
+                        .filter(pendingDeposit -> pendingDeposit.tld().equals(domain.getTld()))
+                        .forEach(
+                            pendingDeposit -> {
+                              // Domains are always deposited in both modes.
+                              domainFragmentCounter.inc();
+                              receiver
+                                  .get(DOMAIN_FRAGMENTS)
+                                  .output(
+                                      KV.of(
+                                          pendingDeposit,
+                                          marshaller.marshalDomain(domain, pendingDeposit.mode())));
+                              // Contacts and hosts are only deposited in RDE, not BRDA.
+                              if (pendingDeposit.mode() == RdeMode.FULL) {
+                                HashSet<Serializable> contacts = new HashSet<>();
+                                contacts.add(domain.getAdminContact().getSqlKey());
+                                contacts.add(domain.getTechContact().getSqlKey());
+                                contacts.add(domain.getRegistrant().getSqlKey());
+                                // Billing contact is not mandatory.
+                                if (domain.getBillingContact() != null) {
+                                  contacts.add(domain.getBillingContact().getSqlKey());
+                                }
+                                referencedContactCounter.inc(contacts.size());
+                                contacts.forEach(
+                                    contactRepoId ->
+                                        receiver
+                                            .get(REFERENCED_CONTACTS)
+                                            .output(KV.of((String) contactRepoId, pendingDeposit)));
+                                if (domain.getNsHosts() != null) {
+                                  referencedHostCounter.inc(domain.getNsHosts().size());
+                                  domain
+                                      .getNsHosts()
+                                      .forEach(
+                                          hostKey ->
+                                              receiver
+                                                  .get(REFERENCED_HOSTS)
+                                                  .output(
+                                                      KV.of(
+                                                          (String) hostKey.getSqlKey(),
+                                                          pendingDeposit)));
+                                }
+                              }
+                            });
+                  }
+                })
+            .withOutputTags(
+                DOMAIN_FRAGMENTS, TupleTagList.of(REFERENCED_CONTACTS).and(REFERENCED_HOSTS)));
+  }
+
+  private PCollection<KV<PendingDeposit, DepositFragment>> processContactHistories(
+      PCollection<KV<String, PendingDeposit>> referencedContacts,
+      PCollection<KV<String, Long>> contactHistories) {
+    Counter contactFragmentCounter = Metrics.counter("RDE", "ContactFragment");
+    return removeUnreferencedResource(referencedContacts, contactHistories, ContactResource.class)
+        .apply(
+            "Map ContactResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      ContactResource contact =
+                          (ContactResource)
+                              loadResourceByHistoryEntryId(
+                                  ContactHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      DepositFragment fragment = marshaller.marshalContact(contact);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same contact could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .map(pendingDeposit -> KV.of(pendingDeposit, fragment))
+                              .collect(toImmutableSet());
+                      contactFragmentCounter.inc(fragments.size());
+                      return fragments;
+                    }));
+  }
+
+  private PCollectionTuple processHostHistories(
+      PCollection<KV<String, PendingDeposit>> referencedHosts,
+      PCollection<KV<String, Long>> hostHistories) {
+    Counter subordinateHostCounter = Metrics.counter("RDE", "SubordinateHostResource");
+    Counter externalHostCounter = Metrics.counter("RDE", "ExternalHostResource");
+    Counter externalHostFragmentCounter = Metrics.counter("RDE", "ExternalHostFragment");
+    return removeUnreferencedResource(referencedHosts, hostHistories, HostResource.class)
+        .apply(
+            "Map external DomainResource to DepositFragment and process subordinate domains",
+            ParDo.of(
+                    new DoFn<KV<String, CoGbkResult>, KV<PendingDeposit, DepositFragment>>() {
+                      @ProcessElement
+                      public void processElement(
+                          @Element KV<String, CoGbkResult> kv, MultiOutputReceiver receiver) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    kv.getKey(),
+                                    kv.getValue().getOnly(REVISION_ID));
+                        // When a host is subordinate, we need to find it's superordinate domain and
+                        // include it in the deposit as well.
+                        if (host.isSubordinate()) {
+                          subordinateHostCounter.inc();
+                          receiver
+                              .get(SUPERORDINATE_DOMAINS)
+                              .output(
+                                  // The output are pairs of
+                                  // (superordinateDomainRepoId,
+                                  //   (subordinateHostRepoId, (pendingDeposit, revisionId))).
+                                  KV.of((String) host.getSuperordinateDomain().getSqlKey(), kv));
+                        } else {
+                          externalHostCounter.inc();
+                          DepositFragment fragment = marshaller.marshalExternalHost(host);
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same host could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .forEach(
+                                  pendingDeposit -> {
+                                    externalHostFragmentCounter.inc();
+                                    receiver
+                                        .get(EXTERNAL_HOST_FRAGMENTS)
+                                        .output(KV.of(pendingDeposit, fragment));
+                                  });
+                        }
+                      }
+                    })
+                .withOutputTags(EXTERNAL_HOST_FRAGMENTS, TupleTagList.of(SUPERORDINATE_DOMAINS)));
+  }
+
+  /**
+   * Process subordinate hosts by making a deposit fragment with pending transfer information
+   * obtained from its superordinate domain.
+   *
+   * @param superordinateDomains Pairs of (superordinateDomainRepoId, (subordinateHostRepoId,
+   *     (pendingDeposit, revisionId))). This collection maps the subordinate host and the pending
+   *     deposit to include it to its superordinate domain.
+   * @param domainHistories Pairs of (domainRepoId, revisionId). This collection helps us find the
+   *     historical superordinate domain from its history entry and is obtained from calling {@link
+   *     #getMostRecentHistoryEntries} for domains.
+   */
+  private PCollection<KV<PendingDeposit, DepositFragment>> processSubordinateHosts(
+      PCollection<KV<String, KV<String, CoGbkResult>>> superordinateDomains,
+      PCollection<KV<String, Long>> domainHistories) {
+    Counter subordinateHostFragmentCounter = Metrics.counter("RDE", "SubordinateHostFragment");
+    Counter referencedSubordinateHostCounter = Metrics.counter("RDE", "ReferencedSubordinateHost");
+    return KeyedPCollectionTuple.of(HOST_TO_PENDING_DEPOSIT, superordinateDomains)
+        .and(REVISION_ID, domainHistories)
+        .apply(
+            "Join HostResource:PendingDeposits with DomainHistory on DomainResource",
+            CoGroupByKey.create())
+        .apply(
+            " Remove unreferenced DomainResource",
+            Filter.by(
+                kv -> {
+                  boolean toInclude =
+                      kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT).iterator().hasNext()
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedSubordinateHostCounter.inc();
+                  }
+                  return toInclude;
+                }))
+        .apply(
+            "Map subordinate HostResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      DomainBase superordinateDomain =
+                          (DomainBase)
+                              loadResourceByHistoryEntryId(
+                                  DomainHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      ImmutableSet.Builder<KV<PendingDeposit, DepositFragment>> results =
+                          new ImmutableSet.Builder<>();
+                      for (KV<String, CoGbkResult> hostToPendingDeposits :
+                          kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT)) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    hostToPendingDeposits.getKey(),
+                                    hostToPendingDeposits.getValue().getOnly(REVISION_ID));
+                        DepositFragment fragment =
+                            marshaller.marshalSubordinateHost(host, superordinateDomain);
+                        Streams.stream(hostToPendingDeposits.getValue().getAll(PENDING_DEPOSIT))
+                            .distinct()
+                            .forEach(
+                                pendingDeposit -> {
+                                  subordinateHostFragmentCounter.inc();
+                                  results.add(KV.of(pendingDeposit, fragment));
+                                });
+                      }
+                      return results.build();
+                    }));
   }
 
   /**
    * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
-   * the original TLD to pending deposit map.
+   * the original pending deposit set.
    */
   @SuppressWarnings("unchecked")
-  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+  static ImmutableSet<PendingDeposit> decodePendingDeposits(String encodedPendingDeposits) {
     try (ObjectInputStream ois =
         new ObjectInputStream(
             new ByteArrayInputStream(
-                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
-      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+                BaseEncoding.base64Url().omitPadding().decode(encodedPendingDeposits)))) {
+      return (ImmutableSet<PendingDeposit>) ois.readObject();
     } catch (IOException | ClassNotFoundException e) {
       throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
     }
   }
 
   /**
-   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
-   * worker by the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
+   * the pipeline launcher as a pipeline option.
    */
-  static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
-      oos.writeObject(pendings);
+      oos.writeObject(pendingDeposits);
       oos.flush();
       return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
     }
@@ -282,13 +684,50 @@ public class RdePipeline implements Serializable {
 
   public static void main(String[] args) throws IOException, ClassNotFoundException {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
-    RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
+    RdePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
+  /**
+   * A utility class that contains {@link TupleTag}s when {@link PCollectionTuple}s and {@link
+   * CoGbkResult}s are used.
+   */
+  protected abstract static class TupleTags {
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> DOMAIN_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_CONTACTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_HOSTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, KV<String, CoGbkResult>>> SUPERORDINATE_DOMAINS =
+        new TupleTag<KV<String, KV<String, CoGbkResult>>>() {};
+
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> EXTERNAL_HOST_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<PendingDeposit> PENDING_DEPOSIT =
+        new TupleTag<PendingDeposit>() {};
+
+    protected static final TupleTag<KV<String, CoGbkResult>> HOST_TO_PENDING_DEPOSIT =
+        new TupleTag<KV<String, CoGbkResult>>() {};
+
+    protected static final TupleTag<Long> REVISION_ID = new TupleTag<Long>() {};
+  }
+
   @Singleton
-  @Component(modules = {CredentialModule.class, ConfigModule.class})
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
   interface RdePipelineComponent {
     RdePipeline rdePipeline();
 

core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java

@@ -31,9 +31,9 @@ public interface RdePipelineOptions extends RegistryPipelineOptions {
   void setValidationMode(String value);
 
   @Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
-  String getGcsBucket();
+  String getRdeStagingBucket();
 
-  void setGcsBucket(String value);
+  void setRdeStagingBucket(String value);
 
   @Description("The Base64-encoded PGP public key to encrypt the deposits.")
   String getStagingKey();

core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java

@@ -25,7 +25,7 @@ import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 
 /**
- * A POJO representing a single subdomain, parsed from a {@code SchemaAndRecord}.
+ * A POJO representing a domain name and associated info, parsed from a {@code SchemaAndRecord}.
  *
  * <p>This is a trivially serializable class that allows Beam to transform the results of a Bigquery
  * query into a standard Java representation, giving us the type guarantees and ease of manipulation
@@ -33,28 +33,31 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
  * function.
  */
 @AutoValue
-public abstract class Subdomain implements Serializable {
+public abstract class DomainNameInfo implements Serializable {
 
   private static final ImmutableList<String> FIELD_NAMES =
       ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");
 
   /** Returns the fully qualified domain name. */
   abstract String domainName();
+
   /** Returns the domain repo ID (the primary key of the domain table). */
   abstract String domainRepoId();
+
   /** Returns the registrar ID of the associated registrar for this domain. */
   abstract String registrarId();
+
   /** Returns the email address of the registrar associated with this domain. */
   abstract String registrarEmailAddress();
 
   /**
-   * Constructs a {@link Subdomain} from an Apache Avro {@code SchemaAndRecord}.
+   * Constructs a {@link DomainNameInfo} from an Apache Avro {@code SchemaAndRecord}.
    *
    * @see <a
    *     href=http://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/generic/GenericData.Record.html>
    *     Apache AVRO GenericRecord</a>
    */
-  static Subdomain parseFromRecord(SchemaAndRecord schemaAndRecord) {
+  static DomainNameInfo parseFromRecord(SchemaAndRecord schemaAndRecord) {
     checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
     GenericRecord record = schemaAndRecord.getRecord();
     return create(
@@ -65,18 +68,15 @@ public abstract class Subdomain implements Serializable {
   }
 
   /**
-   * Creates a concrete {@link Subdomain}.
+   * Creates a concrete {@link DomainNameInfo}.
    *
-   * <p>This should only be used outside this class for testing- instances of {@link Subdomain}
+   * <p>This should only be used outside this class for testing- instances of {@link DomainNameInfo}
    * should otherwise come from {@link #parseFromRecord}.
    */
   @VisibleForTesting
-  static Subdomain create(
-      String domainName,
-      String domainRepoId,
-      String registrarId,
-      String registrarEmailAddress) {
-    return new AutoValue_Subdomain(
+  static DomainNameInfo create(
+      String domainName, String domainRepoId, String registrarId, String registrarEmailAddress) {
+    return new AutoValue_DomainNameInfo(
         domainName, domainRepoId, registrarId, registrarEmailAddress);
   }
 }

core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java

@@ -55,13 +55,14 @@ public class SafeBrowsingTransforms {
       "https://safebrowsing.googleapis.com/v4/threatMatches:find";
 
   /**
-   * {@link DoFn} mapping a {@link Subdomain} to its evaluation report from SafeBrowsing.
+   * {@link DoFn} mapping a {@link DomainNameInfo} to its evaluation report from SafeBrowsing.
    *
    * <p>Refer to the Lookup API documentation for the request/response format and other details.
    *
    * @see <a href=https://developers.google.com/safe-browsing/v4/lookup-api>Lookup API</a>
    */
-  static class EvaluateSafeBrowsingFn extends DoFn<Subdomain, KV<Subdomain, ThreatMatch>> {
+  static class EvaluateSafeBrowsingFn
+      extends DoFn<DomainNameInfo, KV<DomainNameInfo, ThreatMatch>> {
 
     /**
      * Max number of urls we can check in a single query.
@@ -74,10 +75,11 @@ public class SafeBrowsingTransforms {
     private final String apiKey;
 
     /**
-     * Maps a subdomain's {@code fullyQualifiedDomainName} to its corresponding {@link Subdomain} to
-     * facilitate batching SafeBrowsing API requests.
+     * Maps a domain name's {@code fullyQualifiedDomainName} to its corresponding {@link
+     * DomainNameInfo} to facilitate batching SafeBrowsing API requests.
      */
-    private final Map<String, Subdomain> subdomainBuffer = new LinkedHashMap<>(BATCH_SIZE);
+    private final Map<String, DomainNameInfo> domainNameInfoBuffer =
+        new LinkedHashMap<>(BATCH_SIZE);
 
     /**
      * Provides the HTTP client we use to interact with the SafeBrowsing API.
@@ -119,37 +121,38 @@ public class SafeBrowsingTransforms {
       closeableHttpClientSupplier = clientSupplier;
     }
 
-    /** Evaluates any buffered {@link Subdomain} objects upon completing the bundle. */
+    /** Evaluates any buffered {@link DomainNameInfo} objects upon completing the bundle. */
     @FinishBundle
     public void finishBundle(FinishBundleContext context) {
-      if (!subdomainBuffer.isEmpty()) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      if (!domainNameInfoBuffer.isEmpty()) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
         results.forEach((kv) -> context.output(kv, Instant.now(), GlobalWindow.INSTANCE));
       }
     }
 
     /**
-     * Buffers {@link Subdomain} objects until we reach the batch size, then bulk-evaluate the URLs
-     * with the SafeBrowsing API.
+     * Buffers {@link DomainNameInfo} objects until we reach the batch size, then bulk-evaluate the
+     * URLs with the SafeBrowsing API.
      */
     @ProcessElement
     public void processElement(ProcessContext context) {
-      Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.domainName(), subdomain);
-      if (subdomainBuffer.size() >= BATCH_SIZE) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      DomainNameInfo domainNameInfo = context.element();
+      domainNameInfoBuffer.put(domainNameInfo.domainName(), domainNameInfo);
+      if (domainNameInfoBuffer.size() >= BATCH_SIZE) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
         results.forEach(context::output);
       }
     }
 
     /**
-     * Evaluates all {@link Subdomain} objects in the buffer and returns a list of key-value pairs
-     * from {@link Subdomain} to its SafeBrowsing report.
+     * Evaluates all {@link DomainNameInfo} objects in the buffer and returns a list of key-value
+     * pairs from {@link DomainNameInfo} to its SafeBrowsing report.
      *
-     * <p>If a {@link Subdomain} is safe according to the API, it will not emit a report.
+     * <p>If a {@link DomainNameInfo} is safe according to the API, it will not emit a report.
      */
-    private ImmutableSet<KV<Subdomain, ThreatMatch>> evaluateAndFlush() {
-      ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder = new ImmutableSet.Builder<>();
+    private ImmutableSet<KV<DomainNameInfo, ThreatMatch>> evaluateAndFlush() {
+      ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder =
+          new ImmutableSet.Builder<>();
       try {
         URIBuilder uriBuilder = new URIBuilder(SAFE_BROWSING_URL);
         // Add the API key param
@@ -174,7 +177,7 @@ public class SafeBrowsingTransforms {
         throw new RuntimeException("Caught parsing exception, failing pipeline.", e);
       } finally {
         // Flush the buffer
-        subdomainBuffer.clear();
+        domainNameInfoBuffer.clear();
       }
       return resultBuilder.build();
     }
@@ -183,7 +186,7 @@ public class SafeBrowsingTransforms {
     private JSONObject createRequestBody() throws JSONException {
       // Accumulate all domain names to evaluate.
       JSONArray threatArray = new JSONArray();
-      for (String fullyQualifiedDomainName : subdomainBuffer.keySet()) {
+      for (String fullyQualifiedDomainName : domainNameInfoBuffer.keySet()) {
         threatArray.put(new JSONObject().put("url", fullyQualifiedDomainName));
       }
       // Construct the JSON request body
@@ -211,11 +214,11 @@ public class SafeBrowsingTransforms {
      */
     private void processResponse(
         CloseableHttpResponse response,
-        ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder)
+        ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder)
         throws JSONException, IOException {
       int statusCode = response.getStatusLine().getStatusCode();
       if (statusCode != SC_OK) {
-        logger.atWarning().log("Got unexpected status code %s from response", statusCode);
+        logger.atWarning().log("Got unexpected status code %s from response.", statusCode);
       } else {
         // Unpack the response body
         JSONObject responseBody =
@@ -224,18 +227,19 @@ public class SafeBrowsingTransforms {
                     new InputStreamReader(response.getEntity().getContent(), UTF_8)));
         logger.atInfo().log("Got response: %s", responseBody.toString());
         if (responseBody.length() == 0) {
-          logger.atInfo().log("Response was empty, no threats detected");
+          logger.atInfo().log("Response was empty, no threats detected.");
         } else {
-          // Emit all Subdomains with their API results.
+          // Emit all DomainNameInfos with their API results.
           JSONArray threatMatches = responseBody.getJSONArray("matches");
           for (int i = 0; i < threatMatches.length(); i++) {
             JSONObject match = threatMatches.getJSONObject(i);
             String url = match.getJSONObject("threat").getString("url");
-            Subdomain subdomain = subdomainBuffer.get(url);
+            DomainNameInfo domainNameInfo = domainNameInfoBuffer.get(url);
             resultBuilder.add(
                 KV.of(
-                    subdomain,
-                    ThreatMatch.create(match.getString("threatType"), subdomain.domainName())));
+                    domainNameInfo,
+                    ThreatMatch.create(
+                        match.getString("threatType"), domainNameInfo.domainName())));
           }
         }
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -16,6 +16,7 @@ package google.registry.beam.spec11;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
@@ -30,6 +31,7 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.reporting.Spec11ThreatMatch;
 import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
 import google.registry.util.UtilsModule;
@@ -41,6 +43,7 @@ import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -58,7 +61,8 @@ import org.json.JSONObject;
 /**
  * Definition of a Dataflow Flex template, which generates a given month's spec11 report.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=spec11}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
@@ -100,86 +104,106 @@ public class Spec11Pipeline implements Serializable {
 
   void setupPipeline(Pipeline pipeline) {
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
-    PCollection<Subdomain> domains =
+    PCollection<DomainNameInfo> domains =
         options.getDatabase().equals("DATASTORE")
             ? readFromBigQuery(options, pipeline)
             : readFromCloudSql(pipeline);
 
-    PCollection<KV<Subdomain, ThreatMatch>> threatMatches =
+    PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches =
         domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn));
 
     saveToSql(threatMatches, options);
     saveToGcs(threatMatches, options);
   }
 
-  static PCollection<Subdomain> readFromCloudSql(Pipeline pipeline) {
-    Read<Object[], Subdomain> read =
+  static PCollection<DomainNameInfo> readFromCloudSql(Pipeline pipeline) {
+    Read<Object[], KV<String, String>> read =
         RegistryJpaIO.read(
-            "select d, r.emailAddress from Domain d join Registrar r on"
-                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
-                + " and d.deletionTime > now()",
+            "select d.repoId, r.emailAddress from Domain d join Registrar r on"
+                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL' and"
+                + " d.deletionTime > now()",
             false,
             Spec11Pipeline::parseRow);
 
-    return pipeline.apply("Read active domains from Cloud SQL", read);
+    return pipeline
+        .apply("Read active domains from Cloud SQL", read)
+        .apply(
+            "Build DomainNameInfo",
+            ParDo.of(
+                new DoFn<KV<String, String>, DomainNameInfo>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, String> input, OutputReceiver<DomainNameInfo> output) {
+                    DomainBase domainBase =
+                        jpaTm()
+                            .transact(
+                                () ->
+                                    jpaTm()
+                                        .loadByKey(
+                                            VKey.createSql(DomainBase.class, input.getKey())));
+                    String emailAddress = input.getValue();
+                    if (emailAddress == null) {
+                      emailAddress = "";
+                    }
+                    DomainNameInfo domainNameInfo =
+                        DomainNameInfo.create(
+                            domainBase.getDomainName(),
+                            domainBase.getRepoId(),
+                            domainBase.getCurrentSponsorRegistrarId(),
+                            emailAddress);
+                    output.output(domainNameInfo);
+                  }
+                }));
   }
 
-  static PCollection<Subdomain> readFromBigQuery(Spec11PipelineOptions options, Pipeline pipeline) {
+  static PCollection<DomainNameInfo> readFromBigQuery(
+      Spec11PipelineOptions options, Pipeline pipeline) {
     return pipeline.apply(
         "Read active domains from BigQuery",
-        BigQueryIO.read(Subdomain::parseFromRecord)
+        BigQueryIO.read(DomainNameInfo::parseFromRecord)
             .fromQuery(
-                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
+                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "domain_name_infos.sql"))
                     .put("PROJECT_ID", options.getProject())
                     .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
                     .put("REGISTRAR_TABLE", "Registrar")
                     .put("DOMAIN_BASE_TABLE", "DomainBase")
                     .build())
-            .withCoder(SerializableCoder.of(Subdomain.class))
+            .withCoder(SerializableCoder.of(DomainNameInfo.class))
             .usingStandardSql()
             .withoutValidation()
             .withTemplateCompatibility());
   }
 
-  private static Subdomain parseRow(Object[] row) {
-    DomainBase domainBase = (DomainBase) row[0];
-    String emailAddress = (String) row[1];
-    if (emailAddress == null) {
-      emailAddress = "";
-    }
-    return Subdomain.create(
-        domainBase.getDomainName(),
-        domainBase.getRepoId(),
-        domainBase.getCurrentSponsorClientId(),
-        emailAddress);
+  private static KV<String, String> parseRow(Object[] row) {
+    return KV.of((String) row[0], (String) row[1]);
   }
 
   static void saveToSql(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
     String transformId = "Spec11 Threat Matches";
     LocalDate date = LocalDate.parse(options.getDate(), ISODateTimeFormat.date());
     threatMatches.apply(
         "Write to Sql: " + transformId,
-        RegistryJpaIO.<KV<Subdomain, ThreatMatch>>write()
+        RegistryJpaIO.<KV<DomainNameInfo, ThreatMatch>>write()
             .withName(transformId)
             .withBatchSize(options.getSqlWriteBatchSize())
             .withShards(options.getSqlWriteShards())
             .withJpaConverter(
                 (kv) -> {
-                  Subdomain subdomain = kv.getKey();
+                  DomainNameInfo domainNameInfo = kv.getKey();
                   return new Spec11ThreatMatch.Builder()
                       .setThreatTypes(
                           ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
                       .setCheckDate(date)
-                      .setDomainName(subdomain.domainName())
-                      .setDomainRepoId(subdomain.domainRepoId())
-                      .setRegistrarId(subdomain.registrarId())
+                      .setDomainName(domainNameInfo.domainName())
+                      .setDomainRepoId(domainNameInfo.domainRepoId())
+                      .setRegistrarId(domainNameInfo.registrarId())
                       .build();
                 }));
   }
 
   static void saveToGcs(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
     threatMatches
         .apply(
             "Map registrar ID to email/ThreatMatch pair",
@@ -187,7 +211,7 @@ public class Spec11Pipeline implements Serializable {
                     TypeDescriptors.kvs(
                         TypeDescriptors.strings(), TypeDescriptor.of(EmailAndThreatMatch.class)))
                 .via(
-                    (KV<Subdomain, ThreatMatch> kv) ->
+                    (KV<DomainNameInfo, ThreatMatch> kv) ->
                         KV.of(
                             kv.getKey().registrarId(),
                             EmailAndThreatMatch.create(
@@ -198,15 +222,15 @@ public class Spec11Pipeline implements Serializable {
             MapElements.into(TypeDescriptors.strings())
                 .via(
                     (KV<String, Iterable<EmailAndThreatMatch>> kv) -> {
-                      String clientId = kv.getKey();
+                      String registrarId = kv.getKey();
                       checkArgument(
                           kv.getValue().iterator().hasNext(),
                           String.format(
-                              "Registrar with ID %s had no corresponding threats", clientId));
+                              "Registrar with ID %s had no corresponding threats", registrarId));
                       String email = kv.getValue().iterator().next().email();
                       JSONObject output = new JSONObject();
                       try {
-                        output.put(REGISTRAR_CLIENT_ID_FIELD, clientId);
+                        output.put(REGISTRAR_CLIENT_ID_FIELD, registrarId);
                         output.put(REGISTRAR_EMAIL_FIELD, email);
                         JSONArray threatMatchArray = new JSONArray();
                         for (EmailAndThreatMatch emailAndThreatMatch : kv.getValue()) {
@@ -230,7 +254,7 @@ public class Spec11Pipeline implements Serializable {
                         options.getReportingBucketUrl(),
                         getSpec11ReportFilePath(LocalDate.parse(options.getDate()))))
                 .withoutSharding()
-                .withHeader("Map from registrar email / name to detected subdomain threats:"));
+                .withHeader("Map from registrar email / name to detected domain name threats:"));
   }
 
   public static void main(String[] args) {

core/src/main/java/google/registry/bigquery/BigqueryConnection.java

@@ -632,7 +632,7 @@ public class BigqueryConnection implements AutoCloseable {
   private static String summarizeCompletedJob(Job job) {
     JobStatistics stats = job.getStatistics();
     return String.format(
-        "Job took %,.3f seconds after a %,.3f second delay and processed %,d bytes (%s)",
+        "Job took %,.3f seconds after a %,.3f second delay and processed %,d bytes (%s).",
         (stats.getEndTime() - stats.getStartTime()) / 1000.0,
         (stats.getStartTime() - stats.getCreationTime()) / 1000.0,
         stats.getTotalBytesProcessed(),
@@ -706,17 +706,17 @@ public class BigqueryConnection implements AutoCloseable {
   }
 
   /**
-   * Helper that creates a dataset with this name if it doesn't already exist, and returns true
-   * if creation took place.
+   * Helper that creates a dataset with this name if it doesn't already exist, and returns true if
+   * creation took place.
    */
-  public boolean createDatasetIfNeeded(String datasetName) throws IOException {
+  private boolean createDatasetIfNeeded(String datasetName) throws IOException {
     if (!checkDatasetExists(datasetName)) {
       bigquery.datasets()
           .insert(getProjectId(), new Dataset().setDatasetReference(new DatasetReference()
               .setProjectId(getProjectId())
               .setDatasetId(datasetName)))
           .execute();
-      logger.atInfo().log("Created dataset: %s:%s\n", getProjectId(), datasetName);
+      logger.atInfo().log("Created dataset: %s: %s.", getProjectId(), datasetName);
       return true;
     }
     return false;
@@ -732,9 +732,8 @@ public class BigqueryConnection implements AutoCloseable {
                   .setDefaultDataset(getDataset())
                   .setDestinationTable(table))));
     } catch (BigqueryJobFailureException e) {
-      if (e.getReason().equals("duplicate")) {
-        // Table already exists.
-      } else {
+      if (!e.getReason().equals("duplicate")) {
+        // Throw if it failed for any reason other than table already existing.
         throw e;
       }
     }

core/src/main/java/google/registry/bigquery/CheckedBigquery.java

@@ -116,7 +116,7 @@ public class CheckedBigquery {
           .setTableReference(table))
           .execute();
       logger.atInfo().log(
-          "Created BigQuery table %s:%s.%s",
+          "Created BigQuery table %s:%s.%s.",
           table.getProjectId(), table.getDatasetId(), table.getTableId());
     } catch (IOException e) {
       // Swallow errors about a table that exists, and throw any other ones.

core/src/main/java/google/registry/config/CloudTasksUtilsModule.java

@@ -0,0 +1,79 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.config;
+
+import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.cloud.tasks.v2.CloudTasksClient;
+import com.google.cloud.tasks.v2.CloudTasksSettings;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.CloudTasksUtils;
+import google.registry.util.CloudTasksUtils.GcpCloudTasksClient;
+import google.registry.util.CloudTasksUtils.SerializableCloudTasksClient;
+import google.registry.util.GoogleCredentialsBundle;
+import google.registry.util.Retrier;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.function.Supplier;
+import javax.inject.Singleton;
+
+/**
+ * A {@link Module} that provides {@link CloudTasksUtils}.
+ *
+ * <p>The class itself cannot be annotated as {@code Inject} because its requested dependencies use
+ * the {@link Config} qualifier which is not available in the {@code util} package.
+ */
+@Module
+public abstract class CloudTasksUtilsModule {
+
+  @Singleton
+  @Provides
+  public static CloudTasksUtils provideCloudTasksUtils(
+      @Config("projectId") String projectId,
+      @Config("locationId") String locationId,
+      SerializableCloudTasksClient client,
+      Retrier retrier) {
+    return new CloudTasksUtils(retrier, projectId, locationId, client);
+  }
+
+  // Provides a supplier instead of using a Dagger @Provider because the latter is not serializable.
+  @Provides
+  public static Supplier<CloudTasksClient> provideCloudTasksClientSupplier(
+      @DefaultCredential GoogleCredentialsBundle credentials) {
+    return (Supplier<CloudTasksClient> & Serializable)
+        () -> {
+          CloudTasksClient client;
+          try {
+            client =
+                CloudTasksClient.create(
+                    CloudTasksSettings.newBuilder()
+                        .setCredentialsProvider(
+                            FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
+                        .build());
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+          return client;
+        };
+  }
+
+  @Provides
+  public static SerializableCloudTasksClient provideSerializableCloudTasksClient(
+      final Supplier<CloudTasksClient> clientSupplier) {
+    return new GcpCloudTasksClient(clientSupplier);
+  }
+}

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -105,6 +105,12 @@ public final class RegistryConfig {
       return config.appEngine.projectId;
     }
 
+    @Provides
+    @Config("locationId")
+    public static String provideLocationId(RegistryConfigSettings config) {
+      return config.appEngine.locationId;
+    }
+
     /**
      * The filename of the logo to be displayed in the header of the registrar console.
      *
@@ -551,11 +557,23 @@ public final class RegistryConfig {
       return config.registryPolicy.requireSslCertificates;
     }
 
+    /**
+     * Returns the GCE machine type that a CPU-demanding pipeline should use.
+     *
+     * @see google.registry.beam.rde.RdePipeline
+     */
+    @Provides
+    @Config("highPerformanceMachineType")
+    public static String provideHighPerformanceMachineType(RegistryConfigSettings config) {
+      return config.beam.highPerformanceMachineType;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
      * @see google.registry.beam.invoicing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
+     * @see google.registry.beam.invoicing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")
@@ -1132,8 +1150,8 @@ public final class RegistryConfig {
     }
 
     /**
-     * Returns the clientId of the registrar that admins are automatically logged in as if they
-     * aren't otherwise associated with one.
+     * Returns the ID of the registrar that admins are automatically logged in as if they aren't
+     * otherwise associated with one.
      */
     @Provides
     @Config("registryAdminClientId")
@@ -1300,6 +1318,18 @@ public final class RegistryConfig {
     public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
       return ImmutableSet.copyOf(config.sslCertificateValidation.allowedEcdsaCurves);
     }
+
+    @Provides
+    @Config("minMonthsBeforeWipeOut")
+    public static int provideMinMonthsBeforeWipeOut(RegistryConfigSettings config) {
+      return config.contactHistory.minMonthsBeforeWipeOut;
+    }
+
+    @Provides
+    @Config("wipeOutQueryBatchSize")
+    public static int provideWipeOutQueryBatchSize(RegistryConfigSettings config) {
+      return config.contactHistory.wipeOutQueryBatchSize;
+    }
   }
 
   /** Returns the App Engine project ID, which is based off the environment name. */

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -41,10 +41,12 @@ public class RegistryConfigSettings {
   public Keyring keyring;
   public RegistryTool registryTool;
   public SslCertificateValidation sslCertificateValidation;
+  public ContactHistory contactHistory;
 
   /** Configuration options that apply to the entire App Engine project. */
   public static class AppEngine {
     public String projectId;
+    public String locationId;
     public boolean isLocal;
     public String defaultServiceUrl;
     public String backendServiceUrl;
@@ -131,6 +133,7 @@ public class RegistryConfigSettings {
   /** Configuration for Apache Beam (Cloud Dataflow). */
   public static class Beam {
     public String defaultJobRegion;
+    public String highPerformanceMachineType;
     public String stagingBucketUrl;
   }
 
@@ -233,4 +236,10 @@ public class RegistryConfigSettings {
     public String expirationWarningEmailBodyText;
     public String expirationWarningEmailSubjectText;
   }
+
+  /** Configuration for contact history. */
+  public static class ContactHistory {
+    public int minMonthsBeforeWipeOut;
+    public int wipeOutQueryBatchSize;
+  }
 }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -8,6 +8,10 @@
 appEngine:
   # Globally unique App Engine project ID
   projectId: registry-project-id
+  # Location of the App engine project, note that us-central1 and europe-west1 are special in that
+  # they are used without the trailing number in App Engine commands and Google Cloud Console.
+  # See: https://cloud.google.com/appengine/docs/locations
+  locationId: registry-location-id
 
   # whether to use local/test credentials when connecting to the servers
   isLocal: true
@@ -415,6 +419,14 @@ misc:
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
   defaultJobRegion: us-east1
+  # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
+  # to check the VM CPU quota for the job region. In a massively parallel
+  # pipeline this quota can be easily reached and needs to be raised, otherwise
+  # the job will run very slowly. Also note that there is a separate quota for
+  # external IPv4 address in a region, which means that machine type with higher
+  # core count per machine may be preferable in order to preserve IP addresses.
+  # See: https://cloud.google.com/compute/quotas#cpu_quota
+  highPerformanceMachineType: n2-standard-4
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:
@@ -438,6 +450,13 @@ registryTool:
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
 
+# Configuration options for handling contact history.
+contactHistory:
+  # The number of months that a ContactHistory entity should be stored in the database.
+  minMonthsBeforeWipeOut: 18
+  # The batch size for querying ContactHistory table in the database.
+  wipeOutQueryBatchSize: 500
+
 # Configuration options for checking SSL certificates.
 sslCertificateValidation:
   # A map specifying the maximum amount of days the certificate can be valid.
@@ -452,12 +471,36 @@ sslCertificateValidation:
   # The minimum number of days between two successive expiring notification emails.
   expirationWarningIntervalDays: 15
   # Text for expiring certificate notification email subject.
-  expirationWarningEmailSubjectText: Certificate Expring Within 30 Days.
+  expirationWarningEmailSubjectText: "[Important] Expiring SSL certificate for Google Registry EPP connection"
   # Text for expiring certificate notification email body that accepts 3 parameters:
   # registrar name, certificate type, and expiration date, respectively.
   expirationWarningEmailBodyText: |
-    Hello Registrar %s,
-       The %s certificate is expiring on %s.
+    Dear %1$s,
+
+    We would like to inform you that your %2$s SSL certificate will expire at %3$s. Please take note that using expired certificates will prevent successful Registry login.
+
+    Kindly update your production account certificate within the support console using the following steps:
+
+      1. Navigate to support.registry.google and login using your %4$s@registry.google credentials.
+          * If this is your first time logging in, you will be prompted to reset your password, so please keep your new password safe.
+          * If you are already logged in with some other Google account(s) but not your %4$s@registry.google account, you need to click on
+          “Add Account” and login using your %4$s@registry.google credentials.
+      2. Select “Settings > Security” from the left navigation bar.
+      3. Click “Edit” on the top left corner.
+      4. Enter your full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----) in the box.
+      5. Click “Save”. If there are validation issues with the form, you will be prompted to fix them and click “Save” again.
+
+    A failover SSL certificate can also be added in order to prevent connection issues once your main certificate expires. Connecting with either of the certificates will work with our production EPP server.
+
+    Further information about our EPP connection requirements can be found in section 9.2 in the updated Technical Guide in your Google Drive folder.
+
+    Note that account certificate changes take a few minutes to become effective and that the existing connections will remain unaffected by the change.
+
+    If you also would like to update your OT&E account certificate, please send an email from your primary or technical contact to registry-support@google.com and include the full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).
+
+    Regards,
+    Google Registry
+
   # The minimum number of bits an RSA key must contain.
   minimumRsaKeyLength: 2048
   # The ECDSA curves that are allowed for public keys.

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -14,18 +14,15 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
-import java.time.Duration;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import javax.inject.Inject;
 
 /** Action for fanning out cron tasks for each commit log bucket. */
@@ -38,25 +35,27 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  private static final Random random = new Random();
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
-  @Inject TaskQueueUtils taskQueueUtils;
   @Inject @Parameter("endpoint") String endpoint;
   @Inject @Parameter("queue") String queue;
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
+
+
   @Override
   public void run() {
-    Queue taskQueue = getQueue(queue);
     for (int bucketId : CommitLogBucket.getBucketIds()) {
-      long delay =
-          jitterSeconds.map(i -> random.nextInt((int) Duration.ofSeconds(i).toMillis())).orElse(0);
-      TaskOptions taskOptions =
-          TaskOptions.Builder.withUrl(endpoint)
-              .param(BUCKET_PARAM, Integer.toString(bucketId))
-              .countdownMillis(delay);
-      taskQueueUtils.enqueue(taskQueue, taskOptions);
+      cloudTasksUtils.enqueue(
+          queue,
+          CloudTasksUtils.createPostTask(
+              endpoint,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
+              clock,
+              jitterSeconds));
     }
   }
 }

core/src/main/java/google/registry/cron/TldFanoutAction.java

@@ -14,14 +14,10 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Predicates.in;
 import static com.google.common.base.Predicates.not;
-import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.collect.Iterables.getFirst;
 import static com.google.common.collect.Multimaps.filterKeys;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.cron.CronModule.ENDPOINT_PARAM;
@@ -34,23 +30,23 @@ import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
 import static google.registry.model.tld.Registries.getTldsOfType;
 import static google.registry.model.tld.Registry.TldType.REAL;
 import static google.registry.model.tld.Registry.TldType.TEST;
-import static java.util.concurrent.TimeUnit.SECONDS;
 
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.cloud.tasks.v2.Task;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.ParameterMap;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
 import java.util.Random;
 import java.util.stream.Stream;
@@ -105,7 +101,8 @@ public final class TldFanoutAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
   @Inject Response response;
   @Inject @Parameter(ENDPOINT_PARAM) String endpoint;
   @Inject @Parameter(QUEUE_PARAM) String queue;
@@ -115,7 +112,9 @@ public final class TldFanoutAction implements Runnable {
   @Inject @Parameter(EXCLUDE_PARAM) ImmutableSet<String> excludes;
   @Inject @Parameter(JITTER_SECONDS_PARAM) Optional<Integer> jitterSeconds;
   @Inject @ParameterMap ImmutableListMultimap<String, String> params;
-  @Inject TldFanoutAction() {}
+
+  @Inject
+  TldFanoutAction() {}
 
   @Override
   public void run() {
@@ -126,8 +125,7 @@ public final class TldFanoutAction implements Runnable {
         runInEmpty || forEachTestTld || forEachRealTld,
         "At least one of runInEmpty, forEachTestTld, forEachRealTld must be given");
     checkArgument(
-        !(runInEmpty && !excludes.isEmpty()),
-        "Can't specify 'exclude' with 'runInEmpty'");
+        !(runInEmpty && !excludes.isEmpty()), "Can't specify 'exclude' with 'runInEmpty'");
     ImmutableSet<String> tlds =
         runInEmpty
             ? ImmutableSet.of("")
@@ -137,42 +135,34 @@ public final class TldFanoutAction implements Runnable {
                 .filter(not(in(excludes)))
                 .collect(toImmutableSet());
     Multimap<String, String> flowThruParams = filterKeys(params, not(in(CONTROL_PARAMS)));
-    Queue taskQueue = getQueue(queue);
     StringBuilder outputPayload =
         new StringBuilder(
             String.format("OK: Launched the following %d tasks in queue %s\n", tlds.size(), queue));
-    logger.atInfo().log("Launching %d tasks in queue %s", tlds.size(), queue);
+    logger.atInfo().log("Launching %d tasks in queue %s.", tlds.size(), queue);
     if (tlds.isEmpty()) {
       logger.atWarning().log("No TLDs to fan-out!");
     }
     for (String tld : tlds) {
-      TaskOptions taskOptions = createTaskOptions(tld, flowThruParams);
-      TaskHandle taskHandle = taskQueueUtils.enqueue(taskQueue, taskOptions);
+      Task task = createTask(tld, flowThruParams);
+      Task createdTask = cloudTasksUtils.enqueue(queue, task);
       outputPayload.append(
           String.format(
               "- Task: '%s', tld: '%s', endpoint: '%s'\n",
-              taskHandle.getName(), tld, taskOptions.getUrl()));
+              createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri()));
       logger.atInfo().log(
-          "Task: '%s', tld: '%s', endpoint: '%s'", taskHandle.getName(), tld, taskOptions.getUrl());
+          "Task: '%s', tld: '%s', endpoint: '%s'.",
+          createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri());
     }
     response.setContentType(PLAIN_TEXT_UTF_8);
     response.setPayload(outputPayload.toString());
   }
 
-  private TaskOptions createTaskOptions(String tld, Multimap<String, String> params) {
-    TaskOptions options =
-        withUrl(endpoint)
-            .countdownMillis(
-                jitterSeconds
-                    .map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
-                    .orElse(0));
+  private Task createTask(String tld, Multimap<String, String> params) {
     if (!tld.isEmpty()) {
-      options.param(RequestParameters.PARAM_TLD, tld);
+      params = ArrayListMultimap.create(params);
+      params.put(RequestParameters.PARAM_TLD, tld);
     }
-    for (String param : params.keySet()) {
-      // TaskOptions.param() does not accept null values.
-      options.param(param, nullToEmpty(getFirst(params.get(param), null)));
-    }
-    return options;
+    return CloudTasksUtils.createPostTask(
+        endpoint, Service.BACKEND.toString(), params, clock, jitterSeconds);
   }
 }

core/src/main/java/google/registry/dns/DnsQueue.java

@@ -107,7 +107,7 @@ public class DnsQueue {
   private TaskHandle addToQueue(
       TargetType targetType, String targetName, String tld, Duration countdown) {
     logger.atInfo().log(
-        "Adding task type=%s, target=%s, tld=%s to pull queue %s (%d tasks currently on queue)",
+        "Adding task type=%s, target=%s, tld=%s to pull queue %s (%d tasks currently on queue).",
         targetType, targetName, tld, DNS_PULL_QUEUE_NAME, queue.fetchStatistics().getNumTasks());
     return queue.add(
         TaskOptions.Builder.withDefaults()
@@ -166,7 +166,7 @@ public class DnsQueue {
           "There are %d tasks in the DNS queue '%s'.", numTasks, DNS_PULL_QUEUE_NAME);
       return queue.leaseTasks(leaseDuration.getMillis(), MILLISECONDS, leaseTasksBatchSize);
     } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed leasing tasks too fast");
+      logger.atSevere().withCause(e).log("Failed leasing tasks too fast.");
       return ImmutableList.of();
     }
   }
@@ -176,7 +176,7 @@ public class DnsQueue {
     try {
       queue.deleteTask(tasks);
     } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed deleting tasks too fast");
+      logger.atSevere().withCause(e).log("Failed deleting tasks too fast.");
     }
   }
 }

core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java

@@ -104,7 +104,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
         new Duration(enqueuedTime, now));
     logger.atInfo().log(
         "publishDnsWriter latency statistics: TLD: %s, dnsWriter: %s, actionStatus: %s, "
-            + "numItems: %d, timeSinceCreation: %s, timeInQueue: %s",
+            + "numItems: %d, timeSinceCreation: %s, timeInQueue: %s.",
         tld,
         dnsWriter,
         status,
@@ -144,7 +144,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
 
   /** Adds all the domains and hosts in the batch back to the queue to be processed later. */
   private void requeueBatch() {
-    logger.atInfo().log("Requeueing batch for retry");
+    logger.atInfo().log("Requeueing batch for retry.");
     for (String domain : nullToEmpty(domains)) {
       dnsQueue.addDomainRefreshTask(domain);
     }
@@ -158,14 +158,14 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     // LockIndex should always be within [1, numPublishLocks]
     if (lockIndex > numPublishLocks || lockIndex <= 0) {
       logger.atSevere().log(
-          "Lock index should be within [1,%d], got %d instead", numPublishLocks, lockIndex);
+          "Lock index should be within [1,%d], got %d instead.", numPublishLocks, lockIndex);
       return false;
     }
     // Check if the Registry object's num locks has changed since this task was batched
     int registryNumPublishLocks = Registry.get(tld).getNumDnsPublishLocks();
     if (registryNumPublishLocks != numPublishLocks) {
       logger.atWarning().log(
-          "Registry numDnsPublishLocks %d out of sync with parameter %d",
+          "Registry numDnsPublishLocks %d out of sync with parameter %d.",
           registryNumPublishLocks, numPublishLocks);
       return false;
     }
@@ -179,7 +179,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     DnsWriter writer = dnsWriterProxy.getByClassNameForTld(dnsWriter, tld);
 
     if (writer == null) {
-      logger.atWarning().log("Couldn't get writer %s for TLD %s", dnsWriter, tld);
+      logger.atWarning().log("Couldn't get writer %s for TLD %s.", dnsWriter, tld);
       recordActionResult(ActionStatus.BAD_WRITER);
       requeueBatch();
       return;
@@ -190,11 +190,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     for (String domain : nullToEmpty(domains)) {
       if (!DomainNameUtils.isUnder(
           InternetDomainName.from(domain), InternetDomainName.from(tld))) {
-        logger.atSevere().log("%s: skipping domain %s not under tld", tld, domain);
+        logger.atSevere().log("%s: skipping domain %s not under TLD.", tld, domain);
         domainsRejected += 1;
       } else {
         writer.publishDomain(domain);
-        logger.atInfo().log("%s: published domain %s", tld, domain);
+        logger.atInfo().log("%s: published domain %s.", tld, domain);
         domainsPublished += 1;
       }
     }
@@ -206,11 +206,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     for (String host : nullToEmpty(hosts)) {
       if (!DomainNameUtils.isUnder(
           InternetDomainName.from(host), InternetDomainName.from(tld))) {
-        logger.atSevere().log("%s: skipping host %s not under tld", tld, host);
+        logger.atSevere().log("%s: skipping host %s not under TLD.", tld, host);
         hostsRejected += 1;
       } else {
         writer.publishHost(host);
-        logger.atInfo().log("%s: published host %s", tld, host);
+        logger.atInfo().log("%s: published host %s.", tld, host);
         hostsPublished += 1;
       }
     }
@@ -233,7 +233,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
           tld, dnsWriter, commitStatus, duration, domainsPublished, hostsPublished);
       logger.atInfo().log(
           "writer.commit() statistics: TLD: %s, dnsWriter: %s, commitStatus: %s, duration: %s, "
-              + "domainsPublished: %d, domainsRejected: %d, hostsPublished: %d, hostsRejected: %d",
+              + "domainsPublished: %d, domainsRejected: %d, hostsPublished: %d, hostsRejected: %d.",
           tld,
           dnsWriter,
           commitStatus,

core/src/main/java/google/registry/dns/writer/DnsWriter.java

@@ -31,12 +31,12 @@ package google.registry.dns.writer;
 public interface DnsWriter {
 
   /**
-   * Loads {@code domainName} from Datastore and publishes its NS/DS records to the DNS server.
+   * Loads {@code domainName} from the database and publishes its NS/DS records to the DNS server.
    * Replaces existing records for the exact name supplied with an NS record for each name server
    * and a DS record for each delegation signer stored in the registry for the supplied domain name.
    * If the domain is deleted or is in a "non-publish" state then any existing records are deleted.
    *
-   * This must NOT actually perform any action, instead it should stage the action so that it's
+   * <p>This must NOT actually perform any action, instead it should stage the action so that it's
    * performed when {@link #commit()} is called.
    *
    * @param domainName the fully qualified domain name, with no trailing dot
@@ -44,14 +44,14 @@ public interface DnsWriter {
   void publishDomain(String domainName);
 
   /**
-   * Loads {@code hostName} from Datastore and publishes its A/AAAA glue records to the DNS server,
-   * if it is used as an in-bailiwick nameserver. Orphaned glue records are prohibited. Replaces
-   * existing records for the exact name supplied, with an A or AAAA record (as appropriate) for
-   * each address stored in the registry, for the supplied host name. If the host is deleted then
-   * the existing records are deleted. Assumes that this method will only be called for in-bailiwick
-   * hosts. The registry does not have addresses for other hosts.
+   * Loads {@code hostName} from the database and publishes its A/AAAA glue records to the DNS
+   * server, if it is used as an in-bailiwick nameserver. Orphaned glue records are prohibited.
+   * Replaces existing records for the exact name supplied, with an A or AAAA record (as
+   * appropriate) for each address stored in the registry, for the supplied host name. If the host
+   * is deleted then the existing records are deleted. Assumes that this method will only be called
+   * for in-bailiwick hosts. The registry does not have addresses for other hosts.
    *
-   * This must NOT actually perform any action, instead it should stage the action so that it's
+   * <p>This must NOT actually perform any action, instead it should stage the action so that it's
    * performed when {@link #commit()} is called.
    *
    * @param hostName the fully qualified host name, with no trailing dot

core/src/main/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java

@@ -180,11 +180,11 @@ public class CloudDnsWriter extends BaseDnsWriter {
 
     desiredRecords.put(absoluteDomainName, domainRecords.build());
     logger.atFine().log(
-        "Will write %d records for domain %s", domainRecords.build().size(), absoluteDomainName);
+        "Will write %d records for domain '%s'.", domainRecords.build().size(), absoluteDomainName);
   }
 
   private void publishSubordinateHost(String hostName) {
-    logger.atInfo().log("Publishing glue records for %s", hostName);
+    logger.atInfo().log("Publishing glue records for host '%s'.", hostName);
     // Canonicalize name
     String absoluteHostName = getAbsoluteHostName(hostName);
 
@@ -250,7 +250,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
 
     // Host not managed by our registry, no need to update DNS.
     if (!tld.isPresent()) {
-      logger.atSevere().log("publishHost called for invalid host %s", hostName);
+      logger.atSevere().log("publishHost called for invalid host '%s'.", hostName);
       return;
     }
 
@@ -273,7 +273,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
     ImmutableMap<String, ImmutableSet<ResourceRecordSet>> desiredRecordsCopy =
         ImmutableMap.copyOf(desiredRecords);
     retrier.callWithRetry(() -> mutateZone(desiredRecordsCopy), ZoneStateException.class);
-    logger.atInfo().log("Wrote to Cloud DNS");
+    logger.atInfo().log("Wrote to Cloud DNS.");
   }
 
   /** Returns the glue records for in-bailiwick nameservers for the given domain+records. */
@@ -329,7 +329,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
    */
   private Map<String, List<ResourceRecordSet>> getResourceRecordsForDomains(
       Set<String> domainNames) {
-    logger.atFine().log("Fetching records for %s", domainNames);
+    logger.atFine().log("Fetching records for domain '%s'.", domainNames);
     // As per Concurrent.transform() - if numThreads or domainNames.size() < 2, it will not use
     // threading.
     return ImmutableMap.copyOf(
@@ -381,11 +381,11 @@ public class CloudDnsWriter extends BaseDnsWriter {
     ImmutableSet<ResourceRecordSet> intersection =
         Sets.intersection(additions, deletions).immutableCopy();
     logger.atInfo().log(
-        "There are %d common items out of the %d items in 'additions' and %d items in 'deletions'",
+        "There are %d common items out of the %d items in 'additions' and %d items in 'deletions'.",
         intersection.size(), additions.size(), deletions.size());
     // Exit early if we have nothing to update - dnsConnection doesn't work on empty changes
     if (additions.equals(deletions)) {
-      logger.atInfo().log("Returning early because additions is the same as deletions");
+      logger.atInfo().log("Returning early because additions are the same as deletions.");
       return;
     }
     Change change =

core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml

@@ -208,6 +208,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
     <description>

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -157,6 +157,12 @@
     <url-pattern>/_dr/cron/readDnsQueue</url-pattern>
   </servlet-mapping>
 
+  <!-- Replicates SQL transactions to Datastore during the Registry 3.0 migration. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/cron/replicateToDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Publishes DNS updates. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>
@@ -355,6 +361,12 @@
     <url-pattern>/_dr/task/deleteExpiredDomains</url-pattern>
   </servlet-mapping>
 
+  <!-- Background action to send notification emails to registrars with expiring certificate. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/sendExpiringCertificateNotificationEmail</url-pattern>
+  </servlet-mapping>
+
   <!-- Mapreduce to import contacts from escrow file -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>
@@ -385,6 +397,13 @@
     <url-pattern>/_dr/task/relockDomain</url-pattern>
   </servlet-mapping>
 
+  <!-- Background action to wipe out PII fields of ContactHistory entities that
+have been in the database for a certain period of time. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutContactHistoryPii</url-pattern>
+  </servlet-mapping>
+
   <!-- Action to wipeout Cloud SQL data -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/env/crash/default/WEB-INF/cron.xml

@@ -143,7 +143,7 @@
       It also enqueues a new task to wait on the completion of that job and then load the resulting
       snapshot into bigquery.
     </description>
-    <!--
+    <!- -
       Keep google.registry.export.CheckBackupAction.MAXIMUM_BACKUP_RUNNING_TIME less than
       this interval. - ->
     <schedule>every day 06:00</schedule>
@@ -211,6 +211,15 @@
   </cron>
   -->
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <!--
     The next two wipeout jobs are required when crash has production data.
    -->

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -193,6 +193,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
+    <description>
+      This job runs an action that sends emails to partners if their certificates are expiring soon.
+    </description>
+    <schedule>every day 04:30</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>
@@ -330,4 +339,23 @@
     <schedule>every day 15:00</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
+    <description>
+      Replays recent commit logs from Datastore to the SQL secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -120,6 +120,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
     <description>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -168,6 +168,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
+    <description>
+      This job runs an action that sends emails to partners if their certificates are expiring soon.
+    </description>
+    <schedule>every day 04:30</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>
@@ -237,4 +246,23 @@
     <schedule>every 3 minutes</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/export/BackupDatastoreAction.java

@@ -80,7 +80,7 @@ public class BackupDatastoreAction implements Runnable {
       logger.atInfo().log(message);
       response.setPayload(message);
     } catch (Throwable e) {
-      throw new InternalServerErrorException("Exception occurred while backing up datastore.", e);
+      throw new InternalServerErrorException("Exception occurred while backing up Datastore", e);
     }
   }
 }

core/src/main/java/google/registry/export/BigqueryPollJobAction.java

@@ -118,10 +118,10 @@ public class BigqueryPollJobAction implements Runnable {
 
     // Check if the job ended with an error.
     if (job.getStatus().getErrorResult() != null) {
-      logger.atSevere().log("Bigquery job failed - %s - %s", jobRefString, job);
+      logger.atSevere().log("Bigquery job failed - %s - %s.", jobRefString, job);
       return false;
     }
-    logger.atInfo().log("Bigquery job succeeded - %s", jobRefString);
+    logger.atInfo().log("Bigquery job succeeded - %s.", jobRefString);
     return true;
   }
 

core/src/main/java/google/registry/export/CheckBackupAction.java

@@ -171,10 +171,10 @@ public class CheckBackupAction implements Runnable {
         ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
     String message = String.format("Datastore backup %s complete - ", backupName);
     if (exportedKindsToLoad.isEmpty()) {
-      message += "no kinds to load into BigQuery";
+      message += "no kinds to load into BigQuery.";
     } else {
       enqueueUploadBackupTask(backupId, backup.getExportFolderUrl(), exportedKindsToLoad);
-      message += "BigQuery load task enqueued";
+      message += "BigQuery load task enqueued.";
     }
     logger.atInfo().log(message);
     response.setPayload(message);

core/src/main/java/google/registry/export/ExportDomainListsAction.java

@@ -85,7 +85,7 @@ public class ExportDomainListsAction implements Runnable {
   @Override
   public void run() {
     ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
-    logger.atInfo().log("Exporting domain lists for tlds %s", realTlds);
+    logger.atInfo().log("Exporting domain lists for TLDs %s.", realTlds);
     if (tm().isOfy()) {
       mrRunner
           .setJobName("Export domain lists")
@@ -145,7 +145,7 @@ public class ExportDomainListsAction implements Runnable {
       Registry registry = Registry.get(tld);
       if (registry.getDriveFolderId() == null) {
         logger.atInfo().log(
-            "Skipping registered domains export for TLD %s because Drive folder isn't specified",
+            "Skipping registered domains export for TLD %s because Drive folder isn't specified.",
             tld);
       } else {
         String resultMsg =

core/src/main/java/google/registry/export/ExportPremiumTermsAction.java

@@ -110,11 +110,11 @@ public class ExportPremiumTermsAction implements Runnable {
   private Optional<String> checkConfig(Registry registry) {
     if (isNullOrEmpty(registry.getDriveFolderId())) {
       logger.atInfo().log(
-          "Skipping premium terms export for TLD %s because Drive folder isn't specified", tld);
+          "Skipping premium terms export for TLD %s because Drive folder isn't specified.", tld);
       return Optional.of("Skipping export because no Drive folder is associated with this TLD");
     }
     if (!registry.getPremiumListName().isPresent()) {
-      logger.atInfo().log("No premium terms to export for TLD %s", tld);
+      logger.atInfo().log("No premium terms to export for TLD '%s'.", tld);
       return Optional.of("No premium lists configured");
     }
     return Optional.empty();

core/src/main/java/google/registry/export/ExportReservedTermsAction.java

@@ -65,11 +65,11 @@ public class ExportReservedTermsAction implements Runnable {
       String resultMsg;
       if (registry.getReservedListNames().isEmpty() && isNullOrEmpty(registry.getDriveFolderId())) {
         resultMsg = "No reserved lists configured";
-        logger.atInfo().log("No reserved terms to export for TLD %s", tld);
+        logger.atInfo().log("No reserved terms to export for TLD '%s'.", tld);
       } else if (registry.getDriveFolderId() == null) {
         resultMsg = "Skipping export because no Drive folder is associated with this TLD";
         logger.atInfo().log(
-            "Skipping reserved terms export for TLD %s because Drive folder isn't specified", tld);
+            "Skipping reserved terms export for TLD %s because Drive folder isn't specified.", tld);
       } else {
         resultMsg = driveConnection.createOrUpdateFile(
             RESERVED_TERMS_FILENAME,

core/src/main/java/google/registry/export/SyncGroupMembersAction.java

@@ -19,7 +19,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.util.CollectionUtils.nullToEmpty;
-import static google.registry.util.RegistrarUtils.normalizeClientId;
+import static google.registry.util.RegistrarUtils.normalizeRegistrarId;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
@@ -96,16 +96,15 @@ public final class SyncGroupMembersAction implements Runnable {
   }
 
   /**
-   * Returns the Google Groups email address for the given registrar clientId and
-   * RegistrarContact.Type
+   * Returns the Google Groups email address for the given registrar ID and RegistrarContact.Type.
    */
   public static String getGroupEmailAddressForContactType(
-      String clientId, RegistrarContact.Type type, String gSuiteDomainName) {
-    // Take the registrar's clientId, make it lowercase, and remove all characters that aren't
+      String registrarId, RegistrarContact.Type type, String gSuiteDomainName) {
+    // Take the registrar's ID, make it lowercase, and remove all characters that aren't
     // alphanumeric, hyphens, or underscores.
     return String.format(
         "%s-%s-contacts@%s",
-        normalizeClientId(clientId), type.getDisplayName(), gSuiteDomainName);
+        normalizeRegistrarId(registrarId), type.getDisplayName(), gSuiteDomainName);
   }
 
   /**
@@ -150,7 +149,8 @@ public final class SyncGroupMembersAction implements Runnable {
 
   /**
    * Parses the results from Google Groups for each registrar, setting the dirty flag to false in
-   * Datastore for the calls that succeeded and accumulating the errors for the calls that failed.
+   * the database for the calls that succeeded and accumulating the errors for the calls that
+   * failed.
    */
   private static List<Throwable> getErrorsAndUpdateFlagsForSuccesses(
       ImmutableMap<Registrar, Optional<Throwable>> results) {
@@ -175,8 +175,8 @@ public final class SyncGroupMembersAction implements Runnable {
       long totalAdded = 0;
       long totalRemoved = 0;
       for (final RegistrarContact.Type type : RegistrarContact.Type.values()) {
-        groupKey = getGroupEmailAddressForContactType(
-            registrar.getClientId(), type, gSuiteDomainName);
+        groupKey =
+            getGroupEmailAddressForContactType(registrar.getRegistrarId(), type, gSuiteDomainName);
         Set<String> currentMembers = groupsConnection.getMembersOfGroup(groupKey);
         Set<String> desiredMembers =
             registrarContacts
@@ -194,12 +194,14 @@ public final class SyncGroupMembersAction implements Runnable {
         }
       }
       logger.atInfo().log(
-          "Successfully synced contacts for registrar %s: added %d and removed %d",
-          registrar.getClientId(), totalAdded, totalRemoved);
+          "Successfully synced contacts for registrar %s: added %d and removed %d.",
+          registrar.getRegistrarId(), totalAdded, totalRemoved);
     } catch (IOException e) {
       // Package up exception and re-throw with attached additional relevant info.
-      String msg = String.format("Couldn't sync contacts for registrar %s to group %s",
-          registrar.getClientId(), groupKey);
+      String msg =
+          String.format(
+              "Couldn't sync contacts for registrar %s to group %s",
+              registrar.getRegistrarId(), groupKey);
       throw new RuntimeException(msg, e);
     }
   }

core/src/main/java/google/registry/export/UpdateSnapshotViewAction.java

@@ -150,8 +150,7 @@ public class UpdateSnapshotViewAction implements Runnable {
       if (e.getDetails() != null && e.getDetails().getCode() == 404) {
         bigquery.tables().insert(ref.getProjectId(), ref.getDatasetId(), table).execute();
       } else {
-        logger.atWarning().withCause(e).log(
-            "UpdateSnapshotViewAction failed, caught exception %s", e.getDetails());
+        logger.atWarning().withCause(e).log("UpdateSnapshotViewAction errored out.");
       }
     }
   }

core/src/main/java/google/registry/export/UploadDatastoreBackupAction.java

@@ -109,7 +109,7 @@ public class UploadDatastoreBackupAction implements Runnable {
       String message = uploadBackup(backupId, backupFolderUrl, Splitter.on(',').split(backupKinds));
       logger.atInfo().log("Loaded backup successfully: %s", message);
     } catch (Throwable e) {
-      logger.atSevere().withCause(e).log("Error loading backup");
+      logger.atSevere().withCause(e).log("Error loading backup.");
       if (e instanceof IllegalArgumentException) {
         throw new BadRequestException("Error calling load backup: " + e.getMessage(), e);
       } else {
@@ -148,12 +148,12 @@ public class UploadDatastoreBackupAction implements Runnable {
           getQueue(UpdateSnapshotViewAction.QUEUE));
 
       builder.append(String.format(" - %s:%s\n", projectId, jobId));
-      logger.atInfo().log("Submitted load job %s:%s", projectId, jobId);
+      logger.atInfo().log("Submitted load job %s:%s.", projectId, jobId);
     }
     return builder.toString();
   }
 
-  static String sanitizeForBigquery(String backupId) {
+  private static String sanitizeForBigquery(String backupId) {
     return backupId.replaceAll("[^a-zA-Z0-9_]", "_");
   }
 

core/src/main/java/google/registry/export/sheet/SheetSynchronizer.java

@@ -117,7 +117,7 @@ class SheetSynchronizer {
       BatchUpdateValuesResponse response =
           sheetsService.spreadsheets().values().batchUpdate(spreadsheetId, updateRequest).execute();
       Integer cellsUpdated = response.getTotalUpdatedCells();
-      logger.atInfo().log("Updated %d originalVals", cellsUpdated != null ? cellsUpdated : 0);
+      logger.atInfo().log("Updated %d originalVals.", cellsUpdated != null ? cellsUpdated : 0);
     }
 
     // Append extra rows if necessary
@@ -140,7 +140,7 @@ class SheetSynchronizer {
           .setInsertDataOption("INSERT_ROWS")
           .execute();
       logger.atInfo().log(
-          "Appended %d rows to range %s",
+          "Appended %d rows to range %s.",
           data.size() - originalVals.size(), appendResponse.getTableRange());
     // Clear the extra rows if necessary
     } else if (data.size() < originalVals.size()) {
@@ -155,7 +155,7 @@ class SheetSynchronizer {
                   new ClearValuesRequest())
               .execute();
       logger.atInfo().log(
-          "Cleared %d rows from range %s",
+          "Cleared %d rows from range %s.",
           originalVals.size() - data.size(), clearResponse.getClearedRange());
     }
   }

core/src/main/java/google/registry/export/sheet/SyncRegistrarsSheet.java

@@ -47,7 +47,7 @@ import javax.inject.Inject;
 import org.joda.time.DateTime;
 
 /**
- * Class for synchronizing all {@link Registrar} Datastore objects to a Google Spreadsheet.
+ * Class for synchronizing all {@link Registrar} objects to a Google Spreadsheet.
  *
  * @see SyncRegistrarsSheetAction
  */
@@ -82,7 +82,7 @@ class SyncRegistrarsSheet {
         new Ordering<Registrar>() {
           @Override
           public int compare(Registrar left, Registrar right) {
-            return left.getClientId().compareTo(right.getClientId());
+            return left.getRegistrarId().compareTo(right.getRegistrarId());
           }
         }.immutableSortedCopy(Registrar.loadAllCached()).stream()
             .filter(
@@ -116,7 +116,7 @@ class SyncRegistrarsSheet {
                   // and you'll need to remove deleted columns probably like a week after
                   // deployment.
                   //
-                  builder.put("clientIdentifier", convert(registrar.getClientId()));
+                  builder.put("clientIdentifier", convert(registrar.getRegistrarId()));
                   builder.put("registrarName", convert(registrar.getRegistrarName()));
                   builder.put("state", convert(registrar.getState()));
                   builder.put("ianaIdentifier", convert(registrar.getIanaIdentifier()));

core/src/main/java/google/registry/flows/CheckApiAction.java

@@ -150,7 +150,7 @@ public class CheckApiAction implements Runnable {
       return fail(e.getResult().getMsg());
     } catch (Exception e) {
       metricBuilder.status(UNKNOWN_ERROR);
-      logger.atWarning().withCause(e).log("Unknown error");
+      logger.atWarning().withCause(e).log("Unknown error.");
       return fail("Invalid request");
     }
   }

core/src/main/java/google/registry/flows/EppController.java

@@ -61,7 +61,7 @@ public final class EppController {
       boolean isDryRun,
       boolean isSuperuser,
       byte[] inputXmlBytes) {
-    eppMetricBuilder.setClientId(Optional.ofNullable(sessionMetadata.getClientId()));
+    eppMetricBuilder.setRegistrarId(Optional.ofNullable(sessionMetadata.getRegistrarId()));
     try {
       EppInput eppInput;
       try {
@@ -76,7 +76,7 @@ public final class EppController {
                     JSONValue.toJSONString(
                         ImmutableMap.<String, Object>of(
                             "clientId",
-                            nullToEmpty(sessionMetadata.getClientId()),
+                            nullToEmpty(sessionMetadata.getRegistrarId()),
                             "resultCode",
                             e.getResult().getCode().code,
                             "resultMessage",
@@ -130,12 +130,12 @@ public final class EppController {
     } catch (EppException | EppExceptionInProviderException e) {
       // The command failed. Send the client an error message, but only log at INFO since many of
       // these failures are innocuous or due to client error, so there's nothing we have to change.
-      logger.atInfo().withCause(e).log("Flow returned failure response");
+      logger.atInfo().withCause(e).log("Flow returned failure response.");
       EppException eppEx = (EppException) (e instanceof EppException ? e : e.getCause());
       return getErrorResponse(eppEx.getResult(), flowComponent.trid());
     } catch (Throwable e) {
       // Something bad and unexpected happened. Send the client a generic error, and log at SEVERE.
-      logger.atSevere().withCause(e).log("Unexpected failure in flow execution");
+      logger.atSevere().withCause(e).log("Unexpected failure in flow execution.");
       return getErrorResponse(Result.create(Code.COMMAND_FAILED), flowComponent.trid());
     }
   }
@@ -143,9 +143,6 @@ public final class EppController {
   /** Creates a response indicating an EPP failure. */
   @VisibleForTesting
   static EppOutput getErrorResponse(Result result, Trid trid) {
-    return EppOutput.create(new EppResponse.Builder()
-        .setResult(result)
-        .setTrid(trid)
-        .build());
+    return EppOutput.create(new EppResponse.Builder().setResult(result).setTrid(trid).build());
   }
 }

core/src/main/java/google/registry/flows/EppException.java

@@ -25,10 +25,12 @@ import google.registry.model.eppinput.EppInput.InnerCommand;
 import google.registry.model.eppinput.EppInput.ResourceCommandWrapper;
 import google.registry.model.eppoutput.Result;
 import google.registry.model.eppoutput.Result.Code;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Inherited;
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
+import javax.annotation.Nullable;
 
 /** Exception used to propagate all failures containing one or more EPP responses. */
 public abstract class EppException extends Exception {
@@ -37,7 +39,12 @@ public abstract class EppException extends Exception {
 
   /** Create an EppException with a custom message. */
   private EppException(String message) {
-    super(message);
+    this(message, null);
+  }
+
+  /** Create an EppException with a custom message and cause. */
+  private EppException(String message, @Nullable Throwable cause) {
+    super(message, cause);
     Code code = getClass().getAnnotation(EppResultCode.class).value();
     Preconditions.checkState(!code.isSuccess());
     this.result = Result.create(code, message);
@@ -255,4 +262,12 @@ public abstract class EppException extends Exception {
       super("Specified protocol version is not implemented");
     }
   }
+
+  /** Registry is currently undergoing maintenance and is in read-only mode. */
+  @EppResultCode(Code.COMMAND_FAILED)
+  public static class ReadOnlyModeEppException extends EppException {
+    ReadOnlyModeEppException(ReadOnlyModeException cause) {
+      super("Registry is currently undergoing maintenance and is in read-only mode", cause);
+    }
+  }
 }

core/src/main/java/google/registry/flows/EppMetrics.java

@@ -88,7 +88,7 @@ public class EppMetrics {
     String eppStatusCode =
         metric.getStatus().isPresent() ? String.valueOf(metric.getStatus().get().code) : "";
     eppRequestsByRegistrar.increment(
-        metric.getCommandName().orElse(""), metric.getClientId().orElse(""), eppStatusCode);
+        metric.getCommandName().orElse(""), metric.getRegistrarId().orElse(""), eppStatusCode);
     eppRequestsByTld.increment(
         metric.getCommandName().orElse(""), metric.getTld().orElse(""), eppStatusCode);
   }

core/src/main/java/google/registry/flows/EppRequestHandler.java

@@ -84,7 +84,7 @@ public class EppRequestHandler {
         response.setHeader(ProxyHttpHeaders.LOGGED_IN, "true");
       }
     } catch (Exception e) {
-      logger.atWarning().withCause(e).log("handleEppCommand general exception");
+      logger.atWarning().withCause(e).log("handleEppCommand general exception.");
       response.setStatus(SC_BAD_REQUEST);
     }
   }

core/src/main/java/google/registry/flows/EppToolAction.java

@@ -38,7 +38,10 @@ public class EppToolAction implements Runnable {
 
   public static final String PATH = "/_dr/epptool";
 
-  @Inject @Parameter("clientId") String clientId;
+  @Inject
+  @Parameter("clientId")
+  String registrarId;
+
   @Inject @Parameter("superuser") boolean isSuperuser;
   @Inject @Parameter("dryRun") boolean isDryRun;
   @Inject @Parameter("xml") String xml;
@@ -49,8 +52,7 @@ public class EppToolAction implements Runnable {
   public void run() {
     eppRequestHandler.executeEpp(
         new StatelessRequestSessionMetadata(
-            clientId,
-            ProtocolDefinition.getVisibleServiceExtensionUris()),
+            registrarId, ProtocolDefinition.getVisibleServiceExtensionUris()),
         new PasswordOnlyTransportCredentials(),
         EppRequestSource.TOOL,
         isDryRun,