Remove obsolete scrap commands (#1502)

Not sure how this snuck through

build.gradle

@@ -20,6 +20,10 @@ buildscript {
     // Lock buildscript dependencies.
     configurations.classpath {
       resolutionStrategy.activateDependencyLocking()
+
+      // log4j has high-profile security vulnerabilities. It's a transitive
+      // dependency used by Gradle itself during build, and not strictly needed.
+      exclude group: 'org.apache.logging.log4j'
     }
   }
 
@@ -46,7 +50,7 @@ plugins {
   id 'com.diffplug.gradle.spotless' version '3.25.0'
 
   id 'jacoco'
-  id 'com.dorongold.task-tree' version '1.5'
+  id 'com.dorongold.task-tree' version '2.1.0'
 }
 
 node {
@@ -182,6 +186,12 @@ allprojects {
       println "Java dependencies: Using Maven Central..."
       mavenCentral()
       google()
+      maven {
+        url "https://packages.confluent.io/maven/"
+        content {
+          includeGroup "io.confluent"
+        }
+      }
     }
   }
 
@@ -196,9 +206,41 @@ allprojects {
   }
 }
 
+rootProject.ext {
+  pyver = { exe ->
+    try {
+      ext.execInBash(
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
+    } catch (org.gradle.process.internal.ExecException e) {
+      return -1;
+    }
+  }
+
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
+    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
+    // Debian so it seems like that should be available anywhere).
+    def MIN_PY_VER = 0x3070300
+    if (pyver('python') >= MIN_PY_VER) {
+      return 'python'
+    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
+      return '/usr/bin/python3'
+    } else {
+      throw new GradleException("No usable Python version found (build " +
+                                "requires at least python 3.7.3)");
+    }
+  }
+}
+
 task runPresubmits(type: Exec) {
-  executable '/usr/bin/python3'
+
   args('config/presubmits.py')
+
+  doFirst {
+    executable getPythonExecutable()
+  }
 }
 
 def javadocSource = []
@@ -412,9 +454,10 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
@@ -422,18 +465,23 @@ rootProject.ext {
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
-    def checkResult = invokeJavaDiffFormatScript("check")
-    if (checkResult == 'true') {
-      throw new IllegalStateException(
-          "Some Java files need to be reformatted. You may use the "
-              + "'javaIncrementalFormatDryRun' task to review\n "
-              + "the changes, or the 'javaIncrementalFormatApply' task "
-              + "to reformat.")
-    } else if (checkResult != 'false') {
-      throw new RuntimeException(
-          "Failed to invoke format check script:\n" + checkResult)
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
+      def checkResult = invokeJavaDiffFormatScript("check")
+      if (checkResult == 'true') {
+        throw new IllegalStateException(
+            "Some Java files need to be reformatted. You may use the "
+                + "'javaIncrementalFormatDryRun' task to review\n "
+                + "the changes, or the 'javaIncrementalFormatApply' task "
+                + "to reformat.")
+      } else if (checkResult != 'false') {
+        throw new RuntimeException(
+            "Failed to invoke format check script:\n" + checkResult)
+      }
+      println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
     }
-    println("Incremental Java format check ok.")
   }
 }
 
@@ -457,15 +505,14 @@ task javaIncrementalFormatApply {
 task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
-  // Exclude the misbehaving generated-by-Soy Java files
-  exclude "**/*SoyInfo.java"
   destinationDir = file("${buildDir}/docs/javadoc")
   options.encoding = "UTF-8"
   // In a lot of places we don't write @return so suppress warnings about that.
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
   options.tags = ["type:a:Generic Type",
-                  "error:a:Expected Error"]
+                  "error:a:Expected Error",
+                  "invariant:a:Guaranteed Property"]
 }
 
 tasks.build.dependsOn(tasks.javadoc)
@@ -483,3 +530,15 @@ task coreDev {
 }
 
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
+
+// disable javadoc in subprojects, these will break because they don't have
+// the correct classpath (see above).
+gradle.taskGraph.whenReady { graph ->
+  graph.getAllTasks().each { task ->
+    def subprojectJavadoc = (task.path =~ /:.+:javadoc/)
+    if (subprojectJavadoc) {
+        println "Skipping ${task.path} for javadoc (only root javadoc works)"
+        task.enabled = false
+    }
+  }
+}

common/gradle/dependency-locks/testCompile.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testRuntime.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingCompile.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingCompileClasspath.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingRuntime.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingRuntimeClasspath.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/src/testing/java/google/registry/testing/SystemInfo.java

@@ -39,7 +39,7 @@ public final class SystemInfo {
                     pid.getOutputStream().close();
                     pid.waitFor();
                   } catch (IOException e) {
-                    logger.atWarning().withCause(e).log("%s command not available", cmd);
+                    logger.atWarning().withCause(e).log("%s command not available.", cmd);
                     return false;
                   }
                   return true;

config/dependency-license/allowed_licenses.json

@@ -24,6 +24,9 @@
     {
       "moduleLicense": "Apache License v2.0"
     },
+    {
+      "moduleLicense": "Apache License V2.0"
+    },
     {
       "moduleLicense": "Apache License, Version 2.0"
     },

config/nom_build.py

@@ -140,6 +140,8 @@ PROPERTIES = [
              'a BEAM pipeline to image. Setting this property to empty string '
              'will disable image generation.',
              '/usr/bin/dot'),
+    Property('pipeline',
+             'The name of the Beam pipeline being staged.')
 ]
 
 GRADLE_FLAGS = [

config/presubmits.py

@@ -17,9 +17,11 @@ These aren't built in to the static code analysis tools we use (e.g. Checkstyle,
 Error Prone) so we must write them manually.
 """
 
+import json
 import os
 from typing import List, Tuple
 import sys
+import textwrap
 import re
 
 # We should never analyze any generated files
@@ -28,6 +30,13 @@ UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/"
 FORBIDDEN = 1
 REQUIRED = 2
 
+# The list of expected json packages and their licenses.
+# These should be one of the allowed licenses in:
+# config/dependency-license/allowed_licenses.json
+EXPECTED_JS_PACKAGES = [
+    'google-closure-library',   # Owned by Google, Apache 2.0
+]
+
 
 class PresubmitCheck:
 
@@ -79,8 +88,8 @@ PRESUBMITS = {
         r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
         ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
             ".git", "/build/", "/generated/", "/generated_tests/",
-            "node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
-            "registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
+            "node_modules/", "LoggerConfig.java", "registrar_bin.",
+            "registrar_dbg.", "google-java-format-diff.py",
             "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
         }, REQUIRED):
         "File did not include the license header.",
@@ -110,9 +119,10 @@ PRESUBMITS = {
       "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
+    # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
-        "java", {}):
+        "java", {"DatabaseSnapshotTest.java"}):
       "PostgreSQLContainer instantiation must specify docker tag.",
 
     # Various Soy linting checks
@@ -177,52 +187,6 @@ PRESUBMITS = {
         {"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
     ):
         "JavaScript files should not include console logging.",
-    # SQL injection protection rule for java source file:
-    # The sql template passed to createQuery/createNativeQuery methods must be
-    # a variable name in UPPER_CASE_UNDERSCORE format, i.e., a static final
-    # String variable. This forces the use of parameter-binding on all queries
-    # that take parameters.
-    # The rule would forbid invocation of createQuery(Criteria). However, this
-    # can be handled by adding a helper method in an exempted class to make
-    # the calls.
-    # TODO(b/179158393): enable the 'ConstantName' Java style check to ensure
-    # that non-final variables do not use the UPPER_CASE_UNDERSCORE format.
-    PresubmitCheck(
-        # Line 1: the method names we check and the opening parenthesis, which
-        #    marks the beginning of the first parameter
-        # Line 2: The first parameter is a match if is NOT any of the following:
-        #    - final variable name: \s*([A-Z_]+
-        #    - string literal: "([^"]|\\")*"
-        #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
-        # Line 3: , or the closing parenthesis, marking the end of the first
-        #    parameter
-        r'.*\.(query|createQuery|createNativeQuery)\('
-        r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
-        r'(,|\s*\))))',
-        "java",
-        # ActivityReportingQueryBuilder deals with Dremel queries
-        {"src/test", "ActivityReportingQueryBuilder.java",
-         # This class contains helper method to make queries in Beam.
-         "RegistryJpaIO.java",
-         # TODO(b/179158393): Remove everything below, which should be done
-         # using Criteria
-         "ForeignKeyIndex.java",
-         "HistoryEntryDao.java",
-         "JpaTransactionManager.java",
-         "JpaTransactionManagerImpl.java",
-         # CriteriaQueryBuilder is a false positive
-         "CriteriaQueryBuilder.java",
-         "RdapDomainSearchAction.java",
-         "RdapNameserverSearchAction.java",
-         "RdapSearchActionBase.java",
-         "RegistryQuery",
-         },
-    ):
-        "The first String parameter to EntityManager.create(Native)Query "
-        "methods must be one of the following:\n"
-        "  - A String literal\n"
-        "  - Concatenation of String literals only\n"
-        "  - The name of a static final String variable"
 }
 
 # Note that this regex only works for one kind of Flyway file.  If we want to
@@ -310,6 +274,26 @@ def verify_flyway_index():
   return not success
 
 
+def verify_javascript_deps():
+  """Verifies that we haven't introduced any new javascript dependencies."""
+  with open('package.json') as f:
+    package = json.load(f)
+
+  deps = list(package['dependencies'].keys())
+  if deps != EXPECTED_JS_PACKAGES:
+    print('Unexpected javascript dependencies.  Was expecting '
+          '%s, got %s.' % (EXPECTED_JS_PACKAGES, deps))
+    print(textwrap.dedent("""
+        * If the new dependencies are intentional, please verify that the
+        * license is one of the allowed licenses (see
+        * config/dependency-license/allowed_licenses.json) and add an entry
+        * for the package (with the license in a comment) to the
+        * EXPECTED_JS_PACKAGES variable in config/presubmits.py.
+        """))
+    return True
+  return False
+
+
 def get_files():
   for root, dirnames, filenames in os.walk("."):
     for filename in filenames:
@@ -317,6 +301,7 @@ def get_files():
 
 
 if __name__ == "__main__":
+  print('python version is %s' % sys.version)
   failed = False
   for file in get_files():
     error_messages = []
@@ -333,5 +318,8 @@ if __name__ == "__main__":
   # when we put it here it fails fast before all of the tests are run.
   failed |= verify_flyway_index()
 
+  # Make sure we haven't introduced any javascript dependencies.
+  failed |= verify_javascript_deps()
+
   if failed:
     sys.exit(1)

config/show_upgrade_diffs.py

@@ -138,8 +138,8 @@ def main():
     if args.directory and os.path.exists(dir):
         pr(f'Reusing directory {dir}\n')
         os.chdir(dir)
-        run('git', 'fetch', 'git@github.com:google/nomulus', 'master:master')
-        run('git', 'checkout', 'master')
+        run('git', 'fetch', 'git@github.com:google/nomulus', 'master')
+        run('git', 'checkout', 'origin/master')
     else:
         run('git', 'clone', 'git@github.com:google/nomulus', dir)
         os.chdir(dir)
@@ -175,7 +175,7 @@ def main():
         # Print the list of packages that were removed.
         for package in old_packages:
             if package not in new_packages:
-                pr('removed ', b':'.join(package))
+                pr('removed ', b':'.join(package), '\n')
     else:
         pr('Package versions not updated!\n')
 

core/build.gradle

@@ -44,7 +44,6 @@ def outcastTestPatterns = [
     "google/registry/flows/domain/DomainCreateFlowTest.*",
     "google/registry/flows/domain/DomainUpdateFlowTest.*",
     "google/registry/tools/CreateDomainCommandTest.*",
-    "google/registry/tools/server/CreatePremiumListActionTest.*",
 ]
 
 // Tests that fail when running Gradle in a docker container, e. g. when
@@ -70,8 +69,6 @@ def dockerIncompatibleTestPatterns = [
 // Nomulus classes, e.g., threads and objects retained by frameworks.
 // TODO(weiminyu): identify cause and fix offending tests.
 def fragileTestPatterns = [
-    // Problem seems to lie with AppEngine TaskQueue for test.
-    "google/registry/cron/TldFanoutActionTest.*",
     // Test Datastore inexplicably aborts transaction.
     "google/registry/model/tmch/ClaimsListShardTest.*",
     // Changes cache timeouts and for some reason appears to have contention
@@ -185,6 +182,7 @@ dependencies {
   compile deps['com.google.monitoring-client:metrics']
   compile deps['com.google.monitoring-client:stackdriver']
   compile deps['com.google.api-client:google-api-client-java6']
+  compile deps['com.google.api.grpc:proto-google-cloud-tasks-v2']
   compile deps['com.google.apis:google-api-services-admin-directory']
   compile deps['com.google.apis:google-api-services-appengine']
   compile deps['com.google.apis:google-api-services-bigquery']
@@ -217,11 +215,13 @@ dependencies {
   compile deps['com.google.flogger:flogger']
   runtime deps['com.google.flogger:flogger-system-backend']
   compile deps['com.google.guava:guava']
+  compile deps['com.google.protobuf:protobuf-java']
   gradleLint.ignore('unused-dependency') {
     compile deps['com.google.gwt:gwt-user']
   }
   compile deps['com.google.cloud:google-cloud-core']
   compile deps['com.google.cloud:google-cloud-storage']
+  compile deps['com.google.cloud:google-cloud-tasks']
   compile deps['com.google.http-client:google-http-client']
   compile deps['com.google.http-client:google-http-client-appengine']
   compile deps['com.google.http-client:google-http-client-jackson2']
@@ -257,13 +257,12 @@ dependencies {
   compile deps['org.apache.beam:beam-sdks-java-core']
   compile deps['org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core']
   compile deps['org.apache.beam:beam-sdks-java-io-google-cloud-platform']
-  testCompile deps['org.apache.commons:commons-lang3']
+  compile deps['org.apache.commons:commons-lang3']
   testCompile deps['org.apache.commons:commons-text']
   testCompile deps['org.apache.ftpserver:ftplet-api']
   testCompile deps['org.apache.ftpserver:ftpserver-core']
   compile deps['org.apache.httpcomponents:httpclient']
   compile deps['org.apache.httpcomponents:httpcore']
-  runtime deps['org.apache.logging.log4j:log4j-core']
   testCompile deps['org.apache.sshd:sshd-core']
   testCompile deps['org.apache.sshd:sshd-scp']
   testCompile deps['org.apache.sshd:sshd-sftp']
@@ -457,6 +456,22 @@ task soyToJava {
               "--srcs", "${soyFiles.join(',')}",
               "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
     }
+
+    // Replace the "@link" tags after the "@deprecated" tags in the generated
+    // files.  The soy compiler doesn't generate imports for these, causing
+    // us to get warnings when we generate javadocs.
+    // TODO(b/200296387): To be fair, the deprecations are accurate: we're
+    // using the old "SoyInfo" classes instead of the new "Templates" files.
+    // When we convert to the new classes, this hack can go away.
+    def outputs = fileTree(outputDirectory) {
+      include '**/*.java'
+    }
+
+    outputs.each { file ->
+      exec {
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
+      }
+    }
   }
 
   doLast {
@@ -661,9 +676,9 @@ Optional<List<String>> getToolArgsList() {
 
 // To run the nomulus tools with these command line tokens:
 // "--foo", "bar baz", "--qux=quz"
-//   gradle registryTool --args="--foo 'bar baz' --qux=quz"
+//   gradle core:registryTool --args="--foo 'bar baz' --qux=quz"
 // or:
-//   gradle registryTool --PtoolArgs="--foo|bar baz|--qux=quz"
+//   gradle core:registryTool -PtoolArgs="--foo|bar baz|--qux=quz"
 // Note that the delimiting pipe can be backslash escaped if it is part of a
 // parameter.
 ext.createToolTask = {
@@ -687,33 +702,22 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
+createToolTask(
+    'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
 
 createToolTask(
-        'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
 
-project.tasks.create('initSqlPipeline', JavaExec) {
-  main = 'google.registry.beam.initsql.InitSqlPipeline'
+createToolTask(
+    'validateDatastorePipeline', 'google.registry.beam.comparedb.ValidateDatastorePipeline')
 
-  doFirst {
-    getToolArgsList().ifPresent {
-      args it
-    }
 
-    def isDirectRunner =
-        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
-    // The dependency containing DirectRunner is intentionally excluded from the
-    // production binary, so that it won't be chosen by mistake: we definitely do
-    // not want to use it for the real jobs, yet DirectRunner is the default if
-    // the user forgets to override it.
-    // DirectRunner is required for tests and is already on testRuntimeClasspath.
-    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
-    // of defining a new configuration just for the DirectRunner dependency.
-    classpath =
-        isDirectRunner
-            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
-            : sourceSets.main.runtimeClasspath
-  }
-}
+createToolTask(
+    'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+
+createToolTask(
+    'createSyntheticHistoryEntries',
+    'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
 // Caller must provide projectId, GCP region, runner, and the kinds to delete
 // (comma-separated kind names or '*' for all). E.g.:
@@ -756,50 +760,57 @@ createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
 // This packages more code and dependency than necessary. However, without
 // restructuring the source tree it is difficult to generate leaner jars.
 createUberJar(
-    'beam_pipeline_common',
+    'beamPipelineCommon',
     'beam_pipeline_common',
     '')
 
-// Create beam staging task if environment is alpha or crash.
-// All other environments use formally released pipelines through CloudBuild.
+// Create beam staging task if the environment is alpha. Production, sandbox and
+// qa use formally released pipelines through CloudBuild, whereas crash and
+// alpha use the pipelines staged on alpha deployment project.
 //
 // User should install gcloud and login to GCP before invoking this tasks.
-if (environment in ['alpha', 'crash']) {
+if (environment == 'alpha') {
   def pipelines = [
-      [
-          mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
-          metaData: 'google/registry/beam/init_sql_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
-          metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
-          metaData: 'google/registry/beam/spec11_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
-          metaData: 'google/registry/beam/invoicing_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.rde.RdePipeline',
-          metaData: 'google/registry/beam/rde_pipeline_metadata.json'
-      ],
+      initSql            :
+          [
+              mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
+              metaData : 'google/registry/beam/init_sql_pipeline_metadata.json'
+          ],
+      bulkDeleteDatastore:
+          [
+              mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
+              metaData : 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
+          ],
+      spec11             :
+          [
+              mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
+              metaData : 'google/registry/beam/spec11_pipeline_metadata.json'
+          ],
+      invoicing          :
+          [
+              mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
+              metaData : 'google/registry/beam/invoicing_pipeline_metadata.json'
+          ],
+      rde                :
+          [
+              mainClass: 'google.registry.beam.rde.RdePipeline',
+              metaData : 'google/registry/beam/rde_pipeline_metadata.json'
+          ],
   ]
-  project.tasks.create("stage_beam_pipelines") {
+  project.tasks.create("stageBeamPipelines") {
     doLast {
       pipelines.each {
-        def mainClass = it['mainClass']
-        def metaData = it['metaData']
-        def pipelineName = CaseFormat.UPPER_CAMEL.to(
-                CaseFormat.LOWER_UNDERSCORE,
-                mainClass.substring(mainClass.lastIndexOf('.') + 1))
-        def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
-        def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
-        def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath
+        if (rootProject.pipeline == ''|| rootProject.pipeline == it.key) {
+          def mainClass = it.value['mainClass']
+          def metaData = it.value['metaData']
+          def pipelineName = CaseFormat.UPPER_CAMEL.to(
+              CaseFormat.LOWER_UNDERSCORE,
+              mainClass.substring(mainClass.lastIndexOf('.') + 1))
+          def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
+          def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
+          def uberJarName = tasks.beamPipelineCommon.outputs.files.asPath
 
-        def command = "\
+          def command = "\
             gcloud dataflow flex-template build \
             gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
             --image-gcr-path ${imageName}:live \
@@ -809,10 +820,11 @@ if (environment in ['alpha', 'crash']) {
             --jar ${uberJarName} \
             --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
             --project ${gcpProject}".toString()
-        rootProject.ext.execInBash(command, '/tmp')
+          rootProject.ext.execInBash(command, '/tmp')
+        }
       }
     }
-  }.dependsOn(tasks.beam_pipeline_common)
+  }.dependsOn(tasks.beamPipelineCommon)
 }
 
 // A jar with classes and resources from main sourceSet, excluding internal
@@ -1087,6 +1099,10 @@ test {
   // TODO(weiminyu): Remove dependency on sqlIntegrationTest
 }.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
 
+// When we override tests, we also break the cleanTest command.
+cleanTest.dependsOn(cleanFragileTest, cleanOutcastTest, cleanStandardTest,
+                    cleanRegistryToolIntegrationTest, cleanSqlIntegrationTest)
+
 project.build.dependsOn devtool
 project.build.dependsOn buildToolImage
 project.build.dependsOn ':stage'

core/gradle/dependency-locks/compile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,107 +24,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -133,9 +143,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -149,32 +159,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -191,38 +205,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -255,15 +272,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,105 +24,114 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -131,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -147,30 +158,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -186,37 +201,40 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -248,15 +266,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/default.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,108 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -138,9 +148,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -157,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -199,41 +213,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -269,16 +285,18 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,108 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -138,9 +148,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -157,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -199,40 +213,42 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -268,16 +284,18 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,107 +24,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -133,9 +143,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -149,32 +159,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -191,38 +205,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -255,15 +272,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,105 +24,114 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -131,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -147,30 +158,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -186,38 +201,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -249,15 +267,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,107 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,9 +147,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -156,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -198,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -267,16 +283,18 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,107 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,9 +147,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -156,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -198,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -267,16 +283,18 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/runtime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,107 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,9 +147,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -156,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -198,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -267,16 +283,18 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,108 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.10
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -138,9 +148,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
-com.google.protobuf:protobuf-java:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
@@ -157,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -199,40 +213,42 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -268,16 +284,18 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testCompile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,58 +24,60 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -84,37 +88,43 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -122,10 +132,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -158,32 +168,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -200,30 +214,34 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -232,7 +250,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -243,7 +260,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -303,6 +320,8 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -312,8 +331,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,57 +24,59 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -83,36 +87,43 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -120,10 +131,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -156,30 +167,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -195,30 +210,34 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -227,7 +246,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -238,7 +256,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -297,6 +315,8 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -306,8 +326,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testRuntime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,58 +28,60 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -88,38 +92,44 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -127,10 +137,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -166,33 +176,37 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -209,31 +223,35 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -242,8 +260,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -254,7 +270,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -316,6 +332,8 @@ org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -326,8 +344,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,58 +28,60 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -88,38 +92,44 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -127,10 +137,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -166,33 +176,37 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -209,31 +223,35 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -242,8 +260,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -254,7 +270,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -317,6 +333,8 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -327,8 +345,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -21,6 +21,7 @@ import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.ImmutableList;
 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,6 +29,7 @@ import java.io.OutputStream;
 import java.util.Iterator;
 
 /** Utilities for working with backups. */
+@DeleteAfterMigration
 public class BackupUtils {
 
   /** Keys for user metadata fields on commit log files in GCS. */
@@ -41,11 +43,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnlyWithoutBackup().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -14,21 +14,22 @@
 
 package google.registry.backup;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -48,6 +49,7 @@ import org.joda.time.DateTime;
     method = Action.Method.GET,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public final class CommitLogCheckpointAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -56,7 +58,8 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Inject Clock clock;
   @Inject CommitLogCheckpointStrategy strategy;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject CommitLogCheckpointAction() {}
 
   @Override
@@ -64,7 +67,8 @@ public final class CommitLogCheckpointAction implements Runnable {
     final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
     logger.atInfo().log(
         "Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
-    tm().transact(
+    ofyTm()
+        .transact(
             () -> {
               DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
               if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
@@ -73,16 +77,20 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveWithoutBackup()
+                  .saveIgnoringReadOnlyWithoutBackup()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.
-              taskQueueUtils.enqueue(
-                  getQueue(QUEUE_NAME),
-                  withUrl(ExportCommitLogDiffAction.PATH)
-                      .param(LOWER_CHECKPOINT_TIME_PARAM, lastWrittenTime.toString())
-                      .param(
-                          UPPER_CHECKPOINT_TIME_PARAM, checkpoint.getCheckpointTime().toString()));
+              cloudTasksUtils.enqueue(
+                  QUEUE_NAME,
+                  CloudTasksUtils.createPostTask(
+                      ExportCommitLogDiffAction.PATH,
+                      Service.BACKEND.toString(),
+                      ImmutableMultimap.of(
+                          LOWER_CHECKPOINT_TIME_PARAM,
+                          lastWrittenTime.toString(),
+                          UPPER_CHECKPOINT_TIME_PARAM,
+                          checkpoint.getCheckpointTime().toString())));
             });
   }
 }

core/src/main/java/google/registry/backup/CommitLogCheckpointStrategy.java

@@ -23,6 +23,7 @@ import static google.registry.util.DateTimeUtils.earliestOf;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -36,14 +37,14 @@ import org.joda.time.DateTime;
 /**
  * Implementation of the procedure for determining point-in-time consistent commit log checkpoint.
  *
- * <p>This algorithm examines the recently written commit log data and uses a dual-read approach
- * to determine a point-in-time consistent set of checkpoint times for the commit log buckets.  By
+ * <p>This algorithm examines the recently written commit log data and uses a dual-read approach to
+ * determine a point-in-time consistent set of checkpoint times for the commit log buckets. By
  * "consistent" we mean, generally speaking, that if the Datastore were restored by replaying all
  * the commit logs up to the checkpoint times of the buckets, the result would be transactionally
  * correct; there must be no "holes" where restored state depends on non-restored state.
  *
  * <p>The consistency guarantee really has two parts, only one of which is provided by this
- * algorithm.  The procedure below guarantees only that if the resulting checkpoint includes any
+ * algorithm. The procedure below guarantees only that if the resulting checkpoint includes any
  * given commit log, it will also include all the commit logs that were both 1) actually written
  * before that commit log "in real life", and 2) have an earlier timestamp than that commit log.
  * (These criteria do not necessarily imply each other, due to the lack of a global shared clock.)
@@ -51,8 +52,8 @@ import org.joda.time.DateTime;
  * that depends on state from a previous transaction does indeed have a later timestamp.
  *
  * <h2>Procedure description</h2>
- * <pre>
- * {@code
+ *
+ * <pre>{@code
  * ComputeCheckpoint() -> returns a set consisting of a timestamp c(b_i) for every bucket b_i
  *
  * 1) read off the latest commit timestamp t(b_i) for every bucket b_i
@@ -63,35 +64,29 @@ import org.joda.time.DateTime;
  *   a) if S is empty, let T* = +∞ (or the "end of time")
  *   b) else, let T* = T - Δ, for T = min(S) and some small Δ > 0
  * 4) return the set given by: min(t(b_i), T*) for all b_i
- * }
- * </pre>
+ * }</pre>
  *
  * <h2>Correctness proof of algorithm</h2>
  *
- * <p>{@literal
- * As described above, the algorithm is correct as long as it can ensure the following: given a
- * commit log X written at time t(X) to bucket b_x, and another commit log Y that was written "in
- * real life" before X and for which t(Y) < t(X), then if X is included in the checkpoint, so is Y;
- * that is, t(X) <= c(b_x) implies t(Y) <= c(b_y).
- * }
+ * <p>{@literal As described above, the algorithm is correct as long as it can ensure the following:
+ * given a commit log X written at time t(X) to bucket b_x, and another commit log Y that was
+ * written "in real life" before X and for which t(Y) < t(X), then if X is included in the
+ * checkpoint, so is Y; that is, t(X) <= c(b_x) implies t(Y) <= c(b_y). }
  *
- * <p>{@literal
- * To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e. every commit
- * log included in the checkpoint must have been seen in the first pass.  Hence if X was included,
- * then X must have been written by the time we started the second pass.  But since Y was written
- * "in real life" prior to X, we must have seen Y by the second pass too.
- * }
+ * <p>{@literal To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e.
+ * every commit log included in the checkpoint must have been seen in the first pass. Hence if X was
+ * included, then X must have been written by the time we started the second pass. But since Y was
+ * written "in real life" prior to X, we must have seen Y by the second pass too. }
  *
- * <p>{@literal
- * Now assume towards a contradiction that X is indeed included but Y is not, i.e. that we have
- * t(X) <= c(b_x) but t(Y) > c(b_y).  If Y was seen in the first pass, i.e. t(Y) <= t(b_y), then by
- * our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y).  By the definition of
- * c(b_y) it must then equal T*, so we have T* < t(Y).  However, this is a contradiction since
- * t(Y) < t(X) and t(X) <= c(b_x) <= T*.  If instead Y was seen in the second pass but not the
- * first, t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by
- * definition, we again reach the contradiction T* < t(Y).
- * }
+ * <p>{@literal Now assume towards a contradiction that X is indeed included but Y is not, i.e. that
+ * we have t(X) <= c(b_x) but t(Y) > c(b_y). If Y was seen in the first pass, i.e. t(Y) <= t(b_y),
+ * then by our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y). By the definition
+ * of c(b_y) it must then equal T*, so we have T* < t(Y). However, this is a contradiction since
+ * t(Y) < t(X) and t(X) <= c(b_x) <= T*. If instead Y was seen in the second pass but not the first,
+ * t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by definition,
+ * we again reach the contradiction T* < t(Y). }
  */
+@DeleteAfterMigration
 class CommitLogCheckpointStrategy {
 
   @Inject Ofy ofy;

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -20,6 +20,7 @@ import static google.registry.backup.BackupUtils.createDeserializingIterator;
 
 import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
@@ -38,6 +39,7 @@ import java.util.Iterator;
  * <p>This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial
  * population of the Cloud SQL database.
  */
+@DeleteAfterMigration
 public final class CommitLogImports {
 
   private CommitLogImports() {}
@@ -55,10 +57,9 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+  static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
       InputStream inputStream) {
-    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
-        InputStream input = new BufferedInputStream(inputStream)) {
+    try (InputStream input = new BufferedInputStream(inputStream)) {
       Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
       checkState(commitLogs.hasNext());
       checkState(commitLogs.next() instanceof CommitLogCheckpoint);
@@ -105,7 +106,7 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+  static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
     return loadEntitiesByTransaction(inputStream).stream()
         .flatMap(ImmutableList::stream)
         .collect(toImmutableList());

core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static java.lang.Boolean.FALSE;
 import static java.lang.Boolean.TRUE;
 
@@ -35,6 +35,7 @@ import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.CommitLogManifestInput;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
@@ -68,6 +69,7 @@ import org.joda.time.Duration;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 // No longer needed in SQL. Subject to future removal.
 @Deprecated
+@DeleteAfterMigration
 public final class DeleteOldCommitLogsAction implements Runnable {
 
   private static final int NUM_MAP_SHARDS = 20;
@@ -93,7 +95,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
   public void run() {
     DateTime deletionThreshold = clock.nowUtc().minus(maxAge);
     logger.atInfo().log(
-        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s",
+        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s.",
         deletionThreshold);
 
     mrRunner
@@ -208,7 +210,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       getContext().incrementCounter("EPP resources missing pre-threshold revision (SEE LOGS)");
       logger.atSevere().log(
           "EPP resource missing old enough revision: "
-              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s",
+              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s.",
           Key.create(eppResource),
           eppResource.getCreationTime(),
           eppResource.getRevisions().size(),
@@ -286,7 +288,8 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       }
 
       DeletionResult deletionResult =
-          tm().transactNew(
+          ofyTm()
+              .transactNew(
                   () -> {
                     CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
                     // It is possible that the same manifestKey was run twice, if a shard had to be

core/src/main/java/google/registry/backup/EntityImports.java

@@ -20,10 +20,12 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
 import com.google.storage.onestore.v3.OnestoreEntity.Path;
 import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
 /** Utilities for handling imported Datastore entities. */
+@DeleteAfterMigration
 public class EntityImports {
 
   /**

core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java

@@ -40,6 +40,7 @@ import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -63,6 +64,7 @@ import org.joda.time.DateTime;
     method = Action.Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public final class ExportCommitLogDiffAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -100,7 +102,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
 
     // Load the keys of all the manifests to include in this diff.
     List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
-    logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
+    logger.atInfo().log("Found %d manifests to export.", sortedKeys.size());
     // Open an output channel to GCS, wrapped in a stream for convenience.
     try (OutputStream gcsStream =
         gcsUtils.openOutputStream(
@@ -124,7 +126,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
       for (int i = 0; i < keyChunks.size(); i++) {
         // Force the async load to finish.
         Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
-        logger.atInfo().log("Loaded %d manifests", chunkValues.size());
+        logger.atInfo().log("Loaded %d manifests.", chunkValues.size());
         // Since there is no hard bound on how much data this might be, take care not to let the
         // Objectify session cache fill up and potentially run out of memory. This is the only safe
         // point to do this since at this point there is no async load in progress.
@@ -134,12 +136,12 @@ public final class ExportCommitLogDiffAction implements Runnable {
           nextChunkToExport = auditedOfy().load().keys(keyChunks.get(i + 1));
         }
         exportChunk(gcsStream, chunkValues);
-        logger.atInfo().log("Exported %d manifests", chunkValues.size());
+        logger.atInfo().log("Exported %d manifests.", chunkValues.size());
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    logger.atInfo().log("Exported %d manifests in total", sortedKeys.size());
+    logger.atInfo().log("Exported %d total manifests.", sortedKeys.size());
   }
 
   /**

core/src/main/java/google/registry/backup/GcsDiffFileLister.java

@@ -33,6 +33,7 @@ import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import google.registry.backup.BackupModule.Backups;
 import google.registry.gcs.GcsUtils;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Map;
@@ -44,6 +45,7 @@ import javax.inject.Provider;
 import org.joda.time.DateTime;
 
 /** Utility class to list commit logs diff files stored on GCS. */
+@DeleteAfterMigration
 class GcsDiffFileLister {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -91,17 +93,17 @@ class GcsDiffFileLister {
 
         // If we hit a gap, quit.
         if (blobInfo == null) {
-          logger.atInfo().log(
+          logger.atWarning().log(
               "Gap discovered in sequence terminating at %s, missing file: %s",
               sequence.lastKey(), filename);
-          logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+          logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
           return false;
         }
       }
       sequence.put(checkpointTime, blobInfo);
       checkpointTime = getLowerBoundTime(blobInfo);
     }
-    logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+    logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
     return true;
   }
 
@@ -140,7 +142,7 @@ class GcsDiffFileLister {
         }
       }
       if (upperBoundTimesToBlobInfo.isEmpty()) {
-        logger.atInfo().log("No files found");
+        logger.atInfo().log("No files found.");
         return ImmutableList.of();
       }
 
@@ -167,6 +169,10 @@ class GcsDiffFileLister {
             break;
           }
           if (!sequence.containsKey(key)) {
+            // Recalculate the sequence for purely informational purposes.
+            logger.atWarning().log(
+                "Fork found in commit log history.  The following sequence "
+                    + "is disconnected from the sequence of the final commit:");
             constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
             checkForMoreExtraDiffs = true;
             inconsistentFileSet = true;
@@ -185,7 +191,7 @@ class GcsDiffFileLister {
 
     logger.atInfo().log(
         "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
-    logger.atInfo().log("Found %d files to restore", sequence.size());
+    logger.atInfo().log("Found %d files to restore.", sequence.size());
     return ImmutableList.copyOf(sequence.values());
   }
 

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -33,6 +33,8 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
 import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
@@ -67,6 +69,7 @@ import org.joda.time.Seconds;
     method = Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class ReplayCommitLogsToSqlAction implements Runnable {
 
   static final String PATH = "/_dr/task/replayCommitLogsToSql";
@@ -111,7 +114,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
@@ -139,7 +142,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 
@@ -222,8 +225,11 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       // Load and process the Datastore transactions one at a time
       ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
           CommitLogImports.loadEntitiesByTransaction(input);
-      allTransactions.forEach(
-          transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
+      try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+          UpdateAutoTimestamp.disableAutoUpdate()) {
+        allTransactions.forEach(
+            transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
+      }
       // if we succeeded, set the last-seen time
       DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
       jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
@@ -260,7 +266,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             .ifPresent(
                 sqlEntity -> {
                   sqlEntity.beforeSqlSaveOnReplay();
-                  jpaTm().put(sqlEntity);
+                  jpaTm().putIgnoringReadOnlyWithoutBackup(sqlEntity);
                 });
       } else {
         // this should never happen, but we shouldn't fail on it
@@ -269,7 +275,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             ofyPojo.getClass());
       }
     } catch (Throwable t) {
-      logger.atSevere().log("Error when replaying object %s", ofyPojo);
+      logger.atSevere().log("Error when replaying object %s.", ofyPojo);
       throw t;
     }
   }
@@ -293,10 +299,10 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
           && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
           && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
         ReplaySpecializer.beforeSqlDelete(entityVKey);
-        jpaTm().delete(entityVKey);
+        jpaTm().deleteIgnoringReadOnlyWithoutBackup(entityVKey);
       }
     } catch (Throwable t) {
-      logger.atSevere().log("Error when deleting key %s", entityVKey);
+      logger.atSevere().log("Error when deleting key %s.", entityVKey);
       throw t;
     }
   }

core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java

@@ -37,6 +37,7 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
@@ -64,6 +65,7 @@ import org.joda.time.DateTime;
     method = Action.Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class RestoreCommitLogsAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -103,13 +105,13 @@ public class RestoreCommitLogsAction implements Runnable {
         !FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
         "DO NOT RUN IN PRODUCTION OR SANDBOX.");
     if (dryRun) {
-      logger.atInfo().log("Running in dryRun mode");
+      logger.atInfo().log("Running in dry-run mode.");
     }
     String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
     logger.atInfo().log("Restoring from %s.", gcsBucket);
     List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
     if (diffFiles.isEmpty()) {
-      logger.atInfo().log("Nothing to restore");
+      logger.atInfo().log("Nothing to restore.");
       return;
     }
     Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
@@ -143,7 +145,7 @@ public class RestoreCommitLogsAction implements Runnable {
                                 .build()),
                 Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
             .collect(toImmutableList()));
-    logger.atInfo().log("Restore complete");
+    logger.atInfo().log("Restore complete.");
   }
 
   /**

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -19,6 +19,7 @@ import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.appengine.api.datastore.Key;
 import com.google.auto.value.AutoValue;
 import com.google.auto.value.extension.memoized.Memoized;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
 import java.io.Serializable;
@@ -76,6 +77,7 @@ import javax.annotation.Nullable;
  * property type in this class.
  */
 @AutoValue
+@DeleteAfterMigration
 public abstract class VersionedEntity implements Serializable {
 
   private static final long serialVersionUID = 1L;
@@ -105,29 +107,29 @@ public abstract class VersionedEntity implements Serializable {
    * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
    * information.
    */
-  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+  static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
     long commitTimeMillis = manifest.getCommitTime().getMillis();
     return manifest.getDeletions().stream()
         .map(com.googlecode.objectify.Key::getRaw)
-        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+        .map(key -> newBuilder().commitTimeMills(commitTimeMillis).key(key).build());
   }
 
   /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
-  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+  static VersionedEntity fromMutation(CommitLogMutation mutation) {
     return from(
         com.googlecode.objectify.Key.create(mutation).getParent().getId(),
         mutation.getEntityProtoBytes());
   }
 
   public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
-    return builder()
+    return newBuilder()
         .entityProtoBytes(entityProtoBytes)
         .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
         .commitTimeMills(commitTimeMillis)
         .build();
   }
 
-  static Builder builder() {
+  private static Builder newBuilder() {
     return new AutoValue_VersionedEntity.Builder();
   }
 
@@ -142,7 +144,7 @@ public abstract class VersionedEntity implements Serializable {
 
     public abstract VersionedEntity build();
 
-    public Builder entityProtoBytes(byte[] bytes) {
+    Builder entityProtoBytes(byte[] bytes) {
       return entityProtoBytes(new ImmutableBytes(bytes));
     }
   }

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,10 +24,8 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
@@ -84,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -96,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -114,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -126,18 +122,17 @@ public final class AsyncTaskEnqueuer {
   public void enqueueAsyncDelete(
       EppResource resourceToDelete,
       DateTime now,
-      String requestingClientId,
+      String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingClientId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
-            .param(PARAM_REQUESTING_CLIENT_ID, requestingClientId)
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().stringify())
+            .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))
             .param(PARAM_REQUESTED_TIME, now.toString());
@@ -153,7 +148,7 @@ public final class AsyncTaskEnqueuer {
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,
         TaskOptions.Builder.withMethod(Method.PULL)
-            .param(PARAM_HOST_KEY, hostKey.getOfyKey().getString())
+            .param(PARAM_HOST_KEY, hostKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString()));
   }
 

core/src/main/java/google/registry/batch/BatchModule.java

@@ -24,6 +24,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.request.RequestParameters.extractIntParameter;
 import static google.registry.request.RequestParameters.extractLongParameter;
 import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
+import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter;
 import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
@@ -32,10 +33,8 @@ import static google.registry.request.RequestParameters.extractSetOfDatetimePara
 
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.model.ImmutableObject;
 import google.registry.request.Parameter;
 import java.util.Optional;
 import javax.inject.Named;
@@ -78,8 +77,8 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
-  static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
-    return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
+  static String provideResourceKey(HttpServletRequest req) {
+    return extractRequiredParameter(req, PARAM_RESOURCE_KEY);
   }
 
   @Provides
@@ -106,6 +105,13 @@ public class BatchModule {
     return extractIntParameter(req, RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM);
   }
 
+  @Provides
+  @Parameter(ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME)
+  static Optional<DateTime> provideCursorTime(HttpServletRequest req) {
+    return extractOptionalDatetimeParameter(
+        req, ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME);
+  }
+
   @Provides
   @Named(QUEUE_ASYNC_ACTIONS)
   static Queue provideAsyncActionsPushQueue() {

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -73,6 +73,7 @@ import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.mapreduce.inputs.NullInput;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
@@ -114,6 +115,7 @@ import org.joda.time.Duration;
     service = Action.Service.BACKEND,
     path = "/_dr/task/deleteContactsAndHosts",
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class DeleteContactsAndHostsAction implements Runnable {
 
   static final String KIND_CONTACT = getKind(ContactResource.class);
@@ -311,7 +313,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
     @Override
     public void reduce(final DeletionRequest deletionRequest, ReducerInput<Boolean> values) {
       final boolean hasNoActiveReferences = !Iterators.contains(values, true);
-      logger.atInfo().log("Processing async deletion request for %s", deletionRequest.key());
+      logger.atInfo().log("Processing async deletion request for %s.", deletionRequest.key());
       DeletionResult result =
           tm()
               .transactNew(
@@ -343,15 +345,15 @@ public class DeleteContactsAndHostsAction implements Runnable {
       }
       // Contacts and external hosts have a direct client id. For subordinate hosts it needs to be
       // read off of the superordinate domain.
-      String resourceClientId = resource.getPersistedCurrentSponsorClientId();
+      String resourceRegistrarId = resource.getPersistedCurrentSponsorRegistrarId();
       if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
-        resourceClientId =
+        resourceRegistrarId =
             tm().loadByKey(((HostResource) resource).getSuperordinateDomain())
                 .cloneProjectedAtTime(now)
-                .getCurrentSponsorClientId();
+                .getCurrentSponsorRegistrarId();
       }
       boolean requestedByCurrentOwner =
-          resourceClientId.equals(deletionRequest.requestingClientId());
+          resourceRegistrarId.equals(deletionRequest.requestingClientId());
 
       boolean deleteAllowed =
           hasNoActiveReferences && (requestedByCurrentOwner || deletionRequest.isSuperuser());
@@ -371,14 +373,14 @@ public class DeleteContactsAndHostsAction implements Runnable {
 
       HistoryEntry historyEntry =
           HistoryEntry.createBuilderForResource(resource)
-              .setClientId(deletionRequest.requestingClientId())
+              .setRegistrarId(deletionRequest.requestingClientId())
               .setModificationTime(now)
               .setType(getHistoryEntryType(resource, deleteAllowed))
               .build();
 
       PollMessage.OneTime pollMessage =
           new PollMessage.OneTime.Builder()
-              .setClientId(deletionRequest.requestingClientId())
+              .setRegistrarId(deletionRequest.requestingClientId())
               .setMsg(pollMessageText)
               .setParent(historyEntry)
               .setEventTime(now)
@@ -523,18 +525,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(
@@ -605,12 +608,12 @@ public class DeleteContactsAndHostsAction implements Runnable {
   static boolean doesResourceStateAllowDeletion(EppResource resource, DateTime now) {
     Key<EppResource> key = Key.create(resource);
     if (isDeleted(resource, now)) {
-      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted", key);
+      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted.", key);
       return false;
     }
     if (!resource.getStatusValues().contains(PENDING_DELETE)) {
       logger.atWarning().log(
-          "Cannot asynchronously delete %s because it is not in PENDING_DELETE", key);
+          "Cannot asynchronously delete %s because it is not in PENDING_DELETE.", key);
       return false;
     }
     return true;

core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java

@@ -167,7 +167,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
   /** Runs the actual domain delete flow and returns whether the deletion was successful. */
   private boolean runDomainDeleteFlow(DomainBase domain) {
-    logger.atInfo().log("Attempting to delete domain %s", domain.getDomainName());
+    logger.atInfo().log("Attempting to delete domain '%s'.", domain.getDomainName());
     // Create a new transaction that the flow's execution will be enlisted in that loads the domain
     // transactionally. This way we can ensure that nothing else has modified the domain in question
     // in the intervening period since the query above found it.
@@ -203,7 +203,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
     if (eppOutput.isPresent()) {
       if (eppOutput.get().isSuccess()) {
-        logger.atInfo().log("Successfully deleted domain %s", domain.getDomainName());
+        logger.atInfo().log("Successfully deleted domain '%s'.", domain.getDomainName());
       } else {
         logger.atSevere().log(
             "Failed to delete domain %s; EPP response:\n\n%s",

core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java

@@ -135,14 +135,14 @@ public class DeleteLoadTestDataAction implements Runnable {
   }
 
   private void deleteContact(ContactResource contact) {
-    if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorClientId())) {
+    if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorRegistrarId())) {
       return;
     }
     // We cannot remove contacts from domains in the general case, so we cannot delete contacts
     // that are linked to domains (since it would break the foreign keys)
     if (EppResourceUtils.isLinked(contact.createVKey(), clock.nowUtc())) {
       logger.atWarning().log(
-          "Cannot delete contact with repo ID %s since it is referenced from a domain",
+          "Cannot delete contact with repo ID %s since it is referenced from a domain.",
           contact.getRepoId());
       return;
     }
@@ -150,7 +150,7 @@ public class DeleteLoadTestDataAction implements Runnable {
   }
 
   private void deleteHost(HostResource host) {
-    if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorClientId())) {
+    if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorRegistrarId())) {
       return;
     }
     VKey<HostResource> hostVKey = host.createVKey();
@@ -177,7 +177,7 @@ public class DeleteLoadTestDataAction implements Runnable {
         HistoryEntryDao.loadHistoryObjectsForResource(eppResource.createVKey());
     if (isDryRun) {
       logger.atInfo().log(
-          "Would delete repo ID %s along with %d history objects",
+          "Would delete repo ID %s along with %d history objects.",
           eppResource.getRepoId(), historyObjects.size());
     } else {
       historyObjects.forEach(tm()::delete);
@@ -198,7 +198,7 @@ public class DeleteLoadTestDataAction implements Runnable {
 
     @Override
     public final void map(EppResource resource) {
-      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorClientId())) {
+      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorRegistrarId())) {
         deleteResource(resource);
         getContext()
             .incrementCounter(

core/src/main/java/google/registry/batch/DeleteProberDataAction.java

@@ -23,6 +23,7 @@ import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexD
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
 import static google.registry.model.tld.Registries.getTldsOfType;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
@@ -42,6 +43,7 @@ import google.registry.config.RegistryEnvironment;
 import google.registry.dns.DnsQueue;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
+import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.EppResourceUtils;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainHistory;
@@ -54,15 +56,18 @@ import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.inject.Inject;
+import org.hibernate.CacheMode;
+import org.hibernate.ScrollMode;
+import org.hibernate.ScrollableResults;
+import org.hibernate.query.Query;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /**
- * Deletes all prober DomainBases and their subordinate history entries, poll messages, and
- * billing events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
- *
- * <p>See: https://www.youtube.com/watch?v=xuuv0syoHnM
+ * Deletes all prober DomainBases and their subordinate history entries, poll messages, and billing
+ * events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
  */
 @Action(
     service = Action.Service.BACKEND,
@@ -73,10 +78,51 @@ public class DeleteProberDataAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
+  /**
+   * The maximum amount of time we allow a prober domain to be in use.
+   *
+   * <p>In practice, the prober's connection will time out well before this duration. This includes
+   * a decent buffer.
+   */
+  private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
+
+  /**
+   * The minimum amount of time we want a domain to be "soft deleted".
+   *
+   * <p>The domain has to remain soft deleted for at least enough time for the DNS task to run and
+   * remove it from DNS itself. This is probably on the order of minutes.
+   */
+  private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
+
+  private static final DnsQueue dnsQueue = DnsQueue.create();
+
+  // Domains to delete must:
+  // 1. Be in one of the prober TLDs
+  // 2. Not be a nic domain
+  // 3. Have no subordinate hosts
+  // 4. Not still be used (within an hour of creation time)
+  // 5. Either be active (creationTime <= now < deletionTime) or have been deleted a while ago (this
+  //    prevents accidental double-map with the same key from immediately deleting active domains)
+  //
+  // Note: creationTime must be compared to a Java object (CreateAutoTimestamp) but deletionTime can
+  // be compared directly to the SQL timestamp (it's a DateTime)
+  private static final String DOMAIN_QUERY_STRING =
+      "FROM Domain d WHERE d.tld IN :tlds AND d.fullyQualifiedDomainName NOT LIKE 'nic.%' AND"
+          + " (d.subordinateHosts IS EMPTY OR d.subordinateHosts IS NULL) AND d.creationTime <"
+          + " :creationTimeCutoff AND ((d.creationTime <= :nowAutoTimestamp AND d.deletionTime >"
+          + " current_timestamp()) OR d.deletionTime < :nowMinusSoftDeleteDelay) ORDER BY d.repoId";
+
+  /** Number of domains to retrieve and delete per SQL transaction. */
+  private static final int BATCH_SIZE = 1000;
+
   @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
   /** List of TLDs to work on. If empty - will work on all TLDs that end with .test. */
   @Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds;
-  @Inject @Config("registryAdminClientId") String registryAdminClientId;
+
+  @Inject
+  @Config("registryAdminClientId")
+  String registryAdminRegistrarId;
+
   @Inject MapreduceRunner mrRunner;
   @Inject Response response;
   @Inject DeleteProberDataAction() {}
@@ -84,25 +130,14 @@ public class DeleteProberDataAction implements Runnable {
   @Override
   public void run() {
     checkState(
-        !Strings.isNullOrEmpty(registryAdminClientId),
+        !Strings.isNullOrEmpty(registryAdminRegistrarId),
         "Registry admin client ID must be configured for prober data deletion to work");
-    mrRunner
-        .setJobName("Delete prober data")
-        .setModuleName("backend")
-        .runMapOnly(
-            new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId),
-            ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
-        .sendLinkToMapreduceConsole(response);
-  }
-
-  private ImmutableSet<String> getProberRoidSuffixes() {
     checkArgument(
         !PRODUCTION.equals(RegistryEnvironment.get())
             || tlds.stream().allMatch(tld -> tld.endsWith(".test")),
         "On production, can only work on TLDs that end with .test");
     ImmutableSet<String> deletableTlds =
-        getTldsOfType(TldType.TEST)
-            .stream()
+        getTldsOfType(TldType.TEST).stream()
             .filter(tld -> tlds.isEmpty() ? tld.endsWith(".test") : tlds.contains(tld))
             .collect(toImmutableSet());
     checkArgument(
@@ -110,10 +145,161 @@ public class DeleteProberDataAction implements Runnable {
         "If tlds are given, they must all exist and be TEST tlds. Given: %s, not found: %s",
         tlds,
         Sets.difference(tlds, deletableTlds));
-    return deletableTlds
-        .stream()
-        .map(tld -> Registry.get(tld).getRoidSuffix())
-        .collect(toImmutableSet());
+    ImmutableSet<String> proberRoidSuffixes =
+        deletableTlds.stream()
+            .map(tld -> Registry.get(tld).getRoidSuffix())
+            .collect(toImmutableSet());
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Delete prober data")
+          .setModuleName("backend")
+          .runMapOnly(
+              new DeleteProberDataMapper(proberRoidSuffixes, isDryRun, registryAdminRegistrarId),
+              ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      runSqlJob(deletableTlds);
+    }
+  }
+
+  private void runSqlJob(ImmutableSet<String> deletableTlds) {
+    AtomicInteger softDeletedDomains = new AtomicInteger();
+    AtomicInteger hardDeletedDomains = new AtomicInteger();
+    jpaTm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have soft-deleted" : "Soft-deleted", softDeletedDomains.get());
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have hard-deleted" : "Hard-deleted", hardDeletedDomains.get());
+  }
+
+  private void processDomains(
+      ImmutableSet<String> deletableTlds,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    DateTime now = tm().getTransactionTime();
+    // Scroll through domains, soft-deleting as necessary (very few will be soft-deleted) and
+    // keeping track of which domains to hard-delete (there can be many, so we batch them up)
+    ScrollableResults scrollableResult =
+        jpaTm()
+            .query(DOMAIN_QUERY_STRING, DomainBase.class)
+            .setParameter("tlds", deletableTlds)
+            .setParameter(
+                "creationTimeCutoff", CreateAutoTimestamp.create(now.minus(DOMAIN_USED_DURATION)))
+            .setParameter("nowMinusSoftDeleteDelay", now.minus(SOFT_DELETE_DELAY))
+            .setParameter("nowAutoTimestamp", CreateAutoTimestamp.create(now))
+            .unwrap(Query.class)
+            .setCacheMode(CacheMode.IGNORE)
+            .scroll(ScrollMode.FORWARD_ONLY);
+    ImmutableList.Builder<String> domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+    ImmutableList.Builder<String> hostNamesToHardDelete = new ImmutableList.Builder<>();
+    for (int i = 1; scrollableResult.next(); i = (i + 1) % BATCH_SIZE) {
+      DomainBase domain = (DomainBase) scrollableResult.get(0);
+      processDomain(
+          domain,
+          domainRepoIdsToHardDelete,
+          hostNamesToHardDelete,
+          softDeletedDomains,
+          hardDeletedDomains);
+      // Batch the deletion and DB flush + session clearing so we don't OOM
+      if (i == 0) {
+        hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+        domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+        hostNamesToHardDelete = new ImmutableList.Builder<>();
+        jpaTm().getEntityManager().flush();
+        jpaTm().getEntityManager().clear();
+      }
+    }
+    // process the remainder
+    hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+  }
+
+  private void processDomain(
+      DomainBase domain,
+      ImmutableList.Builder<String> domainRepoIdsToHardDelete,
+      ImmutableList.Builder<String> hostNamesToHardDelete,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    // If the domain is still active, that means that the prober encountered a failure and did not
+    // successfully soft-delete the domain (thus leaving its DNS entry published). We soft-delete
+    // it now so that the DNS entry can be handled. The domain will then be hard-deleted the next
+    // time the job is run.
+    if (EppResourceUtils.isActive(domain, tm().getTransactionTime())) {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would soft-delete the active domain: %s (%s).",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue);
+      }
+      softDeletedDomains.incrementAndGet();
+    } else {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would hard-delete the non-active domain: %s (%s) and its dependents.",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        domainRepoIdsToHardDelete.add(domain.getRepoId());
+        hostNamesToHardDelete.addAll(domain.getSubordinateHosts());
+      }
+      hardDeletedDomains.incrementAndGet();
+    }
+  }
+
+  private void hardDeleteDomainsAndHosts(
+      ImmutableList<String> domainRepoIds, ImmutableList<String> hostNames) {
+    jpaTm()
+        .query("DELETE FROM Host WHERE fullyQualifiedHostName IN :hostNames")
+        .setParameter("hostNames", hostNames)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingRecurrence WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM DomainHistory WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM Domain WHERE repoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+  }
+
+  // Take a DNS queue + admin registrar id as input so that it can be called from the mapper as well
+  private static void softDeleteDomain(
+      DomainBase domain, String registryAdminRegistrarId, DnsQueue localDnsQueue) {
+    DomainBase deletedDomain =
+        domain.asBuilder().setDeletionTime(tm().getTransactionTime()).setStatusValues(null).build();
+    DomainHistory historyEntry =
+        new DomainHistory.Builder()
+            .setDomain(domain)
+            .setType(DOMAIN_DELETE)
+            .setModificationTime(tm().getTransactionTime())
+            .setBySuperuser(true)
+            .setReason("Deletion of prober data")
+            .setRegistrarId(registryAdminRegistrarId)
+            .build();
+    // Note that we don't bother handling grace periods, billing events, pending transfers, poll
+    // messages, or auto-renews because those will all be hard-deleted the next time the job runs
+    // anyway.
+    tm().putAllWithoutBackup(ImmutableList.of(deletedDomain, historyEntry));
+    // updating foreign keys is a no-op in SQL
+    updateForeignKeyIndexDeletionTime(deletedDomain);
+    localDnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
   }
 
   /** Provides the map method that runs for each existing DomainBase entity. */
@@ -122,32 +308,17 @@ public class DeleteProberDataAction implements Runnable {
     private static final DnsQueue dnsQueue = DnsQueue.create();
     private static final long serialVersionUID = -7724537393697576369L;
 
-    /**
-     * The maximum amount of time we allow a prober domain to be in use.
-     *
-     * In practice, the prober's connection will time out well before this duration. This includes a
-     * decent buffer.
-     *
-     */
-    private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
-
-    /**
-     * The minimum amount of time we want a domain to be "soft deleted".
-     *
-     * The domain has to remain soft deleted for at least enough time for the DNS task to run and
-     * remove it from DNS itself. This is probably on the order of minutes.
-     */
-    private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
-
     private final ImmutableSet<String> proberRoidSuffixes;
     private final Boolean isDryRun;
-    private final String registryAdminClientId;
+    private final String registryAdminRegistrarId;
 
     public DeleteProberDataMapper(
-        ImmutableSet<String> proberRoidSuffixes, Boolean isDryRun, String registryAdminClientId) {
+        ImmutableSet<String> proberRoidSuffixes,
+        Boolean isDryRun,
+        String registryAdminRegistrarId) {
       this.proberRoidSuffixes = proberRoidSuffixes;
       this.isDryRun = isDryRun;
-      this.registryAdminClientId = registryAdminClientId;
+      this.registryAdminRegistrarId = registryAdminRegistrarId;
     }
 
     @Override
@@ -160,7 +331,7 @@ public class DeleteProberDataAction implements Runnable {
           getContext().incrementCounter("skipped, non-prober data");
         }
       } catch (Throwable t) {
-        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s", key);
+        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s.", key);
         getContext().incrementCounter(String.format("error, kind %s", key.getKind()));
       }
     }
@@ -201,9 +372,9 @@ public class DeleteProberDataAction implements Runnable {
       if (EppResourceUtils.isActive(domain, now)) {
         if (isDryRun) {
           logger.atInfo().log(
-              "Would soft-delete the active domain: %s (%s)", domainName, domainKey);
+              "Would soft-delete the active domain: %s (%s).", domainName, domainKey);
         } else {
-          softDeleteDomain(domain);
+          tm().transact(() -> softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue));
         }
         getContext().incrementCounter("domains soft-deleted");
         return;
@@ -223,8 +394,7 @@ public class DeleteProberDataAction implements Runnable {
           tm().transact(
                   () -> {
                     // This ancestor query selects all descendant HistoryEntries, BillingEvents,
-                    // PollMessages,
-                    // and TLD-specific entities, as well as the domain itself.
+                    // PollMessages, and TLD-specific entities, as well as the domain itself.
                     List<Key<Object>> domainAndDependentKeys =
                         auditedOfy().load().ancestor(domainKey).keys().list();
                     ImmutableSet<Key<?>> allKeys =
@@ -243,32 +413,5 @@ public class DeleteProberDataAction implements Runnable {
       getContext().incrementCounter("domains hard-deleted");
       getContext().incrementCounter("total entities hard-deleted", entitiesDeleted);
     }
-
-    private void softDeleteDomain(final DomainBase domain) {
-      tm().transactNew(
-              () -> {
-                DomainBase deletedDomain =
-                    domain
-                        .asBuilder()
-                        .setDeletionTime(tm().getTransactionTime())
-                        .setStatusValues(null)
-                        .build();
-                DomainHistory historyEntry =
-                    new DomainHistory.Builder()
-                        .setDomain(domain)
-                        .setType(DOMAIN_DELETE)
-                        .setModificationTime(tm().getTransactionTime())
-                        .setBySuperuser(true)
-                        .setReason("Deletion of prober data")
-                        .setClientId(registryAdminClientId)
-                        .build();
-                // Note that we don't bother handling grace periods, billing events, pending
-                // transfers, poll messages, or auto-renews because these will all be hard-deleted
-                // the next time the mapreduce runs anyway.
-                tm().putAll(deletedDomain, historyEntry);
-                updateForeignKeyIndexDeletionTime(deletedDomain);
-                dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
-              });
-    }
   }
 }

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -148,9 +148,9 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
               .reduce(0, Integer::sum);
 
       if (!isDryRun) {
-        logger.atInfo().log("Saved OneTime billing events", numBillingEventsSaved);
+        logger.atInfo().log("Saved OneTime billing events.", numBillingEventsSaved);
       } else {
-        logger.atInfo().log("Generated OneTime billing events (dry run)", numBillingEventsSaved);
+        logger.atInfo().log("Generated OneTime billing events (dry run).", numBillingEventsSaved);
       }
       logger.atInfo().log(
           "Recurring event expansion %s complete for billing event range [%s, %s).",
@@ -324,7 +324,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       DomainHistory historyEntry =
           new DomainHistory.Builder()
               .setBySuperuser(false)
-              .setClientId(recurring.getClientId())
+              .setRegistrarId(recurring.getRegistrarId())
               .setModificationTime(tm().getTransactionTime())
               .setDomain(tm().loadByKey(domainKey))
               .setPeriod(Period.create(1, YEARS))
@@ -354,7 +354,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       syntheticOneTimesBuilder.add(
           new OneTime.Builder()
               .setBillingTime(billingTime)
-              .setClientId(recurring.getClientId())
+              .setRegistrarId(recurring.getRegistrarId())
               .setCost(renewCost)
               .setEventTime(eventTime)
               .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -173,7 +173,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
                     retrier.callWithRetry(
                         () -> dnsQueue.addDomainRefreshTask(domainName),
                         TransientFailureException.class);
-                    logger.atInfo().log("Enqueued DNS refresh for domain %s.", domainName);
+                    logger.atInfo().log("Enqueued DNS refresh for domain '%s'.", domainName);
                   });
           deleteTasksWithRetry(
               refreshRequests,
@@ -353,8 +353,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
       VKey<HostResource> hostKey =
-          VKey.fromWebsafeKey(
-              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
           tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
               .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -203,11 +203,11 @@ public class RelockDomainAction implements Runnable {
         "Domain %s has a pending transfer.",
         domainName);
     checkArgument(
-        domain.getCurrentSponsorClientId().equals(oldLock.getRegistrarId()),
+        domain.getCurrentSponsorRegistrarId().equals(oldLock.getRegistrarId()),
         "Domain %s has been transferred from registrar %s to registrar %s since the unlock.",
         domainName,
         oldLock.getRegistrarId(),
-        domain.getCurrentSponsorClientId());
+        domain.getCurrentSponsorRegistrarId());
   }
 
   private void handleNonRetryableFailure(RegistryLock oldLock, Throwable t) {
@@ -293,7 +293,7 @@ public class RelockDomainAction implements Runnable {
 
   private ImmutableSet<InternetAddress> getEmailRecipients(String registrarId) {
     Registrar registrar =
-        Registrar.loadByClientIdCached(registrarId)
+        Registrar.loadByRegistrarIdCached(registrarId)
             .orElseThrow(
                 () ->
                     new IllegalStateException(String.format("Unknown registrar %s", registrarId)));
@@ -313,7 +313,7 @@ public class RelockDomainAction implements Runnable {
         builder.add(new InternetAddress(registryLockEmailAddress));
       } catch (AddressException e) {
         // This shouldn't stop any other emails going out, so swallow it
-        logger.atWarning().log("Invalid email address %s", registryLockEmailAddress);
+        logger.atWarning().log("Invalid email address '%s'.", registryLockEmailAddress);
       }
     }
     return builder.build();

core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java

@@ -24,6 +24,7 @@ import com.googlecode.objectify.Key;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -56,6 +57,7 @@ import javax.inject.Inject;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 // No longer needed in SQL. Subject to future removal.
 @Deprecated
+@DeleteAfterMigration
 public class ResaveAllEppResourcesAction implements Runnable {
 
   @Inject MapreduceRunner mrRunner;

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -22,7 +22,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.persistence.VKey;
@@ -50,7 +49,7 @@ public class ResaveEntityAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private final Key<ImmutableObject> resourceKey;
+  private final String resourceKey;
   private final DateTime requestedTime;
   private final ImmutableSortedSet<DateTime> resaveTimes;
   private final AsyncTaskEnqueuer asyncTaskEnqueuer;
@@ -58,7 +57,7 @@ public class ResaveEntityAction implements Runnable {
 
   @Inject
   ResaveEntityAction(
-      @Parameter(PARAM_RESOURCE_KEY) Key<ImmutableObject> resourceKey,
+      @Parameter(PARAM_RESOURCE_KEY) String resourceKey,
       @Parameter(PARAM_REQUESTED_TIME) DateTime requestedTime,
       @Parameter(PARAM_RESAVE_TIMES) ImmutableSet<DateTime> resaveTimes,
       AsyncTaskEnqueuer asyncTaskEnqueuer,
@@ -76,13 +75,14 @@ public class ResaveEntityAction implements Runnable {
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
     tm().transact(
             () -> {
-              ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
+              ImmutableObject entity = tm().loadByKey(VKey.create(resourceKey));
               tm().put(
                       (entity instanceof EppResource)
                           ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
                           : entity);
               if (!resaveTimes.isEmpty()) {
-                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.create(resourceKey), requestedTime, resaveTimes);
               }
             });
     response.setPayload("Entity re-saved.");

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -0,0 +1,353 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.joda.time.DateTimeZone.UTC;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.flows.certs.CertificateChecker;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registrar.RegistrarContact.Type;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.EmailMessage;
+import google.registry.util.SendEmailService;
+import java.util.Date;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.mail.internet.AddressException;
+import javax.mail.internet.InternetAddress;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/** An action that sends notification emails to registrars whose certificates are expiring soon. */
+@Action(
+    service = Action.Service.BACKEND,
+    path = SendExpiringCertificateNotificationEmailAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class SendExpiringCertificateNotificationEmailAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
+  /**
+   * Used as an offset when storing the last notification email sent date.
+   *
+   * <p>This is used to handle edges cases when the update happens in between the day switch. For
+   * instance,if the job starts at 2:00 am every day and it finishes at 2:03 of the same day, then
+   * next day at 2am, the date difference will be less than a day, which will lead to the date
+   * difference between two successive email sent date being the expected email interval days + 1;
+   */
+  protected static final Duration UPDATE_TIME_OFFSET = Duration.standardMinutes(10);
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd");
+
+  private final CertificateChecker certificateChecker;
+  private final String expirationWarningEmailBodyText;
+  private final SendEmailService sendEmailService;
+  private final String expirationWarningEmailSubjectText;
+  private final InternetAddress gSuiteOutgoingEmailAddress;
+  private final Response response;
+
+  @Inject
+  public SendExpiringCertificateNotificationEmailAction(
+      @Config("expirationWarningEmailBodyText") String expirationWarningEmailBodyText,
+      @Config("expirationWarningEmailSubjectText") String expirationWarningEmailSubjectText,
+      @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
+      SendEmailService sendEmailService,
+      CertificateChecker certificateChecker,
+      Response response) {
+    this.certificateChecker = certificateChecker;
+    this.expirationWarningEmailSubjectText = expirationWarningEmailSubjectText;
+    this.sendEmailService = sendEmailService;
+    this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress;
+    this.expirationWarningEmailBodyText = expirationWarningEmailBodyText;
+    this.response = response;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int numEmailsSent = sendNotificationEmails();
+      String message =
+          String.format(
+              "Done. Sent %d expiring certificate notification emails in total.", numEmailsSent);
+      logger.atInfo().log(message);
+      response.setStatus(SC_OK);
+      response.setPayload(message);
+    } catch (Exception e) {
+      logger.atWarning().withCause(e).log(
+          "Exception thrown when sending expiring certificate notification emails.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(String.format("Exception thrown with cause: %s", e));
+    }
+  }
+
+  /**
+   * Returns a list of registrars that should receive expiring notification emails.
+   *
+   * <p>There are two certificates that should be considered (the main certificate and failOver
+   * certificate). The registrars should receive notifications if one of the certificate checks
+   * returns true.
+   */
+  @VisibleForTesting
+  ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
+    logger.atInfo().log(
+        "Getting a list of registrars that should receive expiring notification emails.");
+    return Streams.stream(Registrar.loadAllCached())
+        .map(
+            registrar ->
+                RegistrarInfo.create(
+                    registrar,
+                    registrar.getClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringCertNotificationSentDate(),
+                            registrar.getClientCertificate().get()),
+                    registrar.getFailoverClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringFailoverCertNotificationSentDate(),
+                            registrar.getFailoverClientCertificate().get())))
+        .filter(
+            registrarInfo ->
+                registrarInfo.isCertExpiring() || registrarInfo.isFailOverCertExpiring())
+        .collect(toImmutableList());
+  }
+
+  /**
+   * Sends a notification email to the registrar regarding the expiring certificate and returns true
+   * if it's sent successfully.
+   */
+  @VisibleForTesting
+  boolean sendNotificationEmail(
+      Registrar registrar,
+      DateTime lastExpiringCertNotificationSentDate,
+      CertificateType certificateType,
+      Optional<String> certificate) {
+    if (!certificate.isPresent()
+        || !certificateChecker.shouldReceiveExpiringNotification(
+            lastExpiringCertNotificationSentDate, certificate.get())) {
+      return false;
+    }
+    try {
+      ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      ImmutableSet<InternetAddress> ccs = getEmailAddresses(registrar, Type.ADMIN);
+      Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
+      logger.atInfo().log(
+          " %s SSL certificate of registrar '%s' will expire on %s.",
+          certificateType.getDisplayName(),
+          registrar.getRegistrarName(),
+          expirationDate.toString());
+      if (recipients.isEmpty() && ccs.isEmpty()) {
+        logger.atWarning().log(
+            "Registrar %s contains no TECH nor ADMIN email addresses to receive notification"
+                + " email.",
+            registrar.getRegistrarName());
+        return false;
+      }
+      sendEmailService.sendEmail(
+          EmailMessage.newBuilder()
+              .setFrom(gSuiteOutgoingEmailAddress)
+              .setSubject(expirationWarningEmailSubjectText)
+              .setBody(
+                  getEmailBody(
+                      registrar.getRegistrarName(),
+                      certificateType,
+                      expirationDate,
+                      registrar.getRegistrarId()))
+              .setRecipients(recipients)
+              .setCcs(ccs)
+              .build());
+      /*
+       * A duration time offset is used here to ensure that date comparison between two
+       * successive dates is always greater than 1 day. This date is set as last updated date,
+       * for applicable certificate.
+       */
+      updateLastNotificationSentDate(
+          registrar,
+          DateTime.now(UTC).minusMinutes((int) UPDATE_TIME_OFFSET.getStandardMinutes()),
+          certificateType);
+      return true;
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to send expiring certificate notification email to registrar %s.",
+              registrar.getRegistrarName()));
+    }
+  }
+
+  /** Updates the last notification sent date in database. */
+  @VisibleForTesting
+  void updateLastNotificationSentDate(
+      Registrar registrar, DateTime now, CertificateType certificateType) {
+    try {
+      tm().transact(
+              () -> {
+                Registrar.Builder newRegistrar = tm().loadByEntity(registrar).asBuilder();
+                switch (certificateType) {
+                  case PRIMARY:
+                    newRegistrar.setLastExpiringCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  case FAILOVER:
+                    newRegistrar.setLastExpiringFailoverCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  default:
+                    throw new IllegalArgumentException(
+                        String.format(
+                            "Unsupported certificate type: %s being passed in when updating "
+                                + "the last notification sent date to registrar %s.",
+                            certificateType.toString(), registrar.getRegistrarName()));
+                }
+              });
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to update the last notification sent date to Registrar %s for the %s "
+                  + "certificate.",
+              registrar.getRegistrarName(), certificateType.getDisplayName()));
+    }
+  }
+
+  /** Sends notification emails to registrars with expiring certificates. */
+  @VisibleForTesting
+  int sendNotificationEmails() {
+    int numEmailsSent = 0;
+    for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
+      Registrar registrar = registrarInfo.registrar();
+      if (registrarInfo.isCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringCertNotificationSentDate(),
+              CertificateType.PRIMARY,
+              registrar.getClientCertificate())) {
+        numEmailsSent++;
+      }
+      if (registrarInfo.isFailOverCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringFailoverCertNotificationSentDate(),
+              CertificateType.FAILOVER,
+              registrar.getFailoverClientCertificate())) {
+        numEmailsSent++;
+      }
+    }
+    return numEmailsSent;
+  }
+
+  /**
+   * Returns a list of email addresses of the registrar that should receive a notification email.
+   */
+  @VisibleForTesting
+  ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
+    ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
+    ImmutableSet.Builder<InternetAddress> recipientEmails = new ImmutableSet.Builder<>();
+    for (RegistrarContact contact : contacts) {
+      try {
+        recipientEmails.add(new InternetAddress(contact.getEmailAddress()));
+      } catch (AddressException e) {
+        logger.atWarning().withCause(e).log(
+            "Registrar Contact email address %s of Registrar %s is invalid; skipping.",
+            contact.getEmailAddress(), registrar.getRegistrarName());
+      }
+    }
+    return recipientEmails.build();
+  }
+
+  /**
+   * Generates email content by taking registrar name, certificate type and expiration date as
+   * parameters.
+   */
+  @VisibleForTesting
+  @SuppressWarnings("lgtm[java/dereferenced-value-may-be-null]")
+  String getEmailBody(
+      String registrarName, CertificateType type, Date expirationDate, String registrarId) {
+    checkArgumentNotNull(expirationDate, "Expiration date cannot be null");
+    checkArgumentNotNull(type, "Certificate type cannot be null");
+    checkArgumentNotNull(registrarId, "Registrar Id cannot be null");
+    return String.format(
+        expirationWarningEmailBodyText,
+        registrarName,
+        type.getDisplayName(),
+        DATE_FORMATTER.print(new DateTime(expirationDate)),
+        registrarId);
+  }
+
+  /**
+   * Certificate types for X509Certificate.
+   *
+   * <p><b>Note:</b> These types are only used to indicate the type of expiring certificate in
+   * notification emails.
+   */
+  protected enum CertificateType {
+    PRIMARY("primary"),
+    FAILOVER("fail-over");
+
+    private final String displayName;
+
+    CertificateType(String displayName) {
+      this.displayName = displayName;
+    }
+
+    public String getDisplayName() {
+      return displayName;
+    }
+  }
+
+  @AutoValue
+  public abstract static class RegistrarInfo {
+
+    static RegistrarInfo create(
+        Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
+      return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(
+          registrar, isCertExpiring, isFailOverCertExpiring);
+    }
+
+    public abstract Registrar registrar();
+
+    public abstract boolean isCertExpiring();
+
+    public abstract boolean isFailOverCertExpiring();
+  }
+}

core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java

@@ -0,0 +1,138 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.contact.ContactHistory;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * An action that wipes out Personal Identifiable Information (PII) fields of {@link ContactHistory}
+ * entities.
+ *
+ * <p>ContactHistory entities should be retained in the database for only certain amount of time.
+ * This periodic wipe out action only applies to SQL.
+ */
+@Action(
+    service = Service.BACKEND,
+    path = WipeOutContactHistoryPiiAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeOutContactHistoryPiiAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/wipeOutContactHistoryPii";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private final Clock clock;
+  private final Response response;
+  private final int minMonthsBeforeWipeOut;
+  private final int wipeOutQueryBatchSize;
+
+  @Inject
+  public WipeOutContactHistoryPiiAction(
+      Clock clock,
+      @Config("minMonthsBeforeWipeOut") int minMonthsBeforeWipeOut,
+      @Config("wipeOutQueryBatchSize") int wipeOutQueryBatchSize,
+      Response response) {
+    this.clock = clock;
+    this.response = response;
+    this.minMonthsBeforeWipeOut = minMonthsBeforeWipeOut;
+    this.wipeOutQueryBatchSize = wipeOutQueryBatchSize;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int totalNumOfWipedEntities = 0;
+      DateTime wipeOutTime = clock.nowUtc().minusMonths(minMonthsBeforeWipeOut);
+      logger.atInfo().log(
+          "About to wipe out all PII of contact history entities prior to %s.", wipeOutTime);
+
+      int numOfWipedEntities = 0;
+      do {
+        numOfWipedEntities =
+            jpaTm()
+                .transact(
+                    () ->
+                        wipeOutContactHistoryData(
+                            getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime)));
+        totalNumOfWipedEntities += numOfWipedEntities;
+      } while (numOfWipedEntities > 0);
+      String msg =
+          String.format(
+              "Done. Wiped out PII of %d ContactHistory entities in total.",
+              totalNumOfWipedEntities);
+      logger.atInfo().log(msg);
+      response.setPayload(msg);
+      response.setStatus(SC_OK);
+
+    } catch (Exception e) {
+      logger.atSevere().withCause(e).log(
+          "Exception thrown during the process of wiping out contact history PII.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(
+          String.format(
+              "Exception thrown during the process of wiping out contact history PII with cause"
+                  + ": %s",
+              e));
+    }
+  }
+
+  /**
+   * Returns a stream of up to {@link #wipeOutQueryBatchSize} {@link ContactHistory} entities
+   * containing PII that are prior to @param wipeOutTime.
+   */
+  @VisibleForTesting
+  Stream<ContactHistory> getNextContactHistoryEntitiesWithPiiBatch(DateTime wipeOutTime) {
+    // email is one of the required fields in EPP, meaning it's initially not null.
+    // Therefore, checking if it's null is one way to avoid processing contact history entities
+    // that have been processed previously. Refer to RFC 5733 for more information.
+    return jpaTm()
+        .query(
+            "FROM ContactHistory WHERE modificationTime < :wipeOutTime " + "AND email IS NOT NULL",
+            ContactHistory.class)
+        .setParameter("wipeOutTime", wipeOutTime)
+        .setMaxResults(wipeOutQueryBatchSize)
+        .getResultStream();
+  }
+
+  /** Wipes out the PII of each of the {@link ContactHistory} entities in the stream. */
+  @VisibleForTesting
+  int wipeOutContactHistoryData(Stream<ContactHistory> contactHistoryEntities) {
+    AtomicInteger numOfEntities = new AtomicInteger(0);
+    contactHistoryEntities.forEach(
+        contactHistoryEntity -> {
+          jpaTm().update(contactHistoryEntity.asBuilder().wipeOutPii().build());
+          numOfEntities.incrementAndGet();
+        });
+    logger.atInfo().log(
+        "Wiped out all PII fields of %d ContactHistory entities.", numOfEntities.get());
+    return numOfEntities.get();
+  }
+}

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
@@ -45,6 +46,7 @@ import javax.inject.Inject;
     service = Action.Service.BACKEND,
     path = "/_dr/task/wipeOutDatastore",
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class WipeoutDatastoreAction implements Runnable {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -92,7 +94,12 @@ public class WipeoutDatastoreAction implements Runnable {
               .setJobName(createJobName("bulk-delete-datastore-", clock))
               .setContainerSpecGcsPath(
                   String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+              .setParameters(
+                  ImmutableMap.of(
+                      "kindsToDelete",
+                      "*",
+                      "registryEnvironment",
+                      RegistryEnvironment.get().name()));
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -0,0 +1,88 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import java.util.List;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+
+/**
+ * A database snapshot shareable by concurrent queries from multiple database clients. A snapshot is
+ * uniquely identified by its {@link #getSnapshotId snapshotId}, and must stay open until all
+ * concurrent queries to this snapshot have attached to it by calling {@link
+ * google.registry.persistence.transaction.JpaTransactionManager#setDatabaseSnapshot}. However, it
+ * can be closed before those queries complete.
+ *
+ * <p>This feature is <em>Postgresql-only</em>.
+ *
+ * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to avoid
+ * exhausting predicate locks at the SERIALIZABLE level.
+ */
+// TODO(b/193662898): vendor-independent support for richer transaction semantics.
+public class DatabaseSnapshot implements AutoCloseable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private String snapshotId;
+  private EntityManager entityManager;
+  private EntityTransaction transaction;
+
+  private DatabaseSnapshot() {}
+
+  public String getSnapshotId() {
+    checkState(entityManager != null, "Snapshot not opened yet.");
+    checkState(entityManager.isOpen(), "Snapshot already closed.");
+    return snapshotId;
+  }
+
+  private DatabaseSnapshot open() {
+    entityManager = jpaTm().getStandaloneEntityManager();
+    transaction = entityManager.getTransaction();
+    transaction.setRollbackOnly();
+    transaction.begin();
+
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+
+    List<?> snapshotIds =
+        entityManager.createNativeQuery("SELECT pg_export_snapshot();").getResultList();
+    checkState(snapshotIds.size() == 1, "Unexpected number of snapshots: %s", snapshotIds.size());
+    snapshotId = (String) snapshotIds.get(0);
+    return this;
+  }
+
+  @Override
+  public void close() {
+    if (transaction != null && transaction.isActive()) {
+      try {
+        transaction.rollback();
+      } catch (Exception e) {
+        logger.atWarning().withCause(e).log("Failed to close a Database Snapshot");
+      }
+    }
+    if (entityManager != null && entityManager.isOpen()) {
+      entityManager.close();
+    }
+  }
+
+  public static DatabaseSnapshot createSnapshot() {
+    return new DatabaseSnapshot().open();
+  }
+}

core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java

@@ -17,7 +17,6 @@ package google.registry.beam.common;
 import static com.google.common.base.Verify.verify;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.model.contact.ContactResource;
 import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -59,18 +58,16 @@ public class JpaDemoPipeline implements Serializable {
                   public void processElement() {
                     // AppEngineEnvironment is needed as long as JPA entity classes still depends
                     // on Objectify.
-                    try (AppEngineEnvironment allowOfyEntity = new AppEngineEnvironment()) {
-                      int result =
-                          (Integer)
-                              jpaTm()
-                                  .transact(
-                                      () ->
-                                          jpaTm()
-                                              .getEntityManager()
-                                              .createNativeQuery("select 1;")
-                                              .getSingleResult());
-                      verify(result == 1, "Expecting 1, got %s.", result);
-                    }
+                    int result =
+                        (Integer)
+                            jpaTm()
+                                .transact(
+                                    () ->
+                                        jpaTm()
+                                            .getEntityManager()
+                                            .createNativeQuery("select 1;")
+                                            .getSingleResult());
+                    verify(result == 1, "Expecting 1, got %s.", result);
                     counter.inc();
                   }
                 }));

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -20,9 +20,9 @@ import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
-import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
 import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -138,6 +138,9 @@ public final class RegistryJpaIO {
 
     abstract Coder<T> coder();
 
+    @Nullable
+    abstract String snapshotId();
+
     abstract Builder<R, T> toBuilder();
 
     @Override
@@ -145,7 +148,9 @@ public final class RegistryJpaIO {
     public PCollection<T> expand(PBegin input) {
       return input
           .apply("Starting " + name(), Create.of((Void) null))
-          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
+          .apply(
+              "Run query for " + name(),
+              ParDo.of(new QueryRunner<>(query(), resultMapper(), snapshotId())))
           .setCoder(coder())
           .apply("Reshuffle", Reshuffle.viaRandomKey());
     }
@@ -162,6 +167,18 @@ public final class RegistryJpaIO {
       return toBuilder().coder(coder).build();
     }
 
+    /**
+     * Specifies the database snapshot to use for this query.
+     *
+     * <p>This feature is <em>Postgresql-only</em>. User is responsible for keeping the snapshot
+     * available until all JVM workers have started using it by calling {@link
+     * JpaTransactionManager#setDatabaseSnapshot}.
+     */
+    // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+    public Read<R, T> withSnapshot(@Nullable String snapshotId) {
+      return toBuilder().snapshotId(snapshotId).build();
+    }
+
     static <R, T> Builder<R, T> builder() {
       return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
           .name(DEFAULT_NAME)
@@ -179,6 +196,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<R, T> coder(Coder coder);
 
+      abstract Builder<R, T> snapshotId(@Nullable String sharedSnapshotId);
+
       abstract Read<R, T> build();
 
       Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
@@ -201,22 +220,28 @@ public final class RegistryJpaIO {
     static class QueryRunner<R, T> extends DoFn<Void, T> {
       private final RegistryQuery<R> query;
       private final SerializableFunction<R, T> resultMapper;
+      // java.util.Optional is not serializable. Use of Guava Optional is discouraged.
+      @Nullable private final String snapshotId;
 
-      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+      QueryRunner(
+          RegistryQuery<R> query,
+          SerializableFunction<R, T> resultMapper,
+          @Nullable String snapshotId) {
         this.query = query;
         this.resultMapper = resultMapper;
+        this.snapshotId = snapshotId;
       }
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
-        // AppEngineEnvironment is need for handling VKeys, which involve Ofy keys. Unlike
-        // SqlBatchWriter, it is unnecessary to initialize ObjectifyService in this class.
-        try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-          // TODO(b/187210388): JpaTransactionManager should support non-transactional query.
-          jpaTm()
-              .transactNoRetry(
-                  () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
-        }
+        jpaTm()
+            .transactNoRetry(
+                () -> {
+                  if (snapshotId != null) {
+                    jpaTm().setDatabaseSnapshot(snapshotId);
+                  }
+                  query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
+                });
       }
     }
   }
@@ -274,6 +299,12 @@ public final class RegistryJpaIO {
 
     public abstract SerializableFunction<T, Object> jpaConverter();
 
+    /**
+     * Signal to the writer that the {@link UpdateAutoTimestamp} property should be allowed to
+     * manipulate its value before persistence. The default value is {@code true}.
+     */
+    abstract boolean withUpdateAutoTimestamp();
+
     public Write<T> withName(String name) {
       return toBuilder().name(name).build();
     }
@@ -294,6 +325,10 @@ public final class RegistryJpaIO {
       return toBuilder().jpaConverter(jpaConverter).build();
     }
 
+    public Write<T> disableUpdateAutoTimestamp() {
+      return toBuilder().withUpdateAutoTimestamp(false).build();
+    }
+
     abstract Builder<T> toBuilder();
 
     @Override
@@ -310,7 +345,7 @@ public final class RegistryJpaIO {
               GroupIntoBatches.<Integer, T>ofSize(batchSize()).withShardedKey())
           .apply(
               "Write in batch for " + name(),
-              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter())));
+              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter(), withUpdateAutoTimestamp())));
     }
 
     static <T> Builder<T> builder() {
@@ -318,7 +353,8 @@ public final class RegistryJpaIO {
           .name(DEFAULT_NAME)
           .batchSize(DEFAULT_BATCH_SIZE)
           .shards(DEFAULT_SHARDS)
-          .jpaConverter(x -> x);
+          .jpaConverter(x -> x)
+          .withUpdateAutoTimestamp(true);
     }
 
     @AutoValue.Builder
@@ -332,6 +368,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<T> jpaConverter(SerializableFunction<T, Object> jpaConverter);
 
+      abstract Builder<T> withUpdateAutoTimestamp(boolean withUpdateAutoTimestamp);
+
       abstract Write<T> build();
     }
   }
@@ -340,37 +378,38 @@ public final class RegistryJpaIO {
   private static class SqlBatchWriter<T> extends DoFn<KV<ShardedKey<Integer>, Iterable<T>>, Void> {
     private final Counter counter;
     private final SerializableFunction<T, Object> jpaConverter;
+    private final boolean withAutoTimestamp;
 
-    SqlBatchWriter(String type, SerializableFunction<T, Object> jpaConverter) {
+    SqlBatchWriter(
+        String type, SerializableFunction<T, Object> jpaConverter, boolean withAutoTimestamp) {
       counter = Metrics.counter("SQL_WRITE", type);
       this.jpaConverter = jpaConverter;
-    }
-
-    @Setup
-    public void setup() {
-      // AppEngineEnvironment is needed as long as Objectify keys are still involved in the handling
-      // of SQL entities (e.g., in VKeys). ObjectifyService needs to be initialized when conversion
-      // between Ofy entity and Datastore entity is needed.
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ObjectifyService.initOfy();
-      }
+      this.withAutoTimestamp = withAutoTimestamp;
     }
 
     @ProcessElement
     public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ImmutableList<Object> entities =
-            Streams.stream(kv.getValue())
-                .map(this.jpaConverter::apply)
-                // TODO(b/177340730): post migration delete the line below.
-                .filter(Objects::nonNull)
-                .collect(ImmutableList.toImmutableList());
-        try {
-          jpaTm().transact(() -> jpaTm().putAll(entities));
-          counter.inc(entities.size());
-        } catch (RuntimeException e) {
-          processSingly(entities);
-        }
+      if (withAutoTimestamp) {
+        actuallyProcessElement(kv);
+        return;
+      }
+      try (DisableAutoUpdateResource disable = UpdateAutoTimestamp.disableAutoUpdate()) {
+        actuallyProcessElement(kv);
+      }
+    }
+
+    private void actuallyProcessElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
+      ImmutableList<Object> entities =
+          Streams.stream(kv.getValue())
+              .map(this.jpaConverter::apply)
+              // TODO(b/177340730): post migration delete the line below.
+              .filter(Objects::nonNull)
+              .collect(ImmutableList.toImmutableList());
+      try {
+        jpaTm().transact(() -> jpaTm().putAll(entities));
+        counter.inc(entities.size());
+      } catch (RuntimeException e) {
+        processSingly(entities);
       }
     }
 

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -21,7 +21,9 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
+import google.registry.persistence.PersistenceModule.BeamReadOnlyReplicaJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.privileges.secretmanager.SecretManagerModule;
@@ -45,9 +47,26 @@ public interface RegistryPipelineComponent {
   @Config("projectId")
   String getProjectId();
 
+  /** Returns the regular {@link JpaTransactionManager} for general use. */
   @BeamJpaTm
   Lazy<JpaTransactionManager> getJpaTransactionManager();
 
+  /**
+   * Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
+   * ({@link google.registry.model.domain.DomainBase} and {@link
+   * google.registry.model.domain.DomainHistory}). Please refer to {@link
+   * google.registry.model.bulkquery.BulkQueryEntities} for more information.
+   */
+  @BeamBulkQueryJpaTm
+  Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
+
+  /**
+   * A {@link JpaTransactionManager} that uses the Postgres read-only replica if configured (uses
+   * the standard DB otherwise).
+   */
+  @BeamReadOnlyReplicaJpaTm
+  Lazy<JpaTransactionManager> getReadOnlyReplicaJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -34,7 +35,6 @@ import org.apache.beam.sdk.options.Description;
 public interface RegistryPipelineOptions extends GcpOptions {
 
   @Description("The Registry environment.")
-  @Nullable
   RegistryEnvironment getRegistryEnvironment();
 
   void setRegistryEnvironment(RegistryEnvironment environment);
@@ -45,6 +45,12 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setIsolationOverride(TransactionIsolationLevel isolationOverride);
 
+  @Description("The JPA Transaction Manager to use.")
+  @Default.Enum(value = "REGULAR")
+  JpaTransactionManagerType getJpaTransactionManagerType();
+
+  void setJpaTransactionManagerType(JpaTransactionManagerType jpaTransactionManagerType);
+
   @Description("The number of entities to write to the SQL database in one operation.")
   @Default.Integer(20)
   int getSqlWriteBatchSize();

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -20,6 +20,8 @@ import com.google.auto.service.AutoService;
 import com.google.common.flogger.FluentLogger;
 import dagger.Lazy;
 import google.registry.config.RegistryEnvironment;
+import google.registry.config.SystemPropertySetter;
+import google.registry.model.AppEngineEnvironment;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import org.apache.beam.sdk.harness.JvmInitializer;
@@ -35,18 +37,40 @@ import org.apache.beam.sdk.options.PipelineOptions;
 @AutoService(JvmInitializer.class)
 public class RegistryPipelineWorkerInitializer implements JvmInitializer {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  public static final String PROPERTY = "google.registry.beam";
 
   @Override
   public void beforeProcessing(PipelineOptions options) {
     RegistryPipelineOptions registryOptions = options.as(RegistryPipelineOptions.class);
     RegistryEnvironment environment = registryOptions.getRegistryEnvironment();
     if (environment == null || environment.equals(RegistryEnvironment.UNITTEST)) {
-      return;
+      throw new RuntimeException(
+          "A registry environment must be specified in the pipeline options.");
     }
-    logger.atInfo().log("Setting up RegistryEnvironment: %s", environment);
+    logger.atInfo().log("Setting up RegistryEnvironment %s.", environment);
     environment.setup();
-    Lazy<JpaTransactionManager> transactionManagerLazy =
-        toRegistryPipelineComponent(registryOptions).getJpaTransactionManager();
+    RegistryPipelineComponent registryPipelineComponent =
+        toRegistryPipelineComponent(registryOptions);
+    Lazy<JpaTransactionManager> transactionManagerLazy;
+    switch (registryOptions.getJpaTransactionManagerType()) {
+      case BULK_QUERY:
+        transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
+        break;
+      case READ_ONLY_REPLICA:
+        transactionManagerLazy =
+            registryPipelineComponent.getReadOnlyReplicaJpaTransactionManager();
+        break;
+      case REGULAR:
+      default:
+        transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
+    }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
+    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // loads all ofy entities.
+    new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
+        .setEnvironmentForAllThreads();
+    // Set the system property so that we can call IdService.allocateId() without access to
+    // datastore.
+    SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
   }
 }

core/src/main/java/google/registry/beam/common/RegistryQuery.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import google.registry.persistence.transaction.JpaTransactionManager;
 import java.io.Serializable;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -28,6 +29,15 @@ import javax.persistence.criteria.CriteriaQuery;
 
 /** Interface for query instances used by {@link RegistryJpaIO.Read}. */
 public interface RegistryQuery<T> extends Serializable {
+
+  /**
+   * Number of JPA entities to fetch in each batch during a query.
+   *
+   * <p>With Hibernate, for result streaming to work, a query's fetchSize property must be set to a
+   * non-zero value.
+   */
+  int QUERY_FETCH_SIZE = 1000;
+
   Stream<T> stream();
 
   interface CriteriaQuerySupplier<T> extends Supplier<CriteriaQuery<T>>, Serializable {}
@@ -49,6 +59,7 @@ public interface RegistryQuery<T> extends Serializable {
       if (parameters != null) {
         parameters.forEach(query::setParameter);
       }
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
       @SuppressWarnings("unchecked")
       Stream<T> resultStream = query.getResultStream();
       return nativeQuery ? resultStream : resultStream.map(e -> detach(entityManager, e));
@@ -64,11 +75,14 @@ public interface RegistryQuery<T> extends Serializable {
   static <T> RegistryQuery<T> createQuery(
       String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
     return () -> {
-      TypedQuery<T> query = jpaTm().query(jpql, clazz);
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(jpql, clazz);
       if (parameters != null) {
         parameters.forEach(query::setParameter);
       }
-      return query.getResultStream();
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
     };
   }
 
@@ -82,7 +96,13 @@ public interface RegistryQuery<T> extends Serializable {
    * @param <T> Type of each row in the result set.
    */
   static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
-    return () -> jpaTm().query(criteriaQuery.get()).getResultStream();
+    return () -> {
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(criteriaQuery.get());
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
+    };
   }
 
   /**
@@ -108,7 +128,10 @@ public interface RegistryQuery<T> extends Serializable {
       return;
     }
     try {
-      entityManager.detach(object);
+      // TODO(b/193662898): choose detach() or clear() based on the type of transaction.
+      // For context, EntityManager.detach() does not remove all metadata about loaded entities.
+      // See b/193925312 or https://hibernate.atlassian.net/browse/HHH-14735 for details.
+      entityManager.clear();
     } catch (IllegalArgumentException e) {
       // Not an entity. Do nothing.
     }

core/src/main/java/google/registry/beam/comparedb/DatastoreSnapshots.java

@@ -0,0 +1,193 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.initsql.Transforms.createTagForKind;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.Transforms;
+import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.joda.time.DateTime;
+
+/** Utilities for loading Datastore snapshots. */
+@DeleteAfterMigration
+public final class DatastoreSnapshots {
+
+  private DatastoreSnapshots() {}
+
+  /**
+   * Datastore kinds eligible for validation. This set must be consistent with {@link
+   * SqlSnapshots#ALL_SQL_ENTITIES}.
+   */
+  @VisibleForTesting
+  static final ImmutableSet<Class<?>> ALL_DATASTORE_KINDS =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          HistoryEntry.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class);
+
+  /**
+   * Returns the Datastore snapshot right before {@code commitLogToTime} for the user specified
+   * {@code kinds}. The resulting snapshot has all changes that happened before {@code
+   * commitLogToTime}, and none at or after {@code commitLogToTime}.
+   *
+   * <p>If {@code HistoryEntry} is included in {@code kinds}, the result will contain {@code
+   * PCollections} for the child entities, {@code DomainHistory}, {@code ContactHistory}, and {@code
+   * HostHistory}.
+   */
+  static PCollectionTuple loadDatastoreSnapshotByKind(
+      Pipeline pipeline,
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<Class<?>> kinds,
+      Optional<DateTime> compareStartTime) {
+    PCollectionTuple snapshot =
+        pipeline.apply(
+            "Load Datastore snapshot.",
+            Transforms.loadDatastoreSnapshot(
+                exportDir,
+                commitLogDir,
+                commitLogFromTime,
+                commitLogToTime,
+                kinds.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet())));
+
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<?> kind : kinds) {
+      PCollection<VersionedEntity> perKindSnapshot =
+          snapshot.get(createTagForKind(Key.getKind(kind)));
+      if (SqlEntity.class.isAssignableFrom(kind)) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag((Class<? extends SqlEntity>) kind),
+                datastoreEntityToPojo(perKindSnapshot, kind.getSimpleName(), compareStartTime));
+        continue;
+      }
+      Verify.verify(kind == HistoryEntry.class, "Unexpected Non-SqlEntity class: %s", kind);
+      PCollectionTuple historyEntriesByType = splitHistoryEntry(perKindSnapshot, compareStartTime);
+      for (Map.Entry<TupleTag<?>, PCollection<?>> entry :
+          historyEntriesByType.getAll().entrySet()) {
+        perTypeSnapshots = perTypeSnapshots.and(entry.getKey().getId(), entry.getValue());
+      }
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Splits a {@link PCollection} of {@link HistoryEntry HistoryEntries} into three collections of
+   * its child entities by type.
+   */
+  static PCollectionTuple splitHistoryEntry(
+      PCollection<VersionedEntity> historyEntries, Optional<DateTime> compareStartTime) {
+    DateTime nullableStartTime = compareStartTime.orElse(null);
+    return historyEntries.apply(
+        "Split HistoryEntry by Resource Type",
+        ParDo.of(
+                new DoFn<VersionedEntity, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity historyEntry, MultiOutputReceiver out) {
+                    Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(historyEntry))
+                        .filter(e -> isEntityIncludedForComparison(e, nullableStartTime))
+                        .ifPresent(
+                            sqlEntity ->
+                                out.get(createSqlEntityTupleTag(sqlEntity.getClass()))
+                                    .output(sqlEntity));
+                  }
+                })
+            .withOutputTags(
+                createSqlEntityTupleTag(DomainHistory.class),
+                TupleTagList.of(createSqlEntityTupleTag(ContactHistory.class))
+                    .and(createSqlEntityTupleTag(HostHistory.class))));
+  }
+
+  /**
+   * Transforms a {@link PCollection} of {@link VersionedEntity VersionedEntities} to Ofy Java
+   * objects.
+   */
+  static PCollection<SqlEntity> datastoreEntityToPojo(
+      PCollection<VersionedEntity> entities, String desc, Optional<DateTime> compareStartTime) {
+    DateTime nullableStartTime = compareStartTime.orElse(null);
+    return entities.apply(
+        "Datastore Entity to Pojo " + desc,
+        ParDo.of(
+            new DoFn<VersionedEntity, SqlEntity>() {
+              @ProcessElement
+              public void processElement(
+                  @Element VersionedEntity entity, OutputReceiver<SqlEntity> out) {
+                Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(entity))
+                    .filter(e -> isEntityIncludedForComparison(e, nullableStartTime))
+                    .ifPresent(out::output);
+              }
+            }));
+  }
+
+  static boolean isEntityIncludedForComparison(
+      SqlEntity entity, @Nullable DateTime compareStartTime) {
+    if (compareStartTime == null) {
+      return true;
+    }
+    if (entity instanceof HistoryEntry) {
+      return compareStartTime.isBefore(((HistoryEntry) entity).getModificationTime());
+    }
+    if (entity instanceof EppResource) {
+      return compareStartTime.isBefore(((EppResource) entity).getUpdateTimestamp().getTimestamp());
+    }
+    return true;
+  }
+}

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -0,0 +1,154 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import dagger.Component;
+import google.registry.config.CloudTasksUtilsModule;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.util.Clock;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.joda.time.DateTime;
+import org.joda.time.Instant;
+import org.joda.time.Interval;
+
+/** Finds the necessary information for loading the most recent Datastore snapshot. */
+@DeleteAfterMigration
+public class LatestDatastoreSnapshotFinder {
+  private final String projectId;
+  private final GcsUtils gcsUtils;
+  private final Clock clock;
+
+  @Inject
+  LatestDatastoreSnapshotFinder(
+      @Config("projectId") String projectId, GcsUtils gcsUtils, Clock clock) {
+    this.projectId = projectId;
+    this.gcsUtils = gcsUtils;
+    this.clock = clock;
+  }
+
+  /**
+   * Finds information of the most recent Datastore snapshot that ends strictly before {@code
+   * exportEndTimeUpperBound}, including the GCS folder of the exported data files and the start and
+   * stop times of the export. The folder of the CommitLogs is also included in the return.
+   */
+  public DatastoreSnapshotInfo getSnapshotInfo(Instant exportEndTimeUpperBound) {
+    String bucketName = RegistryConfig.getDatastoreBackupsBucket().substring("gs://".length());
+    /**
+     * Find the bucket-relative path to the overall metadata file of the last Datastore export.
+     * Since Datastore export is saved daily, we may need to look back to yesterday. If found, the
+     * return value is like
+     * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
+     */
+    Optional<String> metaFilePathOptional =
+        findNewestExportMetadataFileBeforeTime(bucketName, exportEndTimeUpperBound, 2);
+    if (!metaFilePathOptional.isPresent()) {
+      throw new NoSuchElementException("No exports found over the past 2 days.");
+    }
+    String metaFilePath = metaFilePathOptional.get();
+    String metaFileFolder = metaFilePath.substring(0, metaFilePath.indexOf('/'));
+    Instant exportStartTime = Instant.parse(metaFileFolder.replace('_', '.') + 'Z');
+    BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath));
+    Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+    return DatastoreSnapshotInfo.create(
+        String.format("gs://%s/%s", bucketName, metaFileFolder),
+        getCommitLogDir(),
+        new Interval(exportStartTime, exportEndTime));
+  }
+
+  public String getCommitLogDir() {
+    return "gs://" + projectId + "-commits";
+  }
+
+  /**
+   * Finds the latest Datastore export that ends strictly before {@code endTimeUpperBound} and
+   * returns the bucket-relative path of the overall export metadata file, in the given bucket. The
+   * search goes back for up to {@code lookBackDays} days in time, including today.
+   *
+   * <p>The overall export metadata file is the last file created during a Datastore export. All
+   * data has been exported by the creation time of this file. The name of this file, like that of
+   * all files in the same export, begins with the timestamp when the export starts.
+   *
+   * <p>An example return value: {@code
+   * 2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata}.
+   */
+  private Optional<String> findNewestExportMetadataFileBeforeTime(
+      String bucketName, Instant endTimeUpperBound, int lookBackDays) {
+    DateTime today = clock.nowUtc();
+    for (int day = 0; day < lookBackDays; day++) {
+      String dateString = today.minusDays(day).toString("yyyy-MM-dd");
+      try {
+        Optional<String> metaFilePath =
+            gcsUtils.listFolderObjects(bucketName, dateString).stream()
+                .filter(s -> s.endsWith("overall_export_metadata"))
+                .map(s -> dateString + s)
+                .sorted(Comparator.<String>naturalOrder().reversed())
+                .findFirst();
+        if (metaFilePath.isPresent()) {
+          BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath.get()));
+          Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+          if (exportEndTime.isBefore(endTimeUpperBound)) {
+            return metaFilePath;
+          }
+        }
+      } catch (IOException ioe) {
+        throw new RuntimeException(ioe);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /** Holds information about a Datastore snapshot. */
+  @AutoValue
+  abstract static class DatastoreSnapshotInfo {
+    abstract String exportDir();
+
+    abstract String commitLogDir();
+
+    abstract Interval exportInterval();
+
+    static DatastoreSnapshotInfo create(
+        String exportDir, String commitLogDir, Interval exportOperationInterval) {
+      return new AutoValue_LatestDatastoreSnapshotFinder_DatastoreSnapshotInfo(
+          exportDir, commitLogDir, exportOperationInterval);
+    }
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
+  interface LatestDatastoreSnapshotFinderFinderComponent {
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotInfoFinder();
+  }
+}

core/src/main/java/google/registry/beam/comparedb/SqlSnapshots.java

@@ -0,0 +1,539 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.comparedb.ValidateSqlUtils.getMedianIdForHistoryTable;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Strings;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Streams;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryJpaIO.Read;
+import google.registry.model.EppResource;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.model.bulkquery.DomainBaseLite;
+import google.registry.model.bulkquery.DomainHistoryHost;
+import google.registry.model.bulkquery.DomainHistoryLite;
+import google.registry.model.bulkquery.DomainHost;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import google.registry.util.DateTimeUtils;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.persistence.Entity;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.joda.time.DateTime;
+
+/**
+ * Utilities for loading SQL snapshots.
+ *
+ * <p>For {@link DomainBase} and {@link DomainHistory}, this class assumes the presence of the
+ * {@link google.registry.persistence.PersistenceModule.JpaTransactionManagerType#BULK_QUERY
+ * bulk-query-capable JpaTransactionManager}, and takes advantage of it for higher throughput.
+ *
+ * <p>For now this class is meant for use during the database migration period only. Therefore, it
+ * contains optimizations specifically for the production database at the current size, e.g.,
+ * parallel queries for select tables.
+ */
+@DeleteAfterMigration
+public final class SqlSnapshots {
+
+  private SqlSnapshots() {}
+
+  /**
+   * SQL entity types that are eligible for validation. This set must be consistent with {@link
+   * DatastoreSnapshots#ALL_DATASTORE_KINDS}.
+   */
+  static final ImmutableSet<Class<? extends SqlEntity>> ALL_SQL_ENTITIES =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class,
+          ContactHistory.class,
+          HostHistory.class,
+          DomainHistory.class);
+
+  /**
+   * Loads a SQL snapshot for the given {@code sqlEntityTypes}.
+   *
+   * <p>If {@code snapshotId} is present, all queries use the specified database snapshot,
+   * guaranteeing a consistent result.
+   */
+  public static PCollectionTuple loadCloudSqlSnapshotByType(
+      Pipeline pipeline,
+      ImmutableSet<Class<? extends SqlEntity>> sqlEntityTypes,
+      Optional<String> snapshotId,
+      Optional<DateTime> compareStartTime) {
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<? extends SqlEntity> clazz : sqlEntityTypes) {
+      if (clazz == DomainBase.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainBase.class),
+                loadAndAssembleDomainBase(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == DomainHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainHistory.class),
+                loadAndAssembleDomainHistory(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == ContactHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(ContactHistory.class),
+                loadContactHistory(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == HostHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(HostHistory.class),
+                loadHostHistory(
+                    pipeline, snapshotId, compareStartTime.orElse(DateTimeUtils.START_OF_TIME)));
+        continue;
+      }
+      if (EppResource.class.isAssignableFrom(clazz) && compareStartTime.isPresent()) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(clazz),
+                pipeline.apply(
+                    "SQL Load " + clazz.getSimpleName(),
+                    buildEppResourceQueryWithTimeFilter(
+                            clazz, SqlEntity.class, snapshotId, compareStartTime.get())
+                        .withSnapshot(snapshotId.orElse(null))));
+        continue;
+      }
+      perTypeSnapshots =
+          perTypeSnapshots.and(
+              createSqlEntityTupleTag(clazz),
+              pipeline.apply(
+                  "SQL Load " + clazz.getSimpleName(),
+                  RegistryJpaIO.read(
+                          () -> CriteriaQueryBuilder.create(clazz).build(), SqlEntity.class::cast)
+                      .withSnapshot(snapshotId.orElse(null))));
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Bulk-loads parts of {@link DomainBase} and assembles them in the pipeline.
+   *
+   * @see BulkQueryEntities
+   */
+  public static PCollection<SqlEntity> loadAndAssembleDomainBase(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PCollection<KV<String, Serializable>> baseObjects =
+        readAllAndAssignKey(
+            pipeline,
+            DomainBaseLite.class,
+            DomainBaseLite::getRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriod.class,
+            GracePeriod::getDomainRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DelegationSignerData.class,
+            DelegationSignerData::getDomainRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline, DomainHost.class, DomainHost::getDomainRepoId, snapshotId, compareStartTime);
+
+    DateTime nullableCompareStartTime = compareStartTime.orElse(null);
+    return PCollectionList.of(
+            ImmutableList.of(baseObjects, gracePeriods, delegationSigners, domainHosts))
+        .apply("SQL Merge DomainBase parts", Flatten.pCollections())
+        .apply("Group by Domain Parts by RepoId", GroupByKey.create())
+        .apply(
+            "Assemble DomainBase",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainBaseLite> baseObjects =
+                        partsByType.getAllOf(DomainBaseLite.class);
+                    if (nullableCompareStartTime != null) {
+                      Verify.verify(
+                          baseObjects.size() <= 1,
+                          "Found duplicate DomainBaseLite object per repoId: " + kv.getKey());
+                      if (baseObjects.isEmpty()) {
+                        return;
+                      }
+                    }
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainBaseLite object per repoId: " + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainBase(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(GracePeriod.class),
+                            partsByType.getAllOf(DelegationSignerData.class),
+                            partsByType.getAllOf(DomainHost.class).stream()
+                                .map(DomainHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet())));
+                  }
+                }));
+  }
+
+  /**
+   * Loads all {@link ContactHistory} entities from the database.
+   *
+   * <p>This method uses two queries to load data in parallel. This is a performance optimization
+   * specifically for the production database.
+   */
+  static PCollection<SqlEntity> loadContactHistory(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PartitionedQuery partitionedQuery =
+        buildPartitonedHistoryQuery(ContactHistory.class, compareStartTime);
+    PCollection<SqlEntity> part1 =
+        pipeline.apply(
+            "SQL Load ContactHistory first half",
+            RegistryJpaIO.read(
+                    partitionedQuery.firstHalfQuery(),
+                    partitionedQuery.parameters(),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    PCollection<SqlEntity> part2 =
+        pipeline.apply(
+            "SQL Load ContactHistory second half",
+            RegistryJpaIO.read(
+                    partitionedQuery.secondHalfQuery(),
+                    partitionedQuery.parameters(),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    return PCollectionList.of(part1)
+        .and(part2)
+        .apply("Combine ContactHistory parts", Flatten.pCollections());
+  }
+
+  /** Loads all {@link HostHistory} entities from the database. */
+  static PCollection<SqlEntity> loadHostHistory(
+      Pipeline pipeline, Optional<String> snapshotId, DateTime compareStartTime) {
+    return pipeline.apply(
+        "SQL Load HostHistory",
+        RegistryJpaIO.read(
+                "select c from HostHistory c where :compareStartTime  < modificationTime",
+                ImmutableMap.of("compareStartTime", compareStartTime),
+                false,
+                SqlEntity.class::cast)
+            .withSnapshot(snapshotId.orElse(null)));
+  }
+
+  /**
+   * Bulk-loads all parts of {@link DomainHistory} and assembles them in the pipeline.
+   *
+   * <p>This method uses two queries to load {@link DomainBaseLite} in parallel. This is a
+   * performance optimization specifically for the production database.
+   *
+   * @see BulkQueryEntities
+   */
+  static PCollection<SqlEntity> loadAndAssembleDomainHistory(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PartitionedQuery partitionedQuery =
+        buildPartitonedHistoryQuery(DomainHistoryLite.class, compareStartTime);
+    PCollection<KV<String, Serializable>> baseObjectsPart1 =
+        queryAndAssignKey(
+            pipeline,
+            "first half",
+            partitionedQuery.firstHalfQuery(),
+            partitionedQuery.parameters(),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> baseObjectsPart2 =
+        queryAndAssignKey(
+            pipeline,
+            "second half",
+            partitionedQuery.secondHalfQuery(),
+            partitionedQuery.parameters(),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriodHistory.class,
+            compose(GracePeriodHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DomainDsDataHistory.class,
+            compose(DomainDsDataHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline,
+            DomainHistoryHost.class,
+            compose(DomainHistoryHost::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> transactionRecords =
+        readAllAndAssignKey(
+            pipeline,
+            DomainTransactionRecord.class,
+            compose(DomainTransactionRecord::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+
+    DateTime nullableCompareStartTime = compareStartTime.orElse(null);
+    return PCollectionList.of(
+            ImmutableList.of(
+                baseObjectsPart1,
+                baseObjectsPart2,
+                gracePeriods,
+                delegationSigners,
+                domainHosts,
+                transactionRecords))
+        .apply("Merge DomainHistory parts", Flatten.pCollections())
+        .apply("Group by DomainHistory Parts by DomainHistoryId string", GroupByKey.create())
+        .apply(
+            "Assemble DomainHistory",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainHistoryLite> baseObjects =
+                        partsByType.getAllOf(DomainHistoryLite.class);
+                    if (nullableCompareStartTime != null) {
+                      Verify.verify(
+                          baseObjects.size() <= 1,
+                          "Found duplicate DomainHistoryLite object per domainHistoryId: "
+                              + kv.getKey());
+                      if (baseObjects.isEmpty()) {
+                        return;
+                      }
+                    }
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainHistoryLite object per domainHistoryId: "
+                            + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainHistory(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(DomainDsDataHistory.class),
+                            partsByType.getAllOf(DomainHistoryHost.class).stream()
+                                .map(DomainHistoryHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet()),
+                            partsByType.getAllOf(GracePeriodHistory.class),
+                            partsByType.getAllOf(DomainTransactionRecord.class)));
+                  }
+                }));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> readAllAndAssignKey(
+      Pipeline pipeline,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId,
+      Optional<DateTime> compareStartTime) {
+    Read<R, R> queryObject;
+    if (compareStartTime.isPresent() && EppResource.class.isAssignableFrom(type)) {
+      queryObject =
+          buildEppResourceQueryWithTimeFilter(type, type, snapshotId, compareStartTime.get());
+    } else {
+      queryObject =
+          RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(type).build())
+              .withSnapshot(snapshotId.orElse(null));
+    }
+    return pipeline
+        .apply("SQL Load " + type.getSimpleName(), queryObject)
+        .apply(
+            "Assign Key to " + type.getSimpleName(),
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> queryAndAssignKey(
+      Pipeline pipeline,
+      String diffrentiator,
+      String jplQuery,
+      ImmutableMap<String, Object> queryParameters,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName() + " " + diffrentiator,
+            RegistryJpaIO.read(jplQuery, queryParameters, false, type::cast)
+                .withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName() + " " + diffrentiator,
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  // TODO(b/205988530): don't use beam serializablefunction, make one that extends Java's Function.
+  private static <R, I, T> SerializableFunction<R, T> compose(
+      SerializableFunction<R, I> f1, SerializableFunction<I, T> f2) {
+    return r -> f2.apply(f1.apply(r));
+  }
+
+  static <R, T> Read<R, T> buildEppResourceQueryWithTimeFilter(
+      Class<R> entityType,
+      Class<T> castOutputAsType,
+      Optional<String> snapshotId,
+      DateTime compareStartTime) {
+    String tableName = getJpaEntityName(entityType);
+    String jpql =
+        String.format("select c from %s c where :compareStartTime < updateTimestamp", tableName);
+    return RegistryJpaIO.read(
+            jpql,
+            ImmutableMap.of("compareStartTime", UpdateAutoTimestamp.create(compareStartTime)),
+            false,
+            (R x) -> castOutputAsType.cast(x))
+        .withSnapshot(snapshotId.orElse(null));
+  }
+
+  static PartitionedQuery buildPartitonedHistoryQuery(
+      Class<?> entityType, Optional<DateTime> compareStartTime) {
+    String tableName = getJpaEntityName(entityType);
+    Verify.verify(
+        !Strings.isNullOrEmpty(tableName), "Invalid entity type %s", entityType.getSimpleName());
+    long medianId =
+        getMedianIdForHistoryTable(tableName)
+            .orElseThrow(() -> new IllegalStateException("Not a valid database: no " + tableName));
+    String firstHalfQuery = String.format("select c from %s c where id <= :historyId", tableName);
+    String secondHalfQuery = String.format("select c from %s c where id > :historyId", tableName);
+    if (compareStartTime.isPresent()) {
+      String timeFilter = "  and :compareStartTime  < modificationTime";
+      firstHalfQuery += timeFilter;
+      secondHalfQuery += timeFilter;
+      return PartitionedQuery.createPartitionedQuery(
+          firstHalfQuery,
+          secondHalfQuery,
+          ImmutableMap.of("historyId", medianId, "compareStartTime", compareStartTime.get()));
+    } else {
+      return PartitionedQuery.createPartitionedQuery(
+          firstHalfQuery, secondHalfQuery, ImmutableMap.of("historyId", medianId));
+    }
+  }
+
+  private static String getJpaEntityName(Class entityType) {
+    Entity entityAnnotation = (Entity) entityType.getAnnotation(Entity.class);
+    checkState(
+        entityAnnotation != null, "Unexpected non-entity type %s", entityType.getSimpleName());
+    return Strings.isNullOrEmpty(entityAnnotation.name())
+        ? entityType.getSimpleName()
+        : entityAnnotation.name();
+  }
+
+  /** Contains two queries that partition the target table in two. */
+  @AutoValue
+  abstract static class PartitionedQuery {
+    abstract String firstHalfQuery();
+
+    abstract String secondHalfQuery();
+
+    abstract ImmutableMap<String, Object> parameters();
+
+    public static PartitionedQuery createPartitionedQuery(
+        String firstHalfQuery, String secondHalfQuery, ImmutableMap<String, Object> parameters) {
+      return new AutoValue_SqlSnapshots_PartitionedQuery(
+          firstHalfQuery, secondHalfQuery, parameters);
+    }
+  }
+
+  /** Container that receives mixed-typed data and groups them by {@link Class}. */
+  static class TypedClassifier {
+    private final ImmutableSetMultimap<Class<?>, Object> classifiedEntities;
+
+    TypedClassifier(Iterable<Serializable> inputs) {
+      this.classifiedEntities =
+          Streams.stream(inputs)
+              .collect(ImmutableSetMultimap.toImmutableSetMultimap(Object::getClass, x -> x));
+    }
+
+    <T> ImmutableSet<T> getAllOf(Class<T> clazz) {
+      return classifiedEntities.get(clazz).stream()
+          .map(clazz::cast)
+          .collect(ImmutableSet.toImmutableSet());
+    }
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipeline.java

@@ -0,0 +1,83 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.joda.time.DateTime;
+
+/**
+ * Validates the asynchronous data replication process from Cloud SQL (primary) to Datastore
+ * (secondary).
+ *
+ * <p>This pipeline simply compares the snapshots provided by an invoker, which is responsible for
+ * obtaining two consistent snapshots for the same point of time.
+ */
+// TODO(weiminyu): Implement the invoker action in a followup PR.
+@DeleteAfterMigration
+public class ValidateDatastorePipeline {
+
+  private final ValidateDatastorePipelineOptions options;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+
+  public ValidateDatastorePipeline(
+      ValidateDatastorePipelineOptions options,
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
+    this.options = options;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+  }
+
+  void run(Pipeline pipeline) {
+    DateTime latestCommitLogTime = DateTime.parse(options.getLatestCommitLogTimestamp());
+    DatastoreSnapshotInfo mostRecentExport =
+        datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
+
+    ValidateSqlPipeline.setupPipeline(
+        pipeline,
+        Optional.ofNullable(options.getSqlSnapshotId()),
+        mostRecentExport,
+        latestCommitLogTime,
+        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
+
+    pipeline.run();
+  }
+
+  public static void main(String[] args) {
+    ValidateDatastorePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args)
+            .withValidation()
+            .as(ValidateDatastorePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+    new ValidateDatastorePipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipelineOptions.java

@@ -0,0 +1,39 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.model.annotations.DeleteAfterMigration;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Validation;
+
+/** BEAM pipeline options for {@link ValidateDatastorePipelineOptions}. */
+@DeleteAfterMigration
+public interface ValidateDatastorePipelineOptions extends ValidateSqlPipelineOptions {
+
+  @Description(
+      "The id of the SQL snapshot to be compared with Datastore. "
+          + "If null, the current state of the SQL database is used.")
+  @Nullable
+  String getSqlSnapshotId();
+
+  void setSqlSnapshotId(String snapshotId);
+
+  @Description("The latest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getLatestCommitLogTimestamp();
+
+  void setLatestCommitLogTimestamp(String commitLogEndTimestamp);
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -0,0 +1,264 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.model.server.Lock;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import google.registry.util.RequestStatusChecker;
+import google.registry.util.SystemClock;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
+ * (secondary storage).
+ */
+@DeleteAfterMigration
+public class ValidateSqlPipeline {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final Duration COMMITLOG_START_TIME_MARGIN = Duration.standardMinutes(10);
+
+  /**
+   * Name of the lock used by the commitlog replay process.
+   *
+   * <p>See {@link google.registry.backup.ReplayCommitLogsToSqlAction} for more information.
+   */
+  private static final String COMMITLOG_REPLAY_LOCK_NAME = "ReplayCommitLogsToSqlAction";
+
+  private static final Duration REPLAY_LOCK_LEASE_LENGTH = Duration.standardHours(1);
+  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_TIMEOUT =
+      java.time.Duration.ofMinutes(6);
+  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_DELAY =
+      java.time.Duration.ofSeconds(30);
+
+  private final ValidateSqlPipelineOptions options;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+
+  public ValidateSqlPipeline(
+      ValidateSqlPipelineOptions options, LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
+    this.options = options;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    Optional<Lock> lock = acquireCommitLogReplayLock();
+    if (lock.isPresent()) {
+      logger.atInfo().log("Acquired CommitLog Replay lock.");
+    } else {
+      throw new RuntimeException("Failed to acquire CommitLog Replay lock.");
+    }
+
+    try {
+      DateTime latestCommitLogTime =
+          TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+      DatastoreSnapshotInfo mostRecentExport =
+          datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
+      Preconditions.checkState(
+          latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
+          "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
+      try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+        // Eagerly release the commitlog replay lock so that replay can resume.
+        lock.ifPresent(Lock::releaseSql);
+        lock = Optional.empty();
+
+        logger.atInfo().log(
+            "Starting comparison with export at %s and latestCommitLogTime at %s",
+            mostRecentExport.exportDir(), latestCommitLogTime);
+
+        setupPipeline(
+            pipeline,
+            Optional.of(databaseSnapshot.getSnapshotId()),
+            mostRecentExport,
+            latestCommitLogTime,
+            Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
+        State state = pipeline.run().waitUntilFinish();
+        if (!State.DONE.equals(state)) {
+          throw new IllegalStateException("Unexpected pipeline state: " + state);
+        }
+      }
+    } finally {
+      lock.ifPresent(Lock::releaseSql);
+    }
+  }
+
+  static void setupPipeline(
+      Pipeline pipeline,
+      Optional<String> sqlSnapshotId,
+      DatastoreSnapshotInfo mostRecentExport,
+      DateTime latestCommitLogTime,
+      Optional<DateTime> compareStartTime) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minus(COMMITLOG_START_TIME_MARGIN),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS,
+            compareStartTime);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId, compareStartTime);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      PCollectionList.of(datastoreSnapshot.get(tag))
+          .and(cloudSqlSnapshot.get(tag))
+          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
+          .apply(
+              "Assign primary key to merged " + clazz.getSimpleName(),
+              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
+          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  private static Optional<Lock> acquireCommitLogReplayLock() {
+    Stopwatch stopwatch = Stopwatch.createStarted();
+    while (stopwatch.elapsed().minus(REPLAY_LOCK_ACQUIRE_TIMEOUT).isNegative()) {
+      Optional<Lock> lock = tryAcquireCommitLogReplayLock();
+      if (lock.isPresent()) {
+        return lock;
+      }
+      logger.atInfo().log("Failed to acquired CommitLog Replay lock. Will retry...");
+      try {
+        Thread.sleep(REPLAY_LOCK_ACQUIRE_DELAY.toMillis());
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new RuntimeException("Interrupted.");
+      }
+    }
+    return Optional.empty();
+  }
+
+  private static Optional<Lock> tryAcquireCommitLogReplayLock() {
+    return Lock.acquireSql(
+        COMMITLOG_REPLAY_LOCK_NAME,
+        null,
+        REPLAY_LOCK_LEASE_LENGTH,
+        getLockingRequestStatusChecker(),
+        false);
+  }
+
+  /**
+   * Returns a fake implementation of {@link RequestStatusChecker} that is required for lock
+   * acquisition. The default implementation is AppEngine-specific and is unusable on GCE.
+   */
+  private static RequestStatusChecker getLockingRequestStatusChecker() {
+    return new RequestStatusChecker() {
+      @Override
+      public String getLogId() {
+        return "ValidateSqlPipeline";
+      }
+
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+          DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+              .datastoreSnapshotInfoFinder();
+
+      @Override
+      public boolean isRunning(String requestLogId) {
+        return true;
+      }
+    };
+  }
+
+  public static void main(String[] args) {
+    ValidateSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    MigrationState state =
+        DatabaseMigrationStateSchedule.getValueAtTime(new SystemClock().nowUtc());
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      throw new IllegalStateException("This pipeline is not designed for migration phase " + state);
+    }
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+
+    new ValidateSqlPipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -0,0 +1,34 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.model.annotations.DeleteAfterMigration;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.options.Description;
+
+/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
+@DeleteAfterMigration
+public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {
+
+  @Description(
+      "For history entries and EPP resources, only those modified strictly after this time are "
+          + "included in comparison. Value is in ISO8601 format. "
+          + "Other entity types are not affected.")
+  @Nullable
+  String getComparisonStartTimestamp();
+
+  void setComparisonStartTimestamp(String comparisonStartTimestamp);
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -0,0 +1,387 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.initsql.Transforms;
+import google.registry.config.RegistryEnvironment;
+import google.registry.model.BackupGroupRoot;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.contact.ContactBase;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.eppcommon.AuthInfo;
+import google.registry.model.host.HostHistory;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import java.lang.reflect.Field;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Helpers for use by {@link ValidateSqlPipeline}. */
+@DeleteAfterMigration
+final class ValidateSqlUtils {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ValidateSqlUtils() {}
+
+  private static final ImmutableSet<String> PROBER_CELLS = ImmutableSet.of("IQ", "LG", "TL");
+  private static final ImmutableSet<String> PROBER_TYPES =
+      ImmutableSet.of("ANYT", "ANYTES", "CANARY");
+
+  /**
+   * Query template for finding the median value of the {@code history_revision_id} column in one of
+   * the History tables.
+   *
+   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * history tables. Although the {@code repo_id} column is the leading column in the primary keys
+   * of these tables, in practice and with production data, division by {@code history_revision_id}
+   * works slightly faster for unknown reasons.
+   */
+  private static final String MEDIAN_ID_QUERY_TEMPLATE =
+      "SELECT history_revision_id FROM (                                                        "
+          + "  SELECT"
+          + "    ROW_NUMBER() OVER (ORDER BY history_revision_id ASC) AS rownumber,"
+          + "    history_revision_id"
+          + "  FROM \"%TABLE%\""
+          + ") AS foo\n"
+          + "WHERE rownumber in (select count(*) / 2 + 1 from \"%TABLE%\")";
+
+  static Optional<Long> getMedianIdForHistoryTable(String tableName) {
+    Preconditions.checkArgument(
+        tableName.endsWith("History"), "Table must be one of the History tables.");
+    String sqlText = MEDIAN_ID_QUERY_TEMPLATE.replace("%TABLE%", tableName);
+    List results =
+        jpaTm()
+            .transact(() -> jpaTm().getEntityManager().createNativeQuery(sqlText).getResultList());
+    verify(results.size() < 2, "MidPoint query should have at most one result.");
+    if (results.isEmpty()) {
+      return Optional.empty();
+    }
+    return Optional.of(((BigInteger) results.get(0)).longValue());
+  }
+
+  static TupleTag<SqlEntity> createSqlEntityTupleTag(Class<? extends SqlEntity> actualType) {
+    return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
+  }
+
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+    private final HashMap<String, Counter> totalCounters = new HashMap<>();
+    private final HashMap<String, Counter> missingCounters = new HashMap<>();
+    private final HashMap<String, Counter> unequalCounters = new HashMap<>();
+    private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
+
+    private volatile boolean logPrinted = false;
+
+    private String getCounterKey(Class<?> clazz) {
+      return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
+    }
+
+    private synchronized void ensureCounterExists(String counterKey) {
+      if (totalCounters.containsKey(counterKey)) {
+        return;
+      }
+      totalCounters.put(counterKey, Metrics.counter("CompareDB", "Total Compared: " + counterKey));
+      missingCounters.put(
+          counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
+      unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
+      badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
+    }
+
+    /**
+     * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
+     * This will be removed when we start exporting such entities to GCS.
+     */
+    void logDiff(String key, Object entry0, Object entry1) {
+      if (logPrinted) {
+        return;
+      }
+      logPrinted = true;
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      StringBuilder sb = new StringBuilder();
+      fields0.forEach(
+          (field, value) -> {
+            if (fields1.containsKey(field)) {
+              if (!Objects.equals(value, fields1.get(field))) {
+                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
+              }
+            } else {
+              sb.append(field + "Not found in entity 2\n");
+            }
+          });
+      fields1.forEach(
+          (field, value) -> {
+            if (!fields0.containsKey(field)) {
+              sb.append(field + "Not found in entity 1\n");
+            }
+          });
+      logger.atWarning().log(key + "  " + sb.toString());
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+      ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
+
+      verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
+      verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
+
+      String counterKey = getCounterKey(entities.get(0).getClass());
+      ensureCounterExists(counterKey);
+      totalCounters.get(counterKey).inc();
+
+      if (entities.size() == 1) {
+        if (isSpecialCaseProberEntity(entities.get(0))) {
+          return;
+        }
+        missingCounters.get(counterKey).inc();
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
+        }
+        return;
+      }
+      SqlEntity entity0;
+      SqlEntity entity1;
+
+      try {
+        entity0 = normalizeEntity(entities.get(0));
+        entity1 = normalizeEntity(entities.get(1));
+      } catch (Exception e) {
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          badEntityCounters.get(counterKey).inc();
+        }
+        return;
+      }
+
+      if (!Objects.equals(entity0, entity1)) {
+        unequalCounters.get(counterKey).inc();
+        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+      }
+    }
+  }
+
+  static SqlEntity normalizeEntity(SqlEntity sqlEntity) {
+    if (sqlEntity instanceof EppResource) {
+      return normalizeEppResource(sqlEntity);
+    }
+    if (sqlEntity instanceof HistoryEntry) {
+      return (SqlEntity) normalizeHistoryEntry((HistoryEntry) sqlEntity);
+    }
+    return sqlEntity;
+  }
+
+  /**
+   * Normalizes an {@link EppResource} instance for comparison.
+   *
+   * <p>This method may modify the input object using reflection instead of making a copy with
+   * {@code eppResource.asBuilder().build()}, because when {@code eppResource} is a {@link
+   * google.registry.model.domain.DomainBase}, the {@code build} method accesses the Database, which
+   * we want to avoid.
+   */
+  static SqlEntity normalizeEppResource(SqlEntity eppResource) {
+    try {
+      if (isSpecialCaseProberEntity(eppResource)) {
+        // Clearing some timestamps. See isSpecialCaseProberEntity() for reasons.
+        Field lastUpdateTime = BackupGroupRoot.class.getDeclaredField("updateTimestamp");
+        lastUpdateTime.setAccessible(true);
+        lastUpdateTime.set(eppResource, null);
+        Field deletionTime = EppResource.class.getDeclaredField("deletionTime");
+        deletionTime.setAccessible(true);
+        deletionTime.set(eppResource, null);
+      }
+      Field authField =
+          eppResource instanceof DomainContent
+              ? DomainContent.class.getDeclaredField("authInfo")
+              : eppResource instanceof ContactBase
+                  ? ContactBase.class.getDeclaredField("authInfo")
+                  : null;
+      if (authField != null) {
+        authField.setAccessible(true);
+        AuthInfo authInfo = (AuthInfo) authField.get(eppResource);
+        // When AuthInfo is missing, the authInfo field is null if the object is loaded from
+        // Datastore, or a PasswordAuth with null properties if loaded from SQL. In the second case
+        // we set the authInfo field to null.
+        if (authInfo != null
+            && authInfo.getPw() != null
+            && authInfo.getPw().getRepoId() == null
+            && authInfo.getPw().getValue() == null) {
+          authField.set(eppResource, null);
+        }
+      }
+
+      Field field = EppResource.class.getDeclaredField("revisions");
+      field.setAccessible(true);
+      field.set(eppResource, null);
+      return eppResource;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Normalizes a {@link HistoryEntry} for comparison.
+   *
+   * <p>This method modifies the input using reflection because relevant builder methods performs
+   * unwanted checks and changes.
+   */
+  static HistoryEntry normalizeHistoryEntry(HistoryEntry historyEntry) {
+    // History objects from Datastore do not have details of their EppResource objects
+    // (domainContent, contactBase, hostBase).
+    try {
+      if (historyEntry instanceof DomainHistory) {
+        Field domainContent = DomainHistory.class.getDeclaredField("domainContent");
+        domainContent.setAccessible(true);
+        domainContent.set(historyEntry, null);
+        // Convert empty domainTransactionRecords to null for comparison.
+        Field domainTransactionRecords =
+            HistoryEntry.class.getDeclaredField("domainTransactionRecords");
+        domainTransactionRecords.setAccessible(true);
+        Set<?> domainTransactionRecordsValue = (Set<?>) domainTransactionRecords.get(historyEntry);
+        if (domainTransactionRecordsValue != null && domainTransactionRecordsValue.isEmpty()) {
+          domainTransactionRecords.set(historyEntry, null);
+        }
+        // DomainHistory in Datastore does not have the following properties either:
+        Field nsHosts = DomainHistory.class.getDeclaredField("nsHosts");
+        nsHosts.setAccessible(true);
+        nsHosts.set(historyEntry, null);
+        Field dsDataHistories = DomainHistory.class.getDeclaredField("dsDataHistories");
+        dsDataHistories.setAccessible(true);
+        dsDataHistories.set(historyEntry, null);
+        Field gracePeriodHistories = DomainHistory.class.getDeclaredField("gracePeriodHistories");
+        gracePeriodHistories.setAccessible(true);
+        gracePeriodHistories.set(historyEntry, null);
+      } else if (historyEntry instanceof ContactHistory) {
+        Field contactBase = ContactHistory.class.getDeclaredField("contactBase");
+        contactBase.setAccessible(true);
+        contactBase.set(historyEntry, null);
+      } else if (historyEntry instanceof HostHistory) {
+        Field hostBase = HostHistory.class.getDeclaredField("hostBase");
+        hostBase.setAccessible(true);
+        hostBase.set(historyEntry, null);
+      }
+      return historyEntry;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Returns {@code true} if {@code entity} is created by the prober and needs special treatment.
+   *
+   * <p>{@link EppResource} entities created by the prober are deleted by a cron job that bypasses
+   * the CommitLog mechanism. As a result, their deletions are not propagated to SQL, creating two
+   * types of mismatches: an entity exists in both databases but differs in lastUpdateTime and
+   * deletionTime; an entity only exists in the SQL database.
+   *
+   * <p>In production, there are few placeholder {@link Registrar registrars} that do not exist in
+   * Datastore. They were manually created to in SQL to solve a one-time problem (see b/187946868
+   * for details). They can be ignored in the database comparison.
+   */
+  static boolean isSpecialCaseProberEntity(Object entity) {
+    if (entity instanceof EppResource) {
+      EppResource host = (EppResource) entity;
+      if (host.getPersistedCurrentSponsorRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (entity instanceof HistoryEntry) {
+      HistoryEntry historyEntry = (HistoryEntry) entity;
+      if (historyEntry.getRegistrarId().startsWith("prober-")) {
+        // Not all prober entities have "prober-" as registrar prefix.
+        return true;
+      }
+      if (Objects.equals(historyEntry.getReason(), "Deletion of prober data")) {
+        // Soft-delete event in Datastore that is not propagated to SQL.
+        return true;
+      }
+    }
+    if (entity instanceof DomainHistory) {
+      DomainHistory domainHistory = (DomainHistory) entity;
+      if (domainHistory.getDomainContent().isPresent()
+          && domainHistory.getDomainContent().get().getDomainName().startsWith("prober-")) {
+        // Asynchronously replicated event in SQL.
+        return true;
+      }
+      if (domainHistory.getDomainRepoId() != null) {
+        // Some synthetic events only have domainRepoId.
+        String repoId = domainHistory.getDomainRepoId();
+        if (Transforms.IGNORED_DOMAINS.contains(repoId)) {
+          return true;
+        }
+        String suffix = repoId.substring(repoId.indexOf('-') + 1);
+        String cell = suffix.substring(0, 2);
+        suffix = suffix.substring(2);
+        if (PROBER_CELLS.contains(cell) && PROBER_TYPES.contains(suffix)) {
+          return true;
+        }
+      }
+    }
+    if (entity instanceof ContactHistory) {
+      if (Transforms.IGNORED_CONTACTS.contains(((ContactHistory) entity).getContactRepoId())) {
+        return true;
+      }
+    }
+    if (entity instanceof HostHistory) {
+      if (Transforms.IGNORED_HOSTS.contains(((HostHistory) entity).getHostRepoId())) {
+        return true;
+      }
+    }
+    if (entity instanceof BillingEvent) {
+      BillingEvent event = (BillingEvent) entity;
+      if (event.getRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (entity instanceof PollMessage) {
+      if (((PollMessage) entity).getRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (RegistryEnvironment.get().equals(RegistryEnvironment.PRODUCTION)
+        && entity instanceof Registrar) {
+      Registrar registrar = (Registrar) entity;
+      if (registrar.getRegistrarId().startsWith("prober-wj-")) {
+        return true;
+      }
+    }
+    return false;
+  }
+}

core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java

@@ -25,6 +25,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.datastore.v1.Entity;
+import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.util.Iterator;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
@@ -78,6 +80,7 @@ import org.apache.beam.sdk.values.TupleTagList;
  * types in the Datastore using the {@code --numOfKindsHint} argument. If the default value for this
  * parameter is too low, performance will suffer.
  */
+@DeleteAfterMigration
 public class BulkDeleteDatastorePipeline {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -308,6 +311,11 @@ public class BulkDeleteDatastorePipeline {
 
   public interface BulkDeletePipelineOptions extends GcpOptions {
 
+    @Description("The Registry environment.")
+    RegistryEnvironment getRegistryEnvironment();
+
+    void setRegistryEnvironment(RegistryEnvironment environment);
+
     @Description(
         "The Datastore KINDs to be deleted. The format may be:\n"
             + "\t- The list of kinds to be deleted as a comma-separated string, or\n"

core/src/main/java/google/registry/beam/datastore/DatastoreV1.java

@@ -53,6 +53,7 @@ import com.google.datastore.v1.client.DatastoreOptions;
 import com.google.datastore.v1.client.QuerySplitter;
 import com.google.protobuf.Int32Value;
 import com.google.rpc.Code;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.Serializable;
 import java.util.List;
 import java.util.NoSuchElementException;
@@ -80,6 +81,7 @@ import org.joda.time.Duration;
  * Contains an adaptation of {@link org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read}. See
  * {@link MultiRead} for details.
  */
+@DeleteAfterMigration
 public class DatastoreV1 {
 
   // A package-private constructor to prevent direct instantiation from outside of this package
@@ -169,14 +171,15 @@ public class DatastoreV1 {
       int numSplits;
       try {
         long estimatedSizeBytes = getEstimatedSizeBytes(datastore, query, namespace);
-        logger.atInfo().log("Estimated size bytes for the query is: %s", estimatedSizeBytes);
+        logger.atInfo().log("Estimated size for the query is %d bytes.", estimatedSizeBytes);
         numSplits =
             (int)
                 Math.min(
                     NUM_QUERY_SPLITS_MAX,
                     Math.round(((double) estimatedSizeBytes) / DEFAULT_BUNDLE_SIZE_BYTES));
       } catch (Exception e) {
-        logger.atWarning().log("Failed the fetch estimatedSizeBytes for query: %s", query, e);
+        logger.atWarning().withCause(e).log(
+            "Failed the fetch estimatedSizeBytes for query: %s", query);
         // Fallback in case estimated size is unavailable.
         numSplits = NUM_QUERY_SPLITS_MIN;
       }
@@ -215,7 +218,7 @@ public class DatastoreV1 {
     private static Entity getLatestTableStats(
         String ourKind, @Nullable String namespace, Datastore datastore) throws DatastoreException {
       long latestTimestamp = queryLatestStatisticsTimestamp(datastore, namespace);
-      logger.atInfo().log("Latest stats timestamp for kind %s is %s", ourKind, latestTimestamp);
+      logger.atInfo().log("Latest stats timestamp for kind %s is %s.", ourKind, latestTimestamp);
 
       Query.Builder queryBuilder = Query.newBuilder();
       if (Strings.isNullOrEmpty(namespace)) {
@@ -234,7 +237,7 @@ public class DatastoreV1 {
       long now = System.currentTimeMillis();
       RunQueryResponse response = datastore.runQuery(request);
       logger.atFine().log(
-          "Query for per-kind statistics took %sms", System.currentTimeMillis() - now);
+          "Query for per-kind statistics took %d ms.", System.currentTimeMillis() - now);
 
       QueryResultBatch batch = response.getBatch();
       if (batch.getEntityResultsCount() == 0) {
@@ -330,7 +333,7 @@ public class DatastoreV1 {
           logger.atWarning().log(
               "Failed to translate Gql query '%s': %s", gqlQueryWithZeroLimit, e.getMessage());
           logger.atWarning().log(
-              "User query might have a limit already set, so trying without zero limit");
+              "User query might have a limit already set, so trying without zero limit.");
           // Retry without the zero limit.
           return translateGqlQuery(gql, datastore, namespace);
         } else {
@@ -514,10 +517,10 @@ public class DatastoreV1 {
       @ProcessElement
       public void processElement(ProcessContext c) throws Exception {
         String gqlQuery = c.element();
-        logger.atInfo().log("User query: '%s'", gqlQuery);
+        logger.atInfo().log("User query: '%s'.", gqlQuery);
         Query query =
             translateGqlQueryWithLimitCheck(gqlQuery, datastore, v1Options.getNamespace());
-        logger.atInfo().log("User gql query translated to Query(%s)", query);
+        logger.atInfo().log("User gql query translated to Query(%s).", query);
         c.output(query);
       }
     }
@@ -573,7 +576,7 @@ public class DatastoreV1 {
           estimatedNumSplits = numSplits;
         }
 
-        logger.atInfo().log("Splitting the query into %s splits", estimatedNumSplits);
+        logger.atInfo().log("Splitting the query into %d splits.", estimatedNumSplits);
         List<Query> querySplits;
         try {
           querySplits =
@@ -647,7 +650,7 @@ public class DatastoreV1 {
               throw exception;
             }
             if (!BackOffUtils.next(sleeper, backoff)) {
-              logger.atSevere().log("Aborting after %s retries.", MAX_RETRIES);
+              logger.atSevere().log("Aborting after %d retries.", MAX_RETRIES);
               throw exception;
             }
           }

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -21,12 +21,14 @@ import static com.google.common.base.Strings.isNullOrEmpty;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
+import google.registry.model.annotations.DeleteAfterMigration;
 import org.joda.time.DateTime;
 
 /**
  * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
  * Datastore export and Nomulus CommitLogs that overlap with the export.
  */
+@DeleteAfterMigration
 public final class BackupPaths {
 
   private BackupPaths() {}

core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java

@@ -18,9 +18,11 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.appengine.api.datastore.Entity;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.util.Objects;
 
 /** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */
+@DeleteAfterMigration
 final class DomainBaseUtil {
 
   private DomainBaseUtil() {}

core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java

@@ -20,11 +20,12 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.backup.VersionedEntity;
 import google.registry.beam.common.RegistryJpaIO;
 import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken;
@@ -62,6 +63,7 @@ import org.joda.time.DateTime;
  * <ol>
  *   <li>{@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set
  *       up in the SQL database.
+ *   <li>{@link Cursor}: Logically can depend on {@code Registry}, but without foreign key.
  *   <li>{@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet.
  *   <li>{@link ContactResource}: references {@code Registrar}
  *   <li>{@link RegistrarContact}: references {@code Registrar}.
@@ -93,6 +95,7 @@ import org.joda.time.DateTime;
  * may start writing {@code DomainBase} entities before all {@code Registry}, {@code Registrar} and
  * {@code ContactResource} entities have been persisted.
  */
+@DeleteAfterMigration
 public class InitSqlPipeline implements Serializable {
 
   /**
@@ -101,7 +104,11 @@ public class InitSqlPipeline implements Serializable {
    */
   private static final ImmutableList<Class<?>> PHASE_ONE_ORDERED =
       ImmutableList.of(
-          Registry.class, Registrar.class, ContactResource.class, RegistrarContact.class);
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class);
 
   /**
    * Datastore kinds to be written to the SQL database after the cleansed version of {@link
@@ -219,13 +226,12 @@ public class InitSqlPipeline implements Serializable {
             .withName(transformId)
             .withBatchSize(options.getSqlWriteBatchSize())
             .withShards(options.getSqlWriteShards())
-            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity));
+            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity)
+            .disableUpdateAutoTimestamp());
   }
 
   private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
-    try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-      return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
-    }
+    return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
   }
 
   public static void main(String[] args) {

core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java

@@ -15,10 +15,12 @@
 package google.registry.beam.initsql;
 
 import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.model.annotations.DeleteAfterMigration;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.Validation;
 
 /** Pipeline options for {@link InitSqlPipeline} */
+@DeleteAfterMigration
 public interface InitSqlPipelineOptions extends RegistryPipelineOptions {
 
   @Description("The root directory of the export to load.")

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -22,6 +22,7 @@ import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.util.Comparator.comparing;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -36,6 +37,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.DomainBase;
@@ -79,6 +81,7 @@ import org.joda.time.DateTime;
  * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export
  * files and Nomulus CommitLog files.
  */
+@DeleteAfterMigration
 public final class Transforms {
 
   private Transforms() {}
@@ -100,8 +103,9 @@ public final class Transforms {
   }
 
   /**
-   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
-   * commitLogToTime} for caller specified {@code kinds}.
+   * Composite {@link PTransform transform} that loads the Datastore snapshot right before {@code
+   * commitLogToTime} for caller specified {@code kinds}. The resulting snapshot has all changes
+   * that happened before {@code commitLogToTime}, and none at or after {@code commitLogToTime}.
    *
    * <p>Caller must provide the location of a Datastore export that started AFTER {@code
    * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
@@ -259,16 +263,19 @@ public final class Transforms {
                     .iterator()));
   }
 
-  // Production data repair configs go below. See b/185954992.
+  // Production data repair configs go below. See b/185954992. Note that the CommitLog replay
+  // process does not filter out the ignored entities listed below, a mistake that we do not fix
+  // for operational convenience. Instead, the Database comparison tool will filter them out. See
+  // ValidateSqlUtils.java for more information.
 
-  // Prober domains in bad state, without associated contacts, hosts, billings, and history.
-  // They can be safely ignored.
-  private static final ImmutableSet<String> IGNORED_DOMAINS =
+  // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized
+  // history. They can be safely ignored.
+  public static final ImmutableSet<String> IGNORED_DOMAINS =
       ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT");
 
   // Prober hosts referencing phantom registrars. They and their associated history entries can be
   // safely ignored.
-  private static final ImmutableSet<String> IGNORED_HOSTS =
+  public static final ImmutableSet<String> IGNORED_HOSTS =
       ImmutableSet.of(
           "4E21_WJ0TEST-GOOGLE",
           "4E21_WJ1TEST-GOOGLE",
@@ -277,7 +284,7 @@ public final class Transforms {
 
   // Prober contacts referencing phantom registrars. They and their associated history entries can
   // be safely ignored.
-  private static final ImmutableSet IGNORED_CONTACTS =
+  public static final ImmutableSet<String> IGNORED_CONTACTS =
       ImmutableSet.of(
           "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE");
 
@@ -298,7 +305,14 @@ public final class Transforms {
       return !IGNORED_HOSTS.contains(roid);
     }
     if (entity.getKind().equals("HistoryEntry")) {
-      // Remove production bad data: History of the contacts to be ignored:
+      // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated.
+      // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot
+      // be deserialized.
+      if (Objects.equals(entity.getProperty("type"), "DOMAIN_APPLICATION_CREATE")) {
+        return false;
+      }
+
+      // Remove production bad data: Histories of ignored EPP resources:
       com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent();
       if (parentKey.getKind().equals("ContactResource")) {
         String contactRoid = parentKey.getName();
@@ -308,19 +322,16 @@ public final class Transforms {
         String hostRoid = parentKey.getName();
         return !IGNORED_HOSTS.contains(hostRoid);
       }
-    }
-    // End of production-specific checks.
-
-    if (entity.getKind().equals("HistoryEntry")) {
-      // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated.
-      // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot
-      // be deserialized.
-      return !Objects.equals(entity.getProperty("type"), "DOMAIN_APPLICATION_CREATE");
+      if (parentKey.getKind().equals("DomainBase")) {
+        String domainRoid = parentKey.getName();
+        return !IGNORED_DOMAINS.contains(domainRoid);
+      }
     }
     return true;
   }
 
-  private static Entity repairBadData(Entity entity) {
+  @VisibleForTesting
+  static Entity repairBadData(Entity entity) {
     if (entity.getKind().equals("Cancellation")
         && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) {
       // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the
@@ -328,6 +339,15 @@ public final class Transforms {
       // instead of append. See b/185954992.
       entity.setUnindexedProperty("reason", Reason.RENEW.name());
       entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name()));
+    } else if (entity.getKind().equals("DomainBase")) {
+      // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this.
+      entity.setIndexedProperty(
+          "fullyQualifiedDomainName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedDomainName")));
+    } else if (entity.getKind().equals("HostResource")) {
+      entity.setIndexedProperty(
+          "fullyQualifiedHostName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedHostName")));
     }
     return entity;
   }
@@ -348,7 +368,7 @@ public final class Transforms {
    *     to make Optional work with BEAM)
    */
   @Nullable
-  public static Object convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
+  public static SqlEntity convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
     return dsEntity
         .getEntity()
         .filter(Transforms::isMigratable)
@@ -365,7 +385,8 @@ public final class Transforms {
    * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
    * given {@link Iterable}.
    */
-  static PTransform<PBegin, PCollection<String>> toStringPCollection(Iterable<String> strings) {
+  private static PTransform<PBegin, PCollection<String>> toStringPCollection(
+      Iterable<String> strings) {
     return Create.of(strings).withCoder(StringUtf8Coder.of());
   }
 
@@ -373,7 +394,7 @@ public final class Transforms {
    * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
    * caller-provided {@code transformer}.
    */
-  static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+  private static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
       DoFn<ReadableFile, VersionedEntity> transformer) {
     return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
       @Override
@@ -389,7 +410,7 @@ public final class Transforms {
     private final DateTime fromTime;
     private final DateTime toTime;
 
-    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+    FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
       checkNotNull(fromTime, "fromTime");
       checkNotNull(toTime, "toTime");
       checkArgument(

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -57,7 +57,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=invoicing}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
@@ -125,7 +126,7 @@ public class InvoicingPipeline implements Serializable {
         oneTime.getId(),
         DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
         DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
-        registrar.getClientId(),
+        registrar.getRegistrarId(),
         registrar.getBillingIdentifier().toString(),
         registrar.getPoNumber().orElse(""),
         DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -19,10 +19,13 @@ import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static google.registry.rde.RdeModule.BRDA_QUEUE;
+import static google.registry.rde.RdeModule.RDE_UPLOAD_QUEUE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.auto.value.AutoValue;
 import com.google.cloud.storage.BlobId;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.PgpHelper;
@@ -31,14 +34,20 @@ import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
 import google.registry.model.tld.Registry;
+import google.registry.rde.BrdaCopyAction;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.Ghostryde;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.RdeCounter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.rde.RdeModule;
 import google.registry.rde.RdeResourceType;
+import google.registry.rde.RdeUploadAction;
 import google.registry.rde.RdeUtil;
+import google.registry.request.Action.Service;
+import google.registry.request.RequestParameters;
 import google.registry.tldconfig.idn.IdnTableEnum;
+import google.registry.util.CloudTasksUtils;
 import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
 import google.registry.xml.ValidationMode;
@@ -66,8 +75,12 @@ public class RdeIO {
   abstract static class Write
       extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
 
+    private static final long serialVersionUID = 3334807737227087760L;
+
     abstract GcsUtils gcsUtils();
 
+    abstract CloudTasksUtils cloudTasksUtils();
+
     abstract String rdeBucket();
 
     // It's OK to return a primitive array because we are only using it to construct the
@@ -83,7 +96,9 @@ public class RdeIO {
 
     @AutoValue.Builder
     abstract static class Builder {
-      abstract Builder setGcsUtils(GcsUtils gcsUtils);
+      abstract Builder setGcsUtils(GcsUtils value);
+
+      abstract Builder setCloudTasksUtils(CloudTasksUtils value);
 
       abstract Builder setRdeBucket(String value);
 
@@ -100,7 +115,9 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()));
+          .apply(
+              "Update cursor and enqueue next action",
+              ParDo.of(new CursorUpdater(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -109,6 +126,7 @@ public class RdeIO {
       extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5496375923068400382L;
 
     private final GcsUtils gcsUtils;
     private final String rdeBucket;
@@ -149,13 +167,13 @@ public class RdeIO {
           Optional.ofNullable(key.revision())
               .orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
       String id = RdeUtil.timestampToId(watermark);
-      String prefix = options.getJobName();
-      String basename = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
+      String prefix =
+          options.getJobName()
+              + '/'
+              + RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
       if (key.manual()) {
         checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
-        prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
-      } else {
-        prefix = prefix + "/" + basename;
+        prefix = "manual/" + key.directoryWithTrailingSlash() + prefix;
       }
       BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
       // This file will contain the byte length (ASCII) of the raw unencrypted XML.
@@ -172,7 +190,7 @@ public class RdeIO {
 
       // Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
 
-      logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
+      logger.atInfo().log("Writing files '%s' and '%s'.", xmlFilename, xmlLengthFilename);
       try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
           OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
           OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
@@ -219,7 +237,7 @@ public class RdeIO {
       //
       // This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
       if (mode == RdeMode.FULL) {
-        logger.atInfo().log("Writing %s", reportFilename);
+        logger.atInfo().log("Writing file '%s'.", reportFilename);
         try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
             OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
           counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
@@ -229,7 +247,10 @@ public class RdeIO {
       }
       // Now that we're done, output roll the cursor forward.
       if (key.manual()) {
-        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
+        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task.");
+        // Temporary measure to run RDE in beam in parallel with the daily MapReduce based RDE runs.
+      } else if (tm().isOfy()) {
+        logger.atInfo().log("Ofy is primary TM; not advancing cursor or enqueuing upload task.");
       } else {
         outputReceiver.output(KV.of(key, revision));
       }
@@ -237,10 +258,19 @@ public class RdeIO {
   }
 
   private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5822176227753327224L;
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private CursorUpdater(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
 
     @ProcessElement
-    public void processElement(@Element KV<PendingDeposit, Integer> input) {
+    public void processElement(
+        @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -265,8 +295,33 @@ public class RdeIO {
                     key);
                 tm().put(Cursor.create(key.cursor(), newPosition, registry));
                 logger.atInfo().log(
-                    "Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
+                    "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                if (key.mode() == RdeMode.FULL) {
+                  cloudTasksUtils.enqueue(
+                      RDE_UPLOAD_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          RdeUploadAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                } else {
+                  cloudTasksUtils.enqueue(
+                      BRDA_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          BrdaCopyAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_WATERMARK,
+                              key.watermark().toString(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                }
               });
     }
   }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -14,36 +14,53 @@
 
 package google.registry.beam.rde;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.beam.rde.RdePipeline.TupleTags.DOMAIN_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.EXTERNAL_HOST_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.HOST_TO_PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_CONTACTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
+import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
+import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
-import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.util.CloudTasksUtils;
+import google.registry.util.UtilsModule;
 import google.registry.xml.ValidationMode;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -51,73 +68,158 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.persistence.Entity;
+import javax.persistence.IdClass;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.DateTime;
 
 /**
  * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=rde}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
+ * <p>This pipeline only works for pending deposits with the same watermark, the {@link
+ * google.registry.rde.RdeStagingAction} will batch such pending deposits together and launch
+ * multiple pipelines if multiple watermarks exist.
+ *
+ * <p>The pipeline is broadly divided into two parts -- creating the {@link DepositFragment}s, and
+ * processing them.
+ *
+ * <h1>Creating {@link DepositFragment}</h1>
+ *
+ * <h2>{@link Registrar}</h2>
+ *
+ * Non-test registrar entities are loaded from Cloud SQL and marshalled into deposit fragments. They
+ * are <b>NOT</b> rewound to the watermark.
+ *
+ * <h2>{@link EppResource}</h2>
+ *
+ * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
+ * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
+ * revision ID) from the SQL query.
+ *
+ * <h3>{@link DomainBase}</h3>
+ *
+ * After the most recent (live) domain resources are loaded from the corresponding history objects,
+ * we marshall them to deposit fragments and emit the (pending deposit: deposit fragment) pairs for
+ * further processing. We also find all the contacts and hosts referenced by a given domain and emit
+ * pairs of (contact/host repo ID: pending deposit) for all RDE pending deposits for further
+ * processing.
+ *
+ * <h3>{@link ContactResource}</h3>
+ *
+ * We first join most recent contact histories, represented by (contact repo ID: contact history
+ * revision ID) pairs, with referenced contacts, represented by (contact repo ID: pending deposit)
+ * pairs, on the contact repo ID, to remove unreferenced contact histories. Contact resources are
+ * then loaded from the remaining referenced contact histories, and marshalled into (pending
+ * deposit: deposit fragment) pairs.
+ *
+ * <h3>{@link HostResource}</h3>
+ *
+ * Similar to {@link ContactResource}, we join the most recent host history with referenced hosts to
+ * find most recent referenced hosts. For external hosts we do the same treatment as we did on
+ * contacts and obtain the (pending deposit: deposit fragment) pairs. For subordinate hosts, we need
+ * to find the superordinate domain in order to properly handle pending transfer in the deposit as
+ * well. So we first find the superordinate domain repo ID from the host and join the (superordinate
+ * domain repo ID: (subordinate host repo ID: (pending deposit: revision ID))) pair with the (domain
+ * repo ID: revision ID) pair obtained from the domain history query in order to map the host at
+ * watermark to the domain at watermark. We then proceed to create the (pending deposit: deposit
+ * fragment) pair for subordinate hosts using the added domain information.
+ *
+ * <h1>Processing {@link DepositFragment}</h1>
+ *
+ * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
+ * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
+ * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
+ * pending deposit is then rolled forward, and the next action is enqueued. The latter two
+ * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
+ *
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
 @Singleton
 public class RdePipeline implements Serializable {
 
+  private static final long serialVersionUID = -4866795928854754666L;
   private final transient RdePipelineOptions options;
   private final ValidationMode mode;
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final ImmutableSet<PendingDeposit> pendingDeposits;
+  private final DateTime watermark;
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
+  private final CloudTasksUtils cloudTasksUtils;
+  private final RdeMarshaller marshaller;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
   private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
       Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
 
-  private static final String EPP_RESOURCE_QUERY =
-      "SELECT id FROM %entity% "
-          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
-
-  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
-    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
-        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
-  }
+  // The field name of the EPP resource embedded in its corresponding history entry.
+  private static final ImmutableMap<Class<? extends HistoryEntry>, String> EPP_RESOURCE_FIELD_NAME =
+      ImmutableMap.of(
+          DomainHistory.class,
+          "domainContent",
+          ContactHistory.class,
+          "contactBase",
+          HostHistory.class,
+          "hostBase");
 
   @Inject
-  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
+  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
-    this.pendings = decodePendings(options.getPendings());
-    this.rdeBucket = options.getGcsBucket();
+    this.pendingDeposits = decodePendingDeposits(options.getPendings());
+    ImmutableSet<DateTime> potentialWatermarks =
+        pendingDeposits.stream()
+            .map(PendingDeposit::watermark)
+            .distinct()
+            .collect(toImmutableSet());
+    checkArgument(
+        potentialWatermarks.size() == 1,
+        String.format(
+            "RDE pipeline should only work on pending deposits "
+                + "with the same watermark, but %d were given: %s",
+            potentialWatermarks.size(), potentialWatermarks));
+    this.watermark = potentialWatermarks.asList().get(0);
+    this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
+    this.marshaller = new RdeMarshaller(mode);
   }
 
   PipelineResult run() {
@@ -129,152 +231,452 @@ public class RdePipeline implements Serializable {
   }
 
   PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
-    return PCollectionList.of(processRegistrars(pipeline))
-        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
-        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
-        .and(processNonRegistrarEntities(pipeline, HostResource.class))
-        .apply(Flatten.pCollections())
+    PCollection<KV<PendingDeposit, DepositFragment>> registrarFragments =
+        processRegistrars(pipeline);
+
+    PCollection<KV<String, Long>> domainHistories =
+        getMostRecentHistoryEntries(pipeline, DomainHistory.class);
+
+    PCollection<KV<String, Long>> contactHistories =
+        getMostRecentHistoryEntries(pipeline, ContactHistory.class);
+
+    PCollection<KV<String, Long>> hostHistories =
+        getMostRecentHistoryEntries(pipeline, HostHistory.class);
+
+    PCollectionTuple processedDomainHistories = processDomainHistories(domainHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> domainFragments =
+        processedDomainHistories.get(DOMAIN_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> contactFragments =
+        processContactHistories(
+            processedDomainHistories.get(REFERENCED_CONTACTS), contactHistories);
+
+    PCollectionTuple processedHosts =
+        processHostHistories(processedDomainHistories.get(REFERENCED_HOSTS), hostHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> externalHostFragments =
+        processedHosts.get(EXTERNAL_HOST_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> subordinateHostFragments =
+        processSubordinateHosts(processedHosts.get(SUPERORDINATE_DOMAINS), domainHistories);
+
+    return PCollectionList.of(registrarFragments)
+        .and(domainFragments)
+        .and(contactFragments)
+        .and(externalHostFragments)
+        .and(subordinateHostFragments)
+        .apply(
+            "Combine PendingDeposit:DepositFragment pairs from all entities",
+            Flatten.pCollections())
         .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply("Group by PendingDeposit", GroupByKey.create());
+        .apply("Group DepositFragment by PendingDeposit", GroupByKey.create());
   }
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
     input.apply(
-        "Write to GCS and update cursors",
+        "Write to GCS, update cursors, and enqueue upload tasks",
         RdeIO.Write.builder()
             .setRdeBucket(rdeBucket)
             .setGcsUtils(gcsUtils)
+            .setCloudTasksUtils(cloudTasksUtils)
             .setValidationMode(mode)
             .setStagingKeyBytes(stagingKeyBytes)
             .build());
   }
 
-  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+  private PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    // Note that the namespace in the metric is not being used by Stackdriver, it just has to be
+    // non-empty.
+    // See:
+    // https://stackoverflow.com/questions/48530496/google-dataflow-custom-metrics-not-showing-on-stackdriver
+    Counter includedRegistrarCounter = Metrics.counter("RDE", "IncludedRegistrar");
+    Counter registrarFragmentCounter = Metrics.counter("RDE", "RegistrarFragment");
     return pipeline
         .apply(
-            "Read all production Registrar entities",
+            "Read all production Registrars",
             RegistryJpaIO.read(
                 "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
                 ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
                 String.class,
-                // TODO: consider adding coders for entities and pass them directly instead of using
-                // VKeys.
                 id -> VKey.createSql(Registrar.class, id)))
         .apply(
-            "Marshal Registrar into DepositFragment",
+            "Marshall Registrar into DepositFragment",
             FlatMapElements.into(
-                    TypeDescriptors.kvs(
+                    kvs(
                         TypeDescriptor.of(PendingDeposit.class),
                         TypeDescriptor.of(DepositFragment.class)))
                 .via(
                     (VKey<Registrar> key) -> {
+                      includedRegistrarCounter.inc();
                       Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
-                      DepositFragment fragment =
-                          new RdeMarshaller(mode).marshalRegistrar(registrar);
-                      return pendings.values().stream()
-                          .map(pending -> KV.of(pending, fragment))
-                          .collect(toImmutableSet());
+                      DepositFragment fragment = marshaller.marshalRegistrar(registrar);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          pendingDeposits.stream()
+                              .map(pending -> KV.of(pending, fragment))
+                              .collect(toImmutableSet());
+                      registrarFragmentCounter.inc(fragments.size());
+                      return fragments;
                     }));
   }
 
-  @SuppressWarnings("deprecation") // Reshuffle is still recommended by Dataflow.
-  <T extends EppResource>
-      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
-          Pipeline pipeline, Class<T> clazz) {
-    return createInputs(pipeline, clazz)
-        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
+  /**
+   * Load the most recent history entry before the watermark for a given history entry type.
+   *
+   * <p>Note that deleted and non-production resources are not included.
+   *
+   * @return A KV pair of (repoId, revisionId), used to reconstruct the composite key for the
+   *     history entry.
+   */
+  private <T extends HistoryEntry> PCollection<KV<String, Long>> getMostRecentHistoryEntries(
+      Pipeline pipeline, Class<T> historyClass) {
+    String repoIdFieldName = HistoryEntryDao.REPO_ID_FIELD_NAMES.get(historyClass);
+    String resourceFieldName = EPP_RESOURCE_FIELD_NAME.get(historyClass);
+    return pipeline
         .apply(
-            "Reshuffle KV<PendingDeposit, DepositFragment> of "
-                + clazz.getSimpleName()
-                + " to prevent fusion",
-            Reshuffle.of());
+            String.format("Load most recent %s", historyClass.getSimpleName()),
+            RegistryJpaIO.read(
+                ("SELECT %repoIdField%, id FROM %entity% WHERE (%repoIdField%, modificationTime)"
+                     + " IN (SELECT %repoIdField%, MAX(modificationTime) FROM %entity% WHERE"
+                     + " modificationTime <= :watermark GROUP BY %repoIdField%) AND"
+                     + " %resourceField%.deletionTime > :watermark AND"
+                     + " COALESCE(%resourceField%.creationClientId, '') NOT LIKE 'prober-%' AND"
+                     + " COALESCE(%resourceField%.currentSponsorClientId, '') NOT LIKE 'prober-%'"
+                     + " AND COALESCE(%resourceField%.lastEppUpdateClientId, '') NOT LIKE"
+                     + " 'prober-%' "
+                        + (historyClass == DomainHistory.class
+                            ? "AND %resourceField%.tld IN "
+                                + "(SELECT id FROM Tld WHERE tldType = 'REAL')"
+                            : ""))
+                    .replace("%entity%", historyClass.getSimpleName())
+                    .replace("%repoIdField%", repoIdFieldName)
+                    .replace("%resourceField%", resourceFieldName),
+                ImmutableMap.of("watermark", watermark),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1])))
+        .setCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));
   }
 
-  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
-    return pipeline.apply(
-        "Read all production " + clazz.getSimpleName() + " entities",
-        RegistryJpaIO.read(
-            createEppResourceQuery(clazz),
-            clazz.equals(DomainBase.class)
-                ? ImmutableMap.of("tlds", pendings.keySet())
-                : ImmutableMap.of(),
-            String.class,
-            // TODO: consider adding coders for entities and pass them directly instead of using
-            // VKeys.
-            x -> VKey.create(clazz, x)));
+  private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
+      Class<T> historyEntryClazz, String repoId, long revisionId) {
+    try {
+      Class<?> idClazz = historyEntryClazz.getAnnotation(IdClass.class).value();
+      Serializable idObject =
+          (Serializable)
+              idClazz.getConstructor(String.class, long.class).newInstance(repoId, revisionId);
+      return jpaTm()
+          .transact(() -> jpaTm().loadByKey(VKey.createSql(historyEntryClazz, idObject)))
+          .getResourceAtPointInTime()
+          .map(resource -> resource.cloneProjectedAtTime(watermark))
+          .get();
+    } catch (NoSuchMethodException
+        | InvocationTargetException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot load resource from %s with repoId %s and revisionId %s",
+              historyEntryClazz.getSimpleName(), repoId, revisionId),
+          e);
+    }
   }
 
-  <T extends EppResource>
-      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
-    return FlatMapElements.into(
-            TypeDescriptors.kvs(
-                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
-        .via(
-            (VKey<T> key) -> {
-              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
-              // The set of all TLDs to which this resource should be emitted.
-              ImmutableSet<String> tlds =
-                  clazz.equals(DomainBase.class)
-                      ? ImmutableSet.of(((DomainBase) resource).getTld())
-                      : pendings.keySet();
-              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
-              ImmutableSet<DateTime> dates =
-                  tlds.stream()
-                      .map(pendings::get)
-                      .flatMap(ImmutableSet::stream)
-                      .map(PendingDeposit::watermark)
-                      .collect(toImmutableSet());
-              // Launch asynchronous fetches of point-in-time representations of resource.
-              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
-                  ImmutableMap.copyOf(
-                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
-              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
-              // the result.
-              RdeFragmenter fragmenter =
-                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
-              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
-              for (String tld : tlds) {
-                for (PendingDeposit pending : pendings.get(tld)) {
-                  // Hosts and contacts don't get included in BRDA deposits.
-                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
-                    continue;
+  /**
+   * Remove unreferenced resources by joining the (repoId, pendingDeposit) pair with the (repoId,
+   * revisionId) on the repoId.
+   *
+   * <p>The (repoId, pendingDeposit) pairs denote resources (contact, host) that are referenced from
+   * a domain, that are to be included in the corresponding pending deposit.
+   *
+   * <p>The (repoId, revisionId) paris come from the most recent history entry query, which can be
+   * used to load the embedded resources themselves.
+   *
+   * @return a pair of (repoId, ([pendingDeposit], [revisionId])) where neither the pendingDeposit
+   *     nor the revisionId list is empty.
+   */
+  private static PCollection<KV<String, CoGbkResult>> removeUnreferencedResource(
+      PCollection<KV<String, PendingDeposit>> referencedResources,
+      PCollection<KV<String, Long>> historyEntries,
+      Class<? extends EppResource> resourceClazz) {
+    String resourceName = resourceClazz.getSimpleName();
+    Class<? extends HistoryEntry> historyEntryClazz =
+        RESOURCE_TYPES_TO_HISTORY_TYPES.get(resourceClazz);
+    String historyEntryName = historyEntryClazz.getSimpleName();
+    Counter referencedResourceCounter = Metrics.counter("RDE", "Referenced" + resourceName);
+    return KeyedPCollectionTuple.of(PENDING_DEPOSIT, referencedResources)
+        .and(REVISION_ID, historyEntries)
+        .apply(
+            String.format(
+                "Join PendingDeposit with %s revision ID on %s", historyEntryName, resourceName),
+            CoGroupByKey.create())
+        .apply(
+            String.format("Remove unreferenced %s", resourceName),
+            Filter.by(
+                (KV<String, CoGbkResult> kv) -> {
+                  boolean toInclude =
+                      // If a resource does not have corresponding pending deposit, it is not
+                      // referenced and should not be included.
+                      kv.getValue().getAll(PENDING_DEPOSIT).iterator().hasNext()
+                          // If a resource does not have revision id (this should not happen, as
+                          // every referenced resource must be valid at watermark time, therefore
+                          // be embedded in a history entry valid at watermark time, otherwise
+                          // the domain cannot reference it), there is no way for us to find the
+                          // history entry and load the embedded resource. So we ignore the resource
+                          // to keep the downstream process simple.
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedResourceCounter.inc();
                   }
-                  Optional<DepositFragment> fragment =
-                      fragmenter.marshal(pending.watermark(), pending.mode());
-                  fragment.ifPresent(
-                      depositFragment -> results.add(KV.of(pending, depositFragment)));
-                }
-              }
-              return results;
-            });
+                  return toInclude;
+                }));
+  }
+
+  private PCollectionTuple processDomainHistories(PCollection<KV<String, Long>> domainHistories) {
+    Counter activeDomainCounter = Metrics.counter("RDE", "ActiveDomainBase");
+    Counter domainFragmentCounter = Metrics.counter("RDE", "DomainFragment");
+    Counter referencedContactCounter = Metrics.counter("RDE", "ReferencedContactResource");
+    Counter referencedHostCounter = Metrics.counter("RDE", "ReferencedHostResource");
+    return domainHistories.apply(
+        "Map DomainHistory to DepositFragment "
+            + "and emit referenced ContactResource and HostResource",
+        ParDo.of(
+                new DoFn<KV<String, Long>, KV<PendingDeposit, DepositFragment>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Long> kv, MultiOutputReceiver receiver) {
+                    activeDomainCounter.inc();
+                    DomainBase domain =
+                        (DomainBase)
+                            loadResourceByHistoryEntryId(
+                                DomainHistory.class, kv.getKey(), kv.getValue());
+                    pendingDeposits.stream()
+                        .filter(pendingDeposit -> pendingDeposit.tld().equals(domain.getTld()))
+                        .forEach(
+                            pendingDeposit -> {
+                              // Domains are always deposited in both modes.
+                              domainFragmentCounter.inc();
+                              receiver
+                                  .get(DOMAIN_FRAGMENTS)
+                                  .output(
+                                      KV.of(
+                                          pendingDeposit,
+                                          marshaller.marshalDomain(domain, pendingDeposit.mode())));
+                              // Contacts and hosts are only deposited in RDE, not BRDA.
+                              if (pendingDeposit.mode() == RdeMode.FULL) {
+                                HashSet<Serializable> contacts = new HashSet<>();
+                                contacts.add(domain.getAdminContact().getSqlKey());
+                                contacts.add(domain.getTechContact().getSqlKey());
+                                contacts.add(domain.getRegistrant().getSqlKey());
+                                // Billing contact is not mandatory.
+                                if (domain.getBillingContact() != null) {
+                                  contacts.add(domain.getBillingContact().getSqlKey());
+                                }
+                                referencedContactCounter.inc(contacts.size());
+                                contacts.forEach(
+                                    contactRepoId ->
+                                        receiver
+                                            .get(REFERENCED_CONTACTS)
+                                            .output(KV.of((String) contactRepoId, pendingDeposit)));
+                                if (domain.getNsHosts() != null) {
+                                  referencedHostCounter.inc(domain.getNsHosts().size());
+                                  domain
+                                      .getNsHosts()
+                                      .forEach(
+                                          hostKey ->
+                                              receiver
+                                                  .get(REFERENCED_HOSTS)
+                                                  .output(
+                                                      KV.of(
+                                                          (String) hostKey.getSqlKey(),
+                                                          pendingDeposit)));
+                                }
+                              }
+                            });
+                  }
+                })
+            .withOutputTags(
+                DOMAIN_FRAGMENTS, TupleTagList.of(REFERENCED_CONTACTS).and(REFERENCED_HOSTS)));
+  }
+
+  private PCollection<KV<PendingDeposit, DepositFragment>> processContactHistories(
+      PCollection<KV<String, PendingDeposit>> referencedContacts,
+      PCollection<KV<String, Long>> contactHistories) {
+    Counter contactFragmentCounter = Metrics.counter("RDE", "ContactFragment");
+    return removeUnreferencedResource(referencedContacts, contactHistories, ContactResource.class)
+        .apply(
+            "Map ContactResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      ContactResource contact =
+                          (ContactResource)
+                              loadResourceByHistoryEntryId(
+                                  ContactHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      DepositFragment fragment = marshaller.marshalContact(contact);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same contact could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .map(pendingDeposit -> KV.of(pendingDeposit, fragment))
+                              .collect(toImmutableSet());
+                      contactFragmentCounter.inc(fragments.size());
+                      return fragments;
+                    }));
+  }
+
+  private PCollectionTuple processHostHistories(
+      PCollection<KV<String, PendingDeposit>> referencedHosts,
+      PCollection<KV<String, Long>> hostHistories) {
+    Counter subordinateHostCounter = Metrics.counter("RDE", "SubordinateHostResource");
+    Counter externalHostCounter = Metrics.counter("RDE", "ExternalHostResource");
+    Counter externalHostFragmentCounter = Metrics.counter("RDE", "ExternalHostFragment");
+    return removeUnreferencedResource(referencedHosts, hostHistories, HostResource.class)
+        .apply(
+            "Map external DomainResource to DepositFragment and process subordinate domains",
+            ParDo.of(
+                    new DoFn<KV<String, CoGbkResult>, KV<PendingDeposit, DepositFragment>>() {
+                      @ProcessElement
+                      public void processElement(
+                          @Element KV<String, CoGbkResult> kv, MultiOutputReceiver receiver) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    kv.getKey(),
+                                    kv.getValue().getOnly(REVISION_ID));
+                        // When a host is subordinate, we need to find it's superordinate domain and
+                        // include it in the deposit as well.
+                        if (host.isSubordinate()) {
+                          subordinateHostCounter.inc();
+                          receiver
+                              .get(SUPERORDINATE_DOMAINS)
+                              .output(
+                                  // The output are pairs of
+                                  // (superordinateDomainRepoId,
+                                  //   (subordinateHostRepoId, (pendingDeposit, revisionId))).
+                                  KV.of((String) host.getSuperordinateDomain().getSqlKey(), kv));
+                        } else {
+                          externalHostCounter.inc();
+                          DepositFragment fragment = marshaller.marshalExternalHost(host);
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same host could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .forEach(
+                                  pendingDeposit -> {
+                                    externalHostFragmentCounter.inc();
+                                    receiver
+                                        .get(EXTERNAL_HOST_FRAGMENTS)
+                                        .output(KV.of(pendingDeposit, fragment));
+                                  });
+                        }
+                      }
+                    })
+                .withOutputTags(EXTERNAL_HOST_FRAGMENTS, TupleTagList.of(SUPERORDINATE_DOMAINS)));
+  }
+
+  /**
+   * Process subordinate hosts by making a deposit fragment with pending transfer information
+   * obtained from its superordinate domain.
+   *
+   * @param superordinateDomains Pairs of (superordinateDomainRepoId, (subordinateHostRepoId,
+   *     (pendingDeposit, revisionId))). This collection maps the subordinate host and the pending
+   *     deposit to include it to its superordinate domain.
+   * @param domainHistories Pairs of (domainRepoId, revisionId). This collection helps us find the
+   *     historical superordinate domain from its history entry and is obtained from calling {@link
+   *     #getMostRecentHistoryEntries} for domains.
+   */
+  private PCollection<KV<PendingDeposit, DepositFragment>> processSubordinateHosts(
+      PCollection<KV<String, KV<String, CoGbkResult>>> superordinateDomains,
+      PCollection<KV<String, Long>> domainHistories) {
+    Counter subordinateHostFragmentCounter = Metrics.counter("RDE", "SubordinateHostFragment");
+    Counter referencedSubordinateHostCounter = Metrics.counter("RDE", "ReferencedSubordinateHost");
+    return KeyedPCollectionTuple.of(HOST_TO_PENDING_DEPOSIT, superordinateDomains)
+        .and(REVISION_ID, domainHistories)
+        .apply(
+            "Join HostResource:PendingDeposits with DomainHistory on DomainResource",
+            CoGroupByKey.create())
+        .apply(
+            " Remove unreferenced DomainResource",
+            Filter.by(
+                kv -> {
+                  boolean toInclude =
+                      kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT).iterator().hasNext()
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedSubordinateHostCounter.inc();
+                  }
+                  return toInclude;
+                }))
+        .apply(
+            "Map subordinate HostResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      DomainBase superordinateDomain =
+                          (DomainBase)
+                              loadResourceByHistoryEntryId(
+                                  DomainHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      ImmutableSet.Builder<KV<PendingDeposit, DepositFragment>> results =
+                          new ImmutableSet.Builder<>();
+                      for (KV<String, CoGbkResult> hostToPendingDeposits :
+                          kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT)) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    hostToPendingDeposits.getKey(),
+                                    hostToPendingDeposits.getValue().getOnly(REVISION_ID));
+                        DepositFragment fragment =
+                            marshaller.marshalSubordinateHost(host, superordinateDomain);
+                        Streams.stream(hostToPendingDeposits.getValue().getAll(PENDING_DEPOSIT))
+                            .distinct()
+                            .forEach(
+                                pendingDeposit -> {
+                                  subordinateHostFragmentCounter.inc();
+                                  results.add(KV.of(pendingDeposit, fragment));
+                                });
+                      }
+                      return results.build();
+                    }));
   }
 
   /**
    * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
-   * the original TLD to pending deposit map.
+   * the original pending deposit set.
    */
   @SuppressWarnings("unchecked")
-  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+  static ImmutableSet<PendingDeposit> decodePendingDeposits(String encodedPendingDeposits) {
     try (ObjectInputStream ois =
         new ObjectInputStream(
             new ByteArrayInputStream(
-                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
-      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+                BaseEncoding.base64Url().omitPadding().decode(encodedPendingDeposits)))) {
+      return (ImmutableSet<PendingDeposit>) ois.readObject();
     } catch (IOException | ClassNotFoundException e) {
       throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
     }
   }
 
   /**
-   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
-   * worker by the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
+   * the pipeline launcher as a pipeline option.
    */
-  static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
-      oos.writeObject(pendings);
+      oos.writeObject(pendingDeposits);
       oos.flush();
       return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
     }
@@ -282,13 +684,50 @@ public class RdePipeline implements Serializable {
 
   public static void main(String[] args) throws IOException, ClassNotFoundException {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
-    RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
+    RdePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
+  /**
+   * A utility class that contains {@link TupleTag}s when {@link PCollectionTuple}s and {@link
+   * CoGbkResult}s are used.
+   */
+  protected abstract static class TupleTags {
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> DOMAIN_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_CONTACTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_HOSTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, KV<String, CoGbkResult>>> SUPERORDINATE_DOMAINS =
+        new TupleTag<KV<String, KV<String, CoGbkResult>>>() {};
+
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> EXTERNAL_HOST_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<PendingDeposit> PENDING_DEPOSIT =
+        new TupleTag<PendingDeposit>() {};
+
+    protected static final TupleTag<KV<String, CoGbkResult>> HOST_TO_PENDING_DEPOSIT =
+        new TupleTag<KV<String, CoGbkResult>>() {};
+
+    protected static final TupleTag<Long> REVISION_ID = new TupleTag<Long>() {};
+  }
+
   @Singleton
-  @Component(modules = {CredentialModule.class, ConfigModule.class})
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
   interface RdePipelineComponent {
     RdePipeline rdePipeline();
 

core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java

@@ -31,9 +31,9 @@ public interface RdePipelineOptions extends RegistryPipelineOptions {
   void setValidationMode(String value);
 
   @Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
-  String getGcsBucket();
+  String getRdeStagingBucket();
 
-  void setGcsBucket(String value);
+  void setRdeStagingBucket(String value);
 
   @Description("The Base64-encoded PGP public key to encrypt the deposits.")
   String getStagingKey();

core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java

@@ -25,7 +25,7 @@ import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 
 /**
- * A POJO representing a single subdomain, parsed from a {@code SchemaAndRecord}.
+ * A POJO representing a domain name and associated info, parsed from a {@code SchemaAndRecord}.
  *
  * <p>This is a trivially serializable class that allows Beam to transform the results of a Bigquery
  * query into a standard Java representation, giving us the type guarantees and ease of manipulation
@@ -33,28 +33,31 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
  * function.
  */
 @AutoValue
-public abstract class Subdomain implements Serializable {
+public abstract class DomainNameInfo implements Serializable {
 
   private static final ImmutableList<String> FIELD_NAMES =
       ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");
 
   /** Returns the fully qualified domain name. */
   abstract String domainName();
+
   /** Returns the domain repo ID (the primary key of the domain table). */
   abstract String domainRepoId();
+
   /** Returns the registrar ID of the associated registrar for this domain. */
   abstract String registrarId();
+
   /** Returns the email address of the registrar associated with this domain. */
   abstract String registrarEmailAddress();
 
   /**
-   * Constructs a {@link Subdomain} from an Apache Avro {@code SchemaAndRecord}.
+   * Constructs a {@link DomainNameInfo} from an Apache Avro {@code SchemaAndRecord}.
    *
    * @see <a
    *     href=http://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/generic/GenericData.Record.html>
    *     Apache AVRO GenericRecord</a>
    */
-  static Subdomain parseFromRecord(SchemaAndRecord schemaAndRecord) {
+  static DomainNameInfo parseFromRecord(SchemaAndRecord schemaAndRecord) {
     checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
     GenericRecord record = schemaAndRecord.getRecord();
     return create(
@@ -65,18 +68,15 @@ public abstract class Subdomain implements Serializable {
   }
 
   /**
-   * Creates a concrete {@link Subdomain}.
+   * Creates a concrete {@link DomainNameInfo}.
    *
-   * <p>This should only be used outside this class for testing- instances of {@link Subdomain}
+   * <p>This should only be used outside this class for testing- instances of {@link DomainNameInfo}
    * should otherwise come from {@link #parseFromRecord}.
    */
   @VisibleForTesting
-  static Subdomain create(
-      String domainName,
-      String domainRepoId,
-      String registrarId,
-      String registrarEmailAddress) {
-    return new AutoValue_Subdomain(
+  static DomainNameInfo create(
+      String domainName, String domainRepoId, String registrarId, String registrarEmailAddress) {
+    return new AutoValue_DomainNameInfo(
         domainName, domainRepoId, registrarId, registrarEmailAddress);
   }
 }

core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java

@@ -55,13 +55,14 @@ public class SafeBrowsingTransforms {
       "https://safebrowsing.googleapis.com/v4/threatMatches:find";
 
   /**
-   * {@link DoFn} mapping a {@link Subdomain} to its evaluation report from SafeBrowsing.
+   * {@link DoFn} mapping a {@link DomainNameInfo} to its evaluation report from SafeBrowsing.
    *
    * <p>Refer to the Lookup API documentation for the request/response format and other details.
    *
    * @see <a href=https://developers.google.com/safe-browsing/v4/lookup-api>Lookup API</a>
    */
-  static class EvaluateSafeBrowsingFn extends DoFn<Subdomain, KV<Subdomain, ThreatMatch>> {
+  static class EvaluateSafeBrowsingFn
+      extends DoFn<DomainNameInfo, KV<DomainNameInfo, ThreatMatch>> {
 
     /**
      * Max number of urls we can check in a single query.
@@ -74,10 +75,11 @@ public class SafeBrowsingTransforms {
     private final String apiKey;
 
     /**
-     * Maps a subdomain's {@code fullyQualifiedDomainName} to its corresponding {@link Subdomain} to
-     * facilitate batching SafeBrowsing API requests.
+     * Maps a domain name's {@code fullyQualifiedDomainName} to its corresponding {@link
+     * DomainNameInfo} to facilitate batching SafeBrowsing API requests.
      */
-    private final Map<String, Subdomain> subdomainBuffer = new LinkedHashMap<>(BATCH_SIZE);
+    private final Map<String, DomainNameInfo> domainNameInfoBuffer =
+        new LinkedHashMap<>(BATCH_SIZE);
 
     /**
      * Provides the HTTP client we use to interact with the SafeBrowsing API.
@@ -119,37 +121,38 @@ public class SafeBrowsingTransforms {
       closeableHttpClientSupplier = clientSupplier;
     }
 
-    /** Evaluates any buffered {@link Subdomain} objects upon completing the bundle. */
+    /** Evaluates any buffered {@link DomainNameInfo} objects upon completing the bundle. */
     @FinishBundle
     public void finishBundle(FinishBundleContext context) {
-      if (!subdomainBuffer.isEmpty()) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      if (!domainNameInfoBuffer.isEmpty()) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
         results.forEach((kv) -> context.output(kv, Instant.now(), GlobalWindow.INSTANCE));
       }
     }
 
     /**
-     * Buffers {@link Subdomain} objects until we reach the batch size, then bulk-evaluate the URLs
-     * with the SafeBrowsing API.
+     * Buffers {@link DomainNameInfo} objects until we reach the batch size, then bulk-evaluate the
+     * URLs with the SafeBrowsing API.
      */
     @ProcessElement
     public void processElement(ProcessContext context) {
-      Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.domainName(), subdomain);
-      if (subdomainBuffer.size() >= BATCH_SIZE) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      DomainNameInfo domainNameInfo = context.element();
+      domainNameInfoBuffer.put(domainNameInfo.domainName(), domainNameInfo);
+      if (domainNameInfoBuffer.size() >= BATCH_SIZE) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
         results.forEach(context::output);
       }
     }
 
     /**
-     * Evaluates all {@link Subdomain} objects in the buffer and returns a list of key-value pairs
-     * from {@link Subdomain} to its SafeBrowsing report.
+     * Evaluates all {@link DomainNameInfo} objects in the buffer and returns a list of key-value
+     * pairs from {@link DomainNameInfo} to its SafeBrowsing report.
      *
-     * <p>If a {@link Subdomain} is safe according to the API, it will not emit a report.
+     * <p>If a {@link DomainNameInfo} is safe according to the API, it will not emit a report.
      */
-    private ImmutableSet<KV<Subdomain, ThreatMatch>> evaluateAndFlush() {
-      ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder = new ImmutableSet.Builder<>();
+    private ImmutableSet<KV<DomainNameInfo, ThreatMatch>> evaluateAndFlush() {
+      ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder =
+          new ImmutableSet.Builder<>();
       try {
         URIBuilder uriBuilder = new URIBuilder(SAFE_BROWSING_URL);
         // Add the API key param
@@ -174,7 +177,7 @@ public class SafeBrowsingTransforms {
         throw new RuntimeException("Caught parsing exception, failing pipeline.", e);
       } finally {
         // Flush the buffer
-        subdomainBuffer.clear();
+        domainNameInfoBuffer.clear();
       }
       return resultBuilder.build();
     }
@@ -183,7 +186,7 @@ public class SafeBrowsingTransforms {
     private JSONObject createRequestBody() throws JSONException {
       // Accumulate all domain names to evaluate.
       JSONArray threatArray = new JSONArray();
-      for (String fullyQualifiedDomainName : subdomainBuffer.keySet()) {
+      for (String fullyQualifiedDomainName : domainNameInfoBuffer.keySet()) {
         threatArray.put(new JSONObject().put("url", fullyQualifiedDomainName));
       }
       // Construct the JSON request body
@@ -211,11 +214,11 @@ public class SafeBrowsingTransforms {
      */
     private void processResponse(
         CloseableHttpResponse response,
-        ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder)
+        ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder)
         throws JSONException, IOException {
       int statusCode = response.getStatusLine().getStatusCode();
       if (statusCode != SC_OK) {
-        logger.atWarning().log("Got unexpected status code %s from response", statusCode);
+        logger.atWarning().log("Got unexpected status code %s from response.", statusCode);
       } else {
         // Unpack the response body
         JSONObject responseBody =
@@ -224,18 +227,19 @@ public class SafeBrowsingTransforms {
                     new InputStreamReader(response.getEntity().getContent(), UTF_8)));
         logger.atInfo().log("Got response: %s", responseBody.toString());
         if (responseBody.length() == 0) {
-          logger.atInfo().log("Response was empty, no threats detected");
+          logger.atInfo().log("Response was empty, no threats detected.");
         } else {
-          // Emit all Subdomains with their API results.
+          // Emit all DomainNameInfos with their API results.
           JSONArray threatMatches = responseBody.getJSONArray("matches");
           for (int i = 0; i < threatMatches.length(); i++) {
             JSONObject match = threatMatches.getJSONObject(i);
             String url = match.getJSONObject("threat").getString("url");
-            Subdomain subdomain = subdomainBuffer.get(url);
+            DomainNameInfo domainNameInfo = domainNameInfoBuffer.get(url);
             resultBuilder.add(
                 KV.of(
-                    subdomain,
-                    ThreatMatch.create(match.getString("threatType"), subdomain.domainName())));
+                    domainNameInfo,
+                    ThreatMatch.create(
+                        match.getString("threatType"), domainNameInfo.domainName())));
           }
         }
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -16,6 +16,7 @@ package google.registry.beam.spec11;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
@@ -30,6 +31,7 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.reporting.Spec11ThreatMatch;
 import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
 import google.registry.util.UtilsModule;
@@ -41,6 +43,7 @@ import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -58,7 +61,8 @@ import org.json.JSONObject;
 /**
  * Definition of a Dataflow Flex template, which generates a given month's spec11 report.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=spec11}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
@@ -100,86 +104,106 @@ public class Spec11Pipeline implements Serializable {
 
   void setupPipeline(Pipeline pipeline) {
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
-    PCollection<Subdomain> domains =
+    PCollection<DomainNameInfo> domains =
         options.getDatabase().equals("DATASTORE")
             ? readFromBigQuery(options, pipeline)
             : readFromCloudSql(pipeline);
 
-    PCollection<KV<Subdomain, ThreatMatch>> threatMatches =
+    PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches =
         domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn));
 
     saveToSql(threatMatches, options);
     saveToGcs(threatMatches, options);
   }
 
-  static PCollection<Subdomain> readFromCloudSql(Pipeline pipeline) {
-    Read<Object[], Subdomain> read =
+  static PCollection<DomainNameInfo> readFromCloudSql(Pipeline pipeline) {
+    Read<Object[], KV<String, String>> read =
         RegistryJpaIO.read(
-            "select d, r.emailAddress from Domain d join Registrar r on"
-                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
-                + " and d.deletionTime > now()",
+            "select d.repoId, r.emailAddress from Domain d join Registrar r on"
+                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL' and"
+                + " d.deletionTime > now()",
             false,
             Spec11Pipeline::parseRow);
 
-    return pipeline.apply("Read active domains from Cloud SQL", read);
+    return pipeline
+        .apply("Read active domains from Cloud SQL", read)
+        .apply(
+            "Build DomainNameInfo",
+            ParDo.of(
+                new DoFn<KV<String, String>, DomainNameInfo>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, String> input, OutputReceiver<DomainNameInfo> output) {
+                    DomainBase domainBase =
+                        jpaTm()
+                            .transact(
+                                () ->
+                                    jpaTm()
+                                        .loadByKey(
+                                            VKey.createSql(DomainBase.class, input.getKey())));
+                    String emailAddress = input.getValue();
+                    if (emailAddress == null) {
+                      emailAddress = "";
+                    }
+                    DomainNameInfo domainNameInfo =
+                        DomainNameInfo.create(
+                            domainBase.getDomainName(),
+                            domainBase.getRepoId(),
+                            domainBase.getCurrentSponsorRegistrarId(),
+                            emailAddress);
+                    output.output(domainNameInfo);
+                  }
+                }));
   }
 
-  static PCollection<Subdomain> readFromBigQuery(Spec11PipelineOptions options, Pipeline pipeline) {
+  static PCollection<DomainNameInfo> readFromBigQuery(
+      Spec11PipelineOptions options, Pipeline pipeline) {
     return pipeline.apply(
         "Read active domains from BigQuery",
-        BigQueryIO.read(Subdomain::parseFromRecord)
+        BigQueryIO.read(DomainNameInfo::parseFromRecord)
             .fromQuery(
-                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
+                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "domain_name_infos.sql"))
                     .put("PROJECT_ID", options.getProject())
                     .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
                     .put("REGISTRAR_TABLE", "Registrar")
                     .put("DOMAIN_BASE_TABLE", "DomainBase")
                     .build())
-            .withCoder(SerializableCoder.of(Subdomain.class))
+            .withCoder(SerializableCoder.of(DomainNameInfo.class))
             .usingStandardSql()
             .withoutValidation()
             .withTemplateCompatibility());
   }
 
-  private static Subdomain parseRow(Object[] row) {
-    DomainBase domainBase = (DomainBase) row[0];
-    String emailAddress = (String) row[1];
-    if (emailAddress == null) {
-      emailAddress = "";
-    }
-    return Subdomain.create(
-        domainBase.getDomainName(),
-        domainBase.getRepoId(),
-        domainBase.getCurrentSponsorClientId(),
-        emailAddress);
+  private static KV<String, String> parseRow(Object[] row) {
+    return KV.of((String) row[0], (String) row[1]);
   }
 
   static void saveToSql(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
     String transformId = "Spec11 Threat Matches";
     LocalDate date = LocalDate.parse(options.getDate(), ISODateTimeFormat.date());
     threatMatches.apply(
         "Write to Sql: " + transformId,
-        RegistryJpaIO.<KV<Subdomain, ThreatMatch>>write()
+        RegistryJpaIO.<KV<DomainNameInfo, ThreatMatch>>write()
             .withName(transformId)
             .withBatchSize(options.getSqlWriteBatchSize())
             .withShards(options.getSqlWriteShards())
             .withJpaConverter(
                 (kv) -> {
-                  Subdomain subdomain = kv.getKey();
+                  DomainNameInfo domainNameInfo = kv.getKey();
                   return new Spec11ThreatMatch.Builder()
                       .setThreatTypes(
                           ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
                       .setCheckDate(date)
-                      .setDomainName(subdomain.domainName())
-                      .setDomainRepoId(subdomain.domainRepoId())
-                      .setRegistrarId(subdomain.registrarId())
+                      .setDomainName(domainNameInfo.domainName())
+                      .setDomainRepoId(domainNameInfo.domainRepoId())
+                      .setRegistrarId(domainNameInfo.registrarId())
                       .build();
                 }));
   }
 
   static void saveToGcs(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
     threatMatches
         .apply(
             "Map registrar ID to email/ThreatMatch pair",
@@ -187,7 +211,7 @@ public class Spec11Pipeline implements Serializable {
                     TypeDescriptors.kvs(
                         TypeDescriptors.strings(), TypeDescriptor.of(EmailAndThreatMatch.class)))
                 .via(
-                    (KV<Subdomain, ThreatMatch> kv) ->
+                    (KV<DomainNameInfo, ThreatMatch> kv) ->
                         KV.of(
                             kv.getKey().registrarId(),
                             EmailAndThreatMatch.create(
@@ -198,15 +222,15 @@ public class Spec11Pipeline implements Serializable {
             MapElements.into(TypeDescriptors.strings())
                 .via(
                     (KV<String, Iterable<EmailAndThreatMatch>> kv) -> {
-                      String clientId = kv.getKey();
+                      String registrarId = kv.getKey();
                       checkArgument(
                           kv.getValue().iterator().hasNext(),
                           String.format(
-                              "Registrar with ID %s had no corresponding threats", clientId));
+                              "Registrar with ID %s had no corresponding threats", registrarId));
                       String email = kv.getValue().iterator().next().email();
                       JSONObject output = new JSONObject();
                       try {
-                        output.put(REGISTRAR_CLIENT_ID_FIELD, clientId);
+                        output.put(REGISTRAR_CLIENT_ID_FIELD, registrarId);
                         output.put(REGISTRAR_EMAIL_FIELD, email);
                         JSONArray threatMatchArray = new JSONArray();
                         for (EmailAndThreatMatch emailAndThreatMatch : kv.getValue()) {
@@ -230,7 +254,7 @@ public class Spec11Pipeline implements Serializable {
                         options.getReportingBucketUrl(),
                         getSpec11ReportFilePath(LocalDate.parse(options.getDate()))))
                 .withoutSharding()
-                .withHeader("Map from registrar email / name to detected subdomain threats:"));
+                .withHeader("Map from registrar email / name to detected domain name threats:"));
   }
 
   public static void main(String[] args) {

core/src/main/java/google/registry/bigquery/BigqueryConnection.java

@@ -632,7 +632,7 @@ public class BigqueryConnection implements AutoCloseable {
   private static String summarizeCompletedJob(Job job) {
     JobStatistics stats = job.getStatistics();
     return String.format(
-        "Job took %,.3f seconds after a %,.3f second delay and processed %,d bytes (%s)",
+        "Job took %,.3f seconds after a %,.3f second delay and processed %,d bytes (%s).",
         (stats.getEndTime() - stats.getStartTime()) / 1000.0,
         (stats.getStartTime() - stats.getCreationTime()) / 1000.0,
         stats.getTotalBytesProcessed(),
@@ -706,17 +706,17 @@ public class BigqueryConnection implements AutoCloseable {
   }
 
   /**
-   * Helper that creates a dataset with this name if it doesn't already exist, and returns true
-   * if creation took place.
+   * Helper that creates a dataset with this name if it doesn't already exist, and returns true if
+   * creation took place.
    */
-  public boolean createDatasetIfNeeded(String datasetName) throws IOException {
+  private boolean createDatasetIfNeeded(String datasetName) throws IOException {
     if (!checkDatasetExists(datasetName)) {
       bigquery.datasets()
           .insert(getProjectId(), new Dataset().setDatasetReference(new DatasetReference()
               .setProjectId(getProjectId())
               .setDatasetId(datasetName)))
           .execute();
-      logger.atInfo().log("Created dataset: %s:%s\n", getProjectId(), datasetName);
+      logger.atInfo().log("Created dataset: %s: %s.", getProjectId(), datasetName);
       return true;
     }
     return false;
@@ -732,9 +732,8 @@ public class BigqueryConnection implements AutoCloseable {
                   .setDefaultDataset(getDataset())
                   .setDestinationTable(table))));
     } catch (BigqueryJobFailureException e) {
-      if (e.getReason().equals("duplicate")) {
-        // Table already exists.
-      } else {
+      if (!e.getReason().equals("duplicate")) {
+        // Throw if it failed for any reason other than table already existing.
         throw e;
       }
     }

core/src/main/java/google/registry/bigquery/CheckedBigquery.java

@@ -116,7 +116,7 @@ public class CheckedBigquery {
           .setTableReference(table))
           .execute();
       logger.atInfo().log(
-          "Created BigQuery table %s:%s.%s",
+          "Created BigQuery table %s:%s.%s.",
           table.getProjectId(), table.getDatasetId(), table.getTableId());
     } catch (IOException e) {
       // Swallow errors about a table that exists, and throw any other ones.

core/src/main/java/google/registry/config/CloudTasksUtilsModule.java

@@ -0,0 +1,79 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.config;
+
+import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.cloud.tasks.v2.CloudTasksClient;
+import com.google.cloud.tasks.v2.CloudTasksSettings;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.CloudTasksUtils;
+import google.registry.util.CloudTasksUtils.GcpCloudTasksClient;
+import google.registry.util.CloudTasksUtils.SerializableCloudTasksClient;
+import google.registry.util.GoogleCredentialsBundle;
+import google.registry.util.Retrier;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.function.Supplier;
+import javax.inject.Singleton;
+
+/**
+ * A {@link Module} that provides {@link CloudTasksUtils}.
+ *
+ * <p>The class itself cannot be annotated as {@code Inject} because its requested dependencies use
+ * the {@link Config} qualifier which is not available in the {@code util} package.
+ */
+@Module
+public abstract class CloudTasksUtilsModule {
+
+  @Singleton
+  @Provides
+  public static CloudTasksUtils provideCloudTasksUtils(
+      @Config("projectId") String projectId,
+      @Config("locationId") String locationId,
+      SerializableCloudTasksClient client,
+      Retrier retrier) {
+    return new CloudTasksUtils(retrier, projectId, locationId, client);
+  }
+
+  // Provides a supplier instead of using a Dagger @Provider because the latter is not serializable.
+  @Provides
+  public static Supplier<CloudTasksClient> provideCloudTasksClientSupplier(
+      @DefaultCredential GoogleCredentialsBundle credentials) {
+    return (Supplier<CloudTasksClient> & Serializable)
+        () -> {
+          CloudTasksClient client;
+          try {
+            client =
+                CloudTasksClient.create(
+                    CloudTasksSettings.newBuilder()
+                        .setCredentialsProvider(
+                            FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
+                        .build());
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+          return client;
+        };
+  }
+
+  @Provides
+  public static SerializableCloudTasksClient provideSerializableCloudTasksClient(
+      final Supplier<CloudTasksClient> clientSupplier) {
+    return new GcpCloudTasksClient(clientSupplier);
+  }
+}

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.util.TaskQueueUtils;
 import google.registry.util.YamlUtils;
 import java.lang.annotation.Documented;
@@ -105,6 +106,12 @@ public final class RegistryConfig {
       return config.appEngine.projectId;
     }
 
+    @Provides
+    @Config("locationId")
+    public static String provideLocationId(RegistryConfigSettings config) {
+      return config.appEngine.locationId;
+    }
+
     /**
      * The filename of the logo to be displayed in the header of the registrar console.
      *
@@ -386,19 +393,26 @@ public final class RegistryConfig {
 
     @Provides
     @Config("cloudSqlJdbcUrl")
-    public static String providesCloudSqlJdbcUrl(RegistryConfigSettings config) {
+    public static String provideCloudSqlJdbcUrl(RegistryConfigSettings config) {
       return config.cloudSql.jdbcUrl;
     }
 
     @Provides
     @Config("cloudSqlInstanceConnectionName")
-    public static String providesCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
+    public static String provideCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
       return config.cloudSql.instanceConnectionName;
     }
 
+    @Provides
+    @Config("cloudSqlReplicaInstanceConnectionName")
+    public static Optional<String> provideCloudSqlReplicaInstanceConnectionName(
+        RegistryConfigSettings config) {
+      return Optional.ofNullable(config.cloudSql.replicaInstanceConnectionName);
+    }
+
     @Provides
     @Config("cloudSqlDbInstanceName")
-    public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
+    public static String provideCloudSqlDbInstance(RegistryConfigSettings config) {
       // Format of instanceConnectionName: project-id:region:instance-name
       int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
       return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);
@@ -551,11 +565,23 @@ public final class RegistryConfig {
       return config.registryPolicy.requireSslCertificates;
     }
 
+    /**
+     * Returns the GCE machine type that a CPU-demanding pipeline should use.
+     *
+     * @see google.registry.beam.rde.RdePipeline
+     */
+    @Provides
+    @Config("highPerformanceMachineType")
+    public static String provideHighPerformanceMachineType(RegistryConfigSettings config) {
+      return config.beam.highPerformanceMachineType;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
      * @see google.registry.beam.invoicing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
+     * @see google.registry.beam.invoicing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")
@@ -1132,8 +1158,8 @@ public final class RegistryConfig {
     }
 
     /**
-     * Returns the clientId of the registrar that admins are automatically logged in as if they
-     * aren't otherwise associated with one.
+     * Returns the ID of the registrar that admins are automatically logged in as if they aren't
+     * otherwise associated with one.
      */
     @Provides
     @Config("registryAdminClientId")
@@ -1300,6 +1326,18 @@ public final class RegistryConfig {
     public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
       return ImmutableSet.copyOf(config.sslCertificateValidation.allowedEcdsaCurves);
     }
+
+    @Provides
+    @Config("minMonthsBeforeWipeOut")
+    public static int provideMinMonthsBeforeWipeOut(RegistryConfigSettings config) {
+      return config.contactHistory.minMonthsBeforeWipeOut;
+    }
+
+    @Provides
+    @Config("wipeOutQueryBatchSize")
+    public static int provideWipeOutQueryBatchSize(RegistryConfigSettings config) {
+      return config.contactHistory.wipeOutQueryBatchSize;
+    }
   }
 
   /** Returns the App Engine project ID, which is based off the environment name. */
@@ -1494,6 +1532,31 @@ public final class RegistryConfig {
     return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
   }
 
+  /**
+   * JDBC-specific: driver default batch size is 0, which means that every INSERT statement will be
+   * sent to the database individually. Batching allows us to group together multiple inserts into
+   * one single INSERT statement which can dramatically increase speed in situations with many
+   * inserts.
+   *
+   * <p>Hibernate docs, i.e.
+   * https://docs.jboss.org/hibernate/orm/5.6/userguide/html_single/Hibernate_User_Guide.html,
+   * recommend between 10 and 50.
+   */
+  public static String getHibernateJdbcBatchSize() {
+    return CONFIG_SETTINGS.get().hibernate.jdbcBatchSize;
+  }
+
+  /**
+   * Returns the JDBC fetch size.
+   *
+   * <p>Postgresql-specific: driver default fetch size is 0, which disables streaming result sets.
+   * Here we set a small default geared toward Nomulus server transactions. Large queries can
+   * override the defaults using {@link JpaTransactionManager#setQueryFetchSize}.
+   */
+  public static String getHibernateJdbcFetchSize() {
+    return CONFIG_SETTINGS.get().hibernate.jdbcFetchSize;
+  }
+
   /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
   public static String getContactAndHostRoidSuffix() {
     return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -41,10 +41,12 @@ public class RegistryConfigSettings {
   public Keyring keyring;
   public RegistryTool registryTool;
   public SslCertificateValidation sslCertificateValidation;
+  public ContactHistory contactHistory;
 
   /** Configuration options that apply to the entire App Engine project. */
   public static class AppEngine {
     public String projectId;
+    public String locationId;
     public boolean isLocal;
     public String defaultServiceUrl;
     public String backendServiceUrl;
@@ -118,6 +120,8 @@ public class RegistryConfigSettings {
     public String hikariMinimumIdle;
     public String hikariMaximumPoolSize;
     public String hikariIdleTimeout;
+    public String jdbcBatchSize;
+    public String jdbcFetchSize;
   }
 
   /** Configuration for Cloud SQL. */
@@ -126,11 +130,13 @@ public class RegistryConfigSettings {
     // TODO(05012021): remove username field after it is removed from all yaml files.
     public String username;
     public String instanceConnectionName;
+    public String replicaInstanceConnectionName;
   }
 
   /** Configuration for Apache Beam (Cloud Dataflow). */
   public static class Beam {
     public String defaultJobRegion;
+    public String highPerformanceMachineType;
     public String stagingBucketUrl;
   }
 
@@ -233,4 +239,10 @@ public class RegistryConfigSettings {
     public String expirationWarningEmailBodyText;
     public String expirationWarningEmailSubjectText;
   }
+
+  /** Configuration for contact history. */
+  public static class ContactHistory {
+    public int minMonthsBeforeWipeOut;
+    public int wipeOutQueryBatchSize;
+  }
 }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -8,6 +8,10 @@
 appEngine:
   # Globally unique App Engine project ID
   projectId: registry-project-id
+  # Location of the App engine project, note that us-central1 and europe-west1 are special in that
+  # they are used without the trailing number in App Engine commands and Google Cloud Console.
+  # See: https://cloud.google.com/appengine/docs/locations
+  locationId: registry-location-id
 
   # whether to use local/test credentials when connecting to the servers
   isLocal: true
@@ -217,6 +221,17 @@ hibernate:
   hikariMinimumIdle: 1
   hikariMaximumPoolSize: 10
   hikariIdleTimeout: 300000
+  # The batch size is basically the number of insertions / updates in a single
+  # transaction that will be batched together into one INSERT/UPDATE statement.
+  # A larger batch size is useful when inserting or updating many entities in a
+  # single transaction. Hibernate docs
+  # (https://docs.jboss.org/hibernate/orm/5.6/userguide/html_single/Hibernate_User_Guide.html)
+  # recommend between 10 and 50.
+  jdbcBatchSize: 50
+  # The fetch size is the number of entities retrieved at a time from the
+  # database cursor. Here we set a small default geared toward Nomulus server
+  # transactions. Large queries can override the defaults on a per-query basis.
+  jdbcFetchSize: 20
 
 cloudSql:
   # jdbc url for the Cloud SQL database.
@@ -227,6 +242,10 @@ cloudSql:
   jdbcUrl: jdbc:postgresql://localhost
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
+  # If non-null, we will use this instance for certain read-only actions or
+  # pipelines, e.g. RDE, in order to offload some work from the primary
+  # instance. Expect any write actions on this instance to fail.
+  replicaInstanceConnectionName: null
 
 cloudDns:
   # Set both properties to null in Production.
@@ -415,6 +434,14 @@ misc:
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
   defaultJobRegion: us-east1
+  # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
+  # to check the VM CPU quota for the job region. In a massively parallel
+  # pipeline this quota can be easily reached and needs to be raised, otherwise
+  # the job will run very slowly. Also note that there is a separate quota for
+  # external IPv4 address in a region, which means that machine type with higher
+  # core count per machine may be preferable in order to preserve IP addresses.
+  # See: https://cloud.google.com/compute/quotas#cpu_quota
+  highPerformanceMachineType: n2-standard-4
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:
@@ -438,6 +465,13 @@ registryTool:
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
 
+# Configuration options for handling contact history.
+contactHistory:
+  # The number of months that a ContactHistory entity should be stored in the database.
+  minMonthsBeforeWipeOut: 18
+  # The batch size for querying ContactHistory table in the database.
+  wipeOutQueryBatchSize: 500
+
 # Configuration options for checking SSL certificates.
 sslCertificateValidation:
   # A map specifying the maximum amount of days the certificate can be valid.
@@ -452,12 +486,36 @@ sslCertificateValidation:
   # The minimum number of days between two successive expiring notification emails.
   expirationWarningIntervalDays: 15
   # Text for expiring certificate notification email subject.
-  expirationWarningEmailSubjectText: Certificate Expring Within 30 Days.
+  expirationWarningEmailSubjectText: "[Important] Expiring SSL certificate for Google Registry EPP connection"
   # Text for expiring certificate notification email body that accepts 3 parameters:
   # registrar name, certificate type, and expiration date, respectively.
   expirationWarningEmailBodyText: |
-    Hello Registrar %s,
-       The %s certificate is expiring on %s.
+    Dear %1$s,
+
+    We would like to inform you that your %2$s SSL certificate will expire at %3$s. Please take note that using expired certificates will prevent successful Registry login.
+
+    Kindly update your production account certificate within the support console using the following steps:
+
+      1. Navigate to support.registry.google and login using your %4$s@registry.google credentials.
+          * If this is your first time logging in, you will be prompted to reset your password, so please keep your new password safe.
+          * If you are already logged in with some other Google account(s) but not your %4$s@registry.google account, you need to click on
+          “Add Account” and login using your %4$s@registry.google credentials.
+      2. Select “Settings > Security” from the left navigation bar.
+      3. Click “Edit” on the top left corner.
+      4. Enter your full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----) in the box.
+      5. Click “Save”. If there are validation issues with the form, you will be prompted to fix them and click “Save” again.
+
+    A failover SSL certificate can also be added in order to prevent connection issues once your main certificate expires. Connecting with either of the certificates will work with our production EPP server.
+
+    Further information about our EPP connection requirements can be found in section 9.2 in the updated Technical Guide in your Google Drive folder.
+
+    Note that account certificate changes take a few minutes to become effective and that the existing connections will remain unaffected by the change.
+
+    If you also would like to update your OT&E account certificate, please send an email from your primary or technical contact to registry-support@google.com and include the full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).
+
+    Regards,
+    Google Registry
+
   # The minimum number of bits an RSA key must contain.
   minimumRsaKeyLength: 2048
   # The ECDSA curves that are allowed for public keys.

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -14,18 +14,15 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
-import java.time.Duration;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import javax.inject.Inject;
 
 /** Action for fanning out cron tasks for each commit log bucket. */
@@ -38,25 +35,27 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  private static final Random random = new Random();
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
-  @Inject TaskQueueUtils taskQueueUtils;
   @Inject @Parameter("endpoint") String endpoint;
   @Inject @Parameter("queue") String queue;
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
+
+
   @Override
   public void run() {
-    Queue taskQueue = getQueue(queue);
     for (int bucketId : CommitLogBucket.getBucketIds()) {
-      long delay =
-          jitterSeconds.map(i -> random.nextInt((int) Duration.ofSeconds(i).toMillis())).orElse(0);
-      TaskOptions taskOptions =
-          TaskOptions.Builder.withUrl(endpoint)
-              .param(BUCKET_PARAM, Integer.toString(bucketId))
-              .countdownMillis(delay);
-      taskQueueUtils.enqueue(taskQueue, taskOptions);
+      cloudTasksUtils.enqueue(
+          queue,
+          CloudTasksUtils.createPostTask(
+              endpoint,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
+              clock,
+              jitterSeconds));
     }
   }
 }

core/src/main/java/google/registry/cron/TldFanoutAction.java

@@ -14,14 +14,10 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Predicates.in;
 import static com.google.common.base.Predicates.not;
-import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.collect.Iterables.getFirst;
 import static com.google.common.collect.Multimaps.filterKeys;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.cron.CronModule.ENDPOINT_PARAM;
@@ -34,25 +30,24 @@ import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
 import static google.registry.model.tld.Registries.getTldsOfType;
 import static google.registry.model.tld.Registry.TldType.REAL;
 import static google.registry.model.tld.Registry.TldType.TEST;
-import static java.util.concurrent.TimeUnit.SECONDS;
 
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.cloud.tasks.v2.Task;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.ParameterMap;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import java.util.stream.Stream;
 import javax.inject.Inject;
 
@@ -101,11 +96,10 @@ public final class TldFanoutAction implements Runnable {
           EXCLUDE_PARAM,
           JITTER_SECONDS_PARAM);
 
-  private static final Random random = new Random();
-
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
   @Inject Response response;
   @Inject @Parameter(ENDPOINT_PARAM) String endpoint;
   @Inject @Parameter(QUEUE_PARAM) String queue;
@@ -115,7 +109,9 @@ public final class TldFanoutAction implements Runnable {
   @Inject @Parameter(EXCLUDE_PARAM) ImmutableSet<String> excludes;
   @Inject @Parameter(JITTER_SECONDS_PARAM) Optional<Integer> jitterSeconds;
   @Inject @ParameterMap ImmutableListMultimap<String, String> params;
-  @Inject TldFanoutAction() {}
+
+  @Inject
+  TldFanoutAction() {}
 
   @Override
   public void run() {
@@ -126,8 +122,7 @@ public final class TldFanoutAction implements Runnable {
         runInEmpty || forEachTestTld || forEachRealTld,
         "At least one of runInEmpty, forEachTestTld, forEachRealTld must be given");
     checkArgument(
-        !(runInEmpty && !excludes.isEmpty()),
-        "Can't specify 'exclude' with 'runInEmpty'");
+        !(runInEmpty && !excludes.isEmpty()), "Can't specify 'exclude' with 'runInEmpty'");
     ImmutableSet<String> tlds =
         runInEmpty
             ? ImmutableSet.of("")
@@ -137,42 +132,34 @@ public final class TldFanoutAction implements Runnable {
                 .filter(not(in(excludes)))
                 .collect(toImmutableSet());
     Multimap<String, String> flowThruParams = filterKeys(params, not(in(CONTROL_PARAMS)));
-    Queue taskQueue = getQueue(queue);
     StringBuilder outputPayload =
         new StringBuilder(
             String.format("OK: Launched the following %d tasks in queue %s\n", tlds.size(), queue));
-    logger.atInfo().log("Launching %d tasks in queue %s", tlds.size(), queue);
+    logger.atInfo().log("Launching %d tasks in queue %s.", tlds.size(), queue);
     if (tlds.isEmpty()) {
       logger.atWarning().log("No TLDs to fan-out!");
     }
     for (String tld : tlds) {
-      TaskOptions taskOptions = createTaskOptions(tld, flowThruParams);
-      TaskHandle taskHandle = taskQueueUtils.enqueue(taskQueue, taskOptions);
+      Task task = createTask(tld, flowThruParams);
+      Task createdTask = cloudTasksUtils.enqueue(queue, task);
       outputPayload.append(
           String.format(
               "- Task: '%s', tld: '%s', endpoint: '%s'\n",
-              taskHandle.getName(), tld, taskOptions.getUrl()));
+              createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri()));
       logger.atInfo().log(
-          "Task: '%s', tld: '%s', endpoint: '%s'", taskHandle.getName(), tld, taskOptions.getUrl());
+          "Task: '%s', tld: '%s', endpoint: '%s'.",
+          createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri());
     }
     response.setContentType(PLAIN_TEXT_UTF_8);
     response.setPayload(outputPayload.toString());
   }
 
-  private TaskOptions createTaskOptions(String tld, Multimap<String, String> params) {
-    TaskOptions options =
-        withUrl(endpoint)
-            .countdownMillis(
-                jitterSeconds
-                    .map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
-                    .orElse(0));
+  private Task createTask(String tld, Multimap<String, String> params) {
     if (!tld.isEmpty()) {
-      options.param(RequestParameters.PARAM_TLD, tld);
+      params = ArrayListMultimap.create(params);
+      params.put(RequestParameters.PARAM_TLD, tld);
     }
-    for (String param : params.keySet()) {
-      // TaskOptions.param() does not accept null values.
-      options.param(param, nullToEmpty(getFirst(params.get(param), null)));
-    }
-    return options;
+    return CloudTasksUtils.createPostTask(
+        endpoint, Service.BACKEND.toString(), params, clock, jitterSeconds);
   }
 }

core/src/main/java/google/registry/dns/DnsQueue.java

@@ -107,7 +107,7 @@ public class DnsQueue {
   private TaskHandle addToQueue(
       TargetType targetType, String targetName, String tld, Duration countdown) {
     logger.atInfo().log(
-        "Adding task type=%s, target=%s, tld=%s to pull queue %s (%d tasks currently on queue)",
+        "Adding task type=%s, target=%s, tld=%s to pull queue %s (%d tasks currently on queue).",
         targetType, targetName, tld, DNS_PULL_QUEUE_NAME, queue.fetchStatistics().getNumTasks());
     return queue.add(
         TaskOptions.Builder.withDefaults()
@@ -166,7 +166,7 @@ public class DnsQueue {
           "There are %d tasks in the DNS queue '%s'.", numTasks, DNS_PULL_QUEUE_NAME);
       return queue.leaseTasks(leaseDuration.getMillis(), MILLISECONDS, leaseTasksBatchSize);
     } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed leasing tasks too fast");
+      logger.atSevere().withCause(e).log("Failed leasing tasks too fast.");
       return ImmutableList.of();
     }
   }
@@ -176,7 +176,7 @@ public class DnsQueue {
     try {
       queue.deleteTask(tasks);
     } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed deleting tasks too fast");
+      logger.atSevere().withCause(e).log("Failed deleting tasks too fast.");
     }
   }
 }

core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java

@@ -104,7 +104,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
         new Duration(enqueuedTime, now));
     logger.atInfo().log(
         "publishDnsWriter latency statistics: TLD: %s, dnsWriter: %s, actionStatus: %s, "
-            + "numItems: %d, timeSinceCreation: %s, timeInQueue: %s",
+            + "numItems: %d, timeSinceCreation: %s, timeInQueue: %s.",
         tld,
         dnsWriter,
         status,
@@ -144,7 +144,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
 
   /** Adds all the domains and hosts in the batch back to the queue to be processed later. */
   private void requeueBatch() {
-    logger.atInfo().log("Requeueing batch for retry");
+    logger.atInfo().log("Requeueing batch for retry.");
     for (String domain : nullToEmpty(domains)) {
       dnsQueue.addDomainRefreshTask(domain);
     }
@@ -158,14 +158,14 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     // LockIndex should always be within [1, numPublishLocks]
     if (lockIndex > numPublishLocks || lockIndex <= 0) {
       logger.atSevere().log(
-          "Lock index should be within [1,%d], got %d instead", numPublishLocks, lockIndex);
+          "Lock index should be within [1,%d], got %d instead.", numPublishLocks, lockIndex);
       return false;
     }
     // Check if the Registry object's num locks has changed since this task was batched
     int registryNumPublishLocks = Registry.get(tld).getNumDnsPublishLocks();
     if (registryNumPublishLocks != numPublishLocks) {
       logger.atWarning().log(
-          "Registry numDnsPublishLocks %d out of sync with parameter %d",
+          "Registry numDnsPublishLocks %d out of sync with parameter %d.",
           registryNumPublishLocks, numPublishLocks);
       return false;
     }
@@ -179,7 +179,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     DnsWriter writer = dnsWriterProxy.getByClassNameForTld(dnsWriter, tld);
 
     if (writer == null) {
-      logger.atWarning().log("Couldn't get writer %s for TLD %s", dnsWriter, tld);
+      logger.atWarning().log("Couldn't get writer %s for TLD %s.", dnsWriter, tld);
       recordActionResult(ActionStatus.BAD_WRITER);
       requeueBatch();
       return;
@@ -190,11 +190,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     for (String domain : nullToEmpty(domains)) {
       if (!DomainNameUtils.isUnder(
           InternetDomainName.from(domain), InternetDomainName.from(tld))) {
-        logger.atSevere().log("%s: skipping domain %s not under tld", tld, domain);
+        logger.atSevere().log("%s: skipping domain %s not under TLD.", tld, domain);
         domainsRejected += 1;
       } else {
         writer.publishDomain(domain);
-        logger.atInfo().log("%s: published domain %s", tld, domain);
+        logger.atInfo().log("%s: published domain %s.", tld, domain);
         domainsPublished += 1;
       }
     }
@@ -206,11 +206,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     for (String host : nullToEmpty(hosts)) {
       if (!DomainNameUtils.isUnder(
           InternetDomainName.from(host), InternetDomainName.from(tld))) {
-        logger.atSevere().log("%s: skipping host %s not under tld", tld, host);
+        logger.atSevere().log("%s: skipping host %s not under TLD.", tld, host);
         hostsRejected += 1;
       } else {
         writer.publishHost(host);
-        logger.atInfo().log("%s: published host %s", tld, host);
+        logger.atInfo().log("%s: published host %s.", tld, host);
         hostsPublished += 1;
       }
     }
@@ -233,7 +233,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
           tld, dnsWriter, commitStatus, duration, domainsPublished, hostsPublished);
       logger.atInfo().log(
           "writer.commit() statistics: TLD: %s, dnsWriter: %s, commitStatus: %s, duration: %s, "
-              + "domainsPublished: %d, domainsRejected: %d, hostsPublished: %d, hostsRejected: %d",
+              + "domainsPublished: %d, domainsRejected: %d, hostsPublished: %d, hostsRejected: %d.",
           tld,
           dnsWriter,
           commitStatus,

core/src/main/java/google/registry/dns/writer/DnsWriter.java

@@ -31,12 +31,12 @@ package google.registry.dns.writer;
 public interface DnsWriter {
 
   /**
-   * Loads {@code domainName} from Datastore and publishes its NS/DS records to the DNS server.
+   * Loads {@code domainName} from the database and publishes its NS/DS records to the DNS server.
    * Replaces existing records for the exact name supplied with an NS record for each name server
    * and a DS record for each delegation signer stored in the registry for the supplied domain name.
    * If the domain is deleted or is in a "non-publish" state then any existing records are deleted.
    *
-   * This must NOT actually perform any action, instead it should stage the action so that it's
+   * <p>This must NOT actually perform any action, instead it should stage the action so that it's
    * performed when {@link #commit()} is called.
    *
    * @param domainName the fully qualified domain name, with no trailing dot
@@ -44,14 +44,14 @@ public interface DnsWriter {
   void publishDomain(String domainName);
 
   /**
-   * Loads {@code hostName} from Datastore and publishes its A/AAAA glue records to the DNS server,
-   * if it is used as an in-bailiwick nameserver. Orphaned glue records are prohibited. Replaces
-   * existing records for the exact name supplied, with an A or AAAA record (as appropriate) for
-   * each address stored in the registry, for the supplied host name. If the host is deleted then
-   * the existing records are deleted. Assumes that this method will only be called for in-bailiwick
-   * hosts. The registry does not have addresses for other hosts.
+   * Loads {@code hostName} from the database and publishes its A/AAAA glue records to the DNS
+   * server, if it is used as an in-bailiwick nameserver. Orphaned glue records are prohibited.
+   * Replaces existing records for the exact name supplied, with an A or AAAA record (as
+   * appropriate) for each address stored in the registry, for the supplied host name. If the host
+   * is deleted then the existing records are deleted. Assumes that this method will only be called
+   * for in-bailiwick hosts. The registry does not have addresses for other hosts.
    *
-   * This must NOT actually perform any action, instead it should stage the action so that it's
+   * <p>This must NOT actually perform any action, instead it should stage the action so that it's
    * performed when {@link #commit()} is called.
    *
    * @param hostName the fully qualified host name, with no trailing dot

core/src/main/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java

@@ -180,11 +180,11 @@ public class CloudDnsWriter extends BaseDnsWriter {
 
     desiredRecords.put(absoluteDomainName, domainRecords.build());
     logger.atFine().log(
-        "Will write %d records for domain %s", domainRecords.build().size(), absoluteDomainName);
+        "Will write %d records for domain '%s'.", domainRecords.build().size(), absoluteDomainName);
   }
 
   private void publishSubordinateHost(String hostName) {
-    logger.atInfo().log("Publishing glue records for %s", hostName);
+    logger.atInfo().log("Publishing glue records for host '%s'.", hostName);
     // Canonicalize name
     String absoluteHostName = getAbsoluteHostName(hostName);
 
@@ -250,7 +250,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
 
     // Host not managed by our registry, no need to update DNS.
     if (!tld.isPresent()) {
-      logger.atSevere().log("publishHost called for invalid host %s", hostName);
+      logger.atSevere().log("publishHost called for invalid host '%s'.", hostName);
       return;
     }
 
@@ -273,7 +273,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
     ImmutableMap<String, ImmutableSet<ResourceRecordSet>> desiredRecordsCopy =
         ImmutableMap.copyOf(desiredRecords);
     retrier.callWithRetry(() -> mutateZone(desiredRecordsCopy), ZoneStateException.class);
-    logger.atInfo().log("Wrote to Cloud DNS");
+    logger.atInfo().log("Wrote to Cloud DNS.");
   }
 
   /** Returns the glue records for in-bailiwick nameservers for the given domain+records. */
@@ -329,7 +329,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
    */
   private Map<String, List<ResourceRecordSet>> getResourceRecordsForDomains(
       Set<String> domainNames) {
-    logger.atFine().log("Fetching records for %s", domainNames);
+    logger.atFine().log("Fetching records for domain '%s'.", domainNames);
     // As per Concurrent.transform() - if numThreads or domainNames.size() < 2, it will not use
     // threading.
     return ImmutableMap.copyOf(
@@ -381,11 +381,11 @@ public class CloudDnsWriter extends BaseDnsWriter {
     ImmutableSet<ResourceRecordSet> intersection =
         Sets.intersection(additions, deletions).immutableCopy();
     logger.atInfo().log(
-        "There are %d common items out of the %d items in 'additions' and %d items in 'deletions'",
+        "There are %d common items out of the %d items in 'additions' and %d items in 'deletions'.",
         intersection.size(), additions.size(), deletions.size());
     // Exit early if we have nothing to update - dnsConnection doesn't work on empty changes
     if (additions.equals(deletions)) {
-      logger.atInfo().log("Returning early because additions is the same as deletions");
+      logger.atInfo().log("Returning early because additions are the same as deletions.");
       return;
     }
     Change change =

core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml

@@ -208,6 +208,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
     <description>

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -157,6 +157,12 @@
     <url-pattern>/_dr/cron/readDnsQueue</url-pattern>
   </servlet-mapping>
 
+  <!-- Replicates SQL transactions to Datastore during the Registry 3.0 migration. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/cron/replicateToDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Publishes DNS updates. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>
@@ -355,6 +361,12 @@
     <url-pattern>/_dr/task/deleteExpiredDomains</url-pattern>
   </servlet-mapping>
 
+  <!-- Background action to send notification emails to registrars with expiring certificate. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/sendExpiringCertificateNotificationEmail</url-pattern>
+  </servlet-mapping>
+
   <!-- Mapreduce to import contacts from escrow file -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>
@@ -385,6 +397,13 @@
     <url-pattern>/_dr/task/relockDomain</url-pattern>
   </servlet-mapping>
 
+  <!-- Background action to wipe out PII fields of ContactHistory entities that
+have been in the database for a certain period of time. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutContactHistoryPii</url-pattern>
+  </servlet-mapping>
+
   <!-- Action to wipeout Cloud SQL data -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/env/crash/default/WEB-INF/cron.xml

@@ -102,6 +102,7 @@
     <target>backend</target>
   </cron>
 
+  <!-- Disabled for sql-only tests.
   <cron>
     <url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
     <description>
@@ -110,6 +111,7 @@
     <schedule>every 3 minutes synchronized</schedule>
     <target>backend</target>
   </cron>
+  -->
 
   <cron>
     <url><![CDATA[/_dr/task/deleteContactsAndHosts]]></url>
@@ -133,6 +135,7 @@
     <target>backend</target>
   </cron>
 
+  <!-- Disabled for sql-only tests
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>
@@ -140,16 +143,14 @@
       It also enqueues a new task to wait on the completion of that job and then load the resulting
       snapshot into bigquery.
     </description>
-    <!--
+    <!- -
       Keep google.registry.export.CheckBackupAction.MAXIMUM_BACKUP_RUNNING_TIME less than
-      this interval. -->
+      this interval. - ->
     <schedule>every day 06:00</schedule>
     <target>backend</target>
   </cron>
+  -->
 
-  <!--
-    Removed for the duration of load testing
-    TODO(b/71607184): Restore after loadtesting is done
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
     <description>
@@ -159,7 +160,6 @@
     <timezone>UTC</timezone>
     <target>backend</target>
   </cron>
-  -->
 
   <!-- TODO: Add borgmon job to check that these files are created and updated successfully. -->
   <cron>
@@ -200,6 +200,7 @@
     <target>backend</target>
   </cron>
 
+  <!-- Disabled for sql-only tests.
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
     <description>
@@ -208,4 +209,35 @@
     <schedule>every 3 minutes</schedule>
     <target>backend</target>
   </cron>
+  -->
+
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
+  <!--
+    The next two wipeout jobs are required when crash has production data.
+   -->
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
+    <description>
+      This job runs an action that deletes all data in Cloud SQL.
+    </description>
+    <schedule>every saturday 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutDatastore]]></url>
+    <description>
+      This job runs an action that deletes all data in Cloud Datastore.
+    </description>
+    <schedule>every saturday 03:07</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -36,6 +36,19 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url>/_dr/task/rdeStaging?beam=true</url>
+    <description>
+      This job generates a full RDE escrow deposit as a single gigantic XML
+      document using the Beam pipeline regardless of the current TM
+      configuration and streams it to cloud storage. It does not trigger the
+      subsequent upload tasks and is meant to run parallel with the main cron
+      job in order to compare the results from both runs.
+    </description>
+    <schedule>every 8 hours from 00:07 to 20:00</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
     <description>
@@ -193,6 +206,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
+    <description>
+      This job runs an action that sends emails to partners if their certificates are expiring soon.
+    </description>
+    <schedule>every day 04:30</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>
@@ -330,4 +352,32 @@
     <schedule>every day 15:00</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
+    <description>
+      Replays recent commit logs from Datastore to the SQL secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -120,6 +120,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
     <description>