Use the replica jpaTm in FKI and EppResource cache methods (#1503)

The cached methods are only used in situations where we don't really
care about being 100% synchronously up to date (e.g. whois), and they're
not used frequently anyway, so it's safe to use the replica in these
locations.

build.gradle

@@ -20,6 +20,10 @@ buildscript {
     // Lock buildscript dependencies.
     configurations.classpath {
       resolutionStrategy.activateDependencyLocking()
+
+      // log4j has high-profile security vulnerabilities. It's a transitive
+      // dependency used by Gradle itself during build, and not strictly needed.
+      exclude group: 'org.apache.logging.log4j'
     }
   }
 
@@ -46,7 +50,7 @@ plugins {
   id 'com.diffplug.gradle.spotless' version '3.25.0'
 
   id 'jacoco'
-  id 'com.dorongold.task-tree' version '1.5'
+  id 'com.dorongold.task-tree' version '2.1.0'
 }
 
 node {
@@ -182,6 +186,12 @@ allprojects {
       println "Java dependencies: Using Maven Central..."
       mavenCentral()
       google()
+      maven {
+        url "https://packages.confluent.io/maven/"
+        content {
+          includeGroup "io.confluent"
+        }
+      }
     }
   }
 
@@ -196,9 +206,41 @@ allprojects {
   }
 }
 
+rootProject.ext {
+  pyver = { exe ->
+    try {
+      ext.execInBash(
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
+    } catch (org.gradle.process.internal.ExecException e) {
+      return -1;
+    }
+  }
+
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
+    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
+    // Debian so it seems like that should be available anywhere).
+    def MIN_PY_VER = 0x3070300
+    if (pyver('python') >= MIN_PY_VER) {
+      return 'python'
+    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
+      return '/usr/bin/python3'
+    } else {
+      throw new GradleException("No usable Python version found (build " +
+                                "requires at least python 3.7.3)");
+    }
+  }
+}
+
 task runPresubmits(type: Exec) {
-  executable '/usr/bin/python3'
+
   args('config/presubmits.py')
+
+  doFirst {
+    executable getPythonExecutable()
+  }
 }
 
 def javadocSource = []
@@ -412,9 +454,10 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
@@ -422,18 +465,23 @@ rootProject.ext {
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
-    def checkResult = invokeJavaDiffFormatScript("check")
-    if (checkResult == 'true') {
-      throw new IllegalStateException(
-          "Some Java files need to be reformatted. You may use the "
-              + "'javaIncrementalFormatDryRun' task to review\n "
-              + "the changes, or the 'javaIncrementalFormatApply' task "
-              + "to reformat.")
-    } else if (checkResult != 'false') {
-      throw new RuntimeException(
-          "Failed to invoke format check script:\n" + checkResult)
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
+      def checkResult = invokeJavaDiffFormatScript("check")
+      if (checkResult == 'true') {
+        throw new IllegalStateException(
+            "Some Java files need to be reformatted. You may use the "
+                + "'javaIncrementalFormatDryRun' task to review\n "
+                + "the changes, or the 'javaIncrementalFormatApply' task "
+                + "to reformat.")
+      } else if (checkResult != 'false') {
+        throw new RuntimeException(
+            "Failed to invoke format check script:\n" + checkResult)
+      }
+      println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
     }
-    println("Incremental Java format check ok.")
   }
 }
 
@@ -457,15 +505,14 @@ task javaIncrementalFormatApply {
 task javadoc(type: Javadoc) {
   source javadocSource
   classpath = files(javadocClasspath)
-  // Exclude the misbehaving generated-by-Soy Java files
-  exclude "**/*SoyInfo.java"
   destinationDir = file("${buildDir}/docs/javadoc")
   options.encoding = "UTF-8"
   // In a lot of places we don't write @return so suppress warnings about that.
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
   options.tags = ["type:a:Generic Type",
-                  "error:a:Expected Error"]
+                  "error:a:Expected Error",
+                  "invariant:a:Guaranteed Property"]
 }
 
 tasks.build.dependsOn(tasks.javadoc)
@@ -483,3 +530,15 @@ task coreDev {
 }
 
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
+
+// disable javadoc in subprojects, these will break because they don't have
+// the correct classpath (see above).
+gradle.taskGraph.whenReady { graph ->
+  graph.getAllTasks().each { task ->
+    def subprojectJavadoc = (task.path =~ /:.+:javadoc/)
+    if (subprojectJavadoc) {
+        println "Skipping ${task.path} for javadoc (only root javadoc works)"
+        task.enabled = false
+    }
+  }
+}

common/gradle/dependency-locks/testCompile.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testRuntime.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingCompile.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingCompileClasspath.lockfile

@@ -4,7 +4,7 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingRuntime.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/gradle/dependency-locks/testingRuntimeClasspath.lockfile

@@ -4,8 +4,8 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava

common/src/testing/java/google/registry/testing/SystemInfo.java

@@ -39,7 +39,7 @@ public final class SystemInfo {
                     pid.getOutputStream().close();
                     pid.waitFor();
                   } catch (IOException e) {
-                    logger.atWarning().withCause(e).log("%s command not available", cmd);
+                    logger.atWarning().withCause(e).log("%s command not available.", cmd);
                     return false;
                   }
                   return true;

config/dependency-license/allowed_licenses.json

@@ -24,6 +24,9 @@
     {
       "moduleLicense": "Apache License v2.0"
     },
+    {
+      "moduleLicense": "Apache License V2.0"
+    },
     {
       "moduleLicense": "Apache License, Version 2.0"
     },

config/nom_build.py

@@ -140,6 +140,8 @@ PROPERTIES = [
              'a BEAM pipeline to image. Setting this property to empty string '
              'will disable image generation.',
              '/usr/bin/dot'),
+    Property('pipeline',
+             'The name of the Beam pipeline being staged.')
 ]
 
 GRADLE_FLAGS = [

config/presubmits.py

@@ -17,9 +17,11 @@ These aren't built in to the static code analysis tools we use (e.g. Checkstyle,
 Error Prone) so we must write them manually.
 """
 
+import json
 import os
 from typing import List, Tuple
 import sys
+import textwrap
 import re
 
 # We should never analyze any generated files
@@ -28,6 +30,13 @@ UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/"
 FORBIDDEN = 1
 REQUIRED = 2
 
+# The list of expected json packages and their licenses.
+# These should be one of the allowed licenses in:
+# config/dependency-license/allowed_licenses.json
+EXPECTED_JS_PACKAGES = [
+    'google-closure-library',   # Owned by Google, Apache 2.0
+]
+
 
 class PresubmitCheck:
 
@@ -79,8 +88,8 @@ PRESUBMITS = {
         r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
         ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
             ".git", "/build/", "/generated/", "/generated_tests/",
-            "node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
-            "registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
+            "node_modules/", "LoggerConfig.java", "registrar_bin.",
+            "registrar_dbg.", "google-java-format-diff.py",
             "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
         }, REQUIRED):
         "File did not include the license header.",
@@ -110,9 +119,10 @@ PRESUBMITS = {
       "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
+    # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
-        "java", {}):
+        "java", {"DatabaseSnapshotTest.java"}):
       "PostgreSQLContainer instantiation must specify docker tag.",
 
     # Various Soy linting checks
@@ -177,50 +187,6 @@ PRESUBMITS = {
         {"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
     ):
         "JavaScript files should not include console logging.",
-    # SQL injection protection rule for java source file:
-    # The sql template passed to createQuery/createNativeQuery methods must be
-    # a variable name in UPPER_CASE_UNDERSCORE format, i.e., a static final
-    # String variable. This forces the use of parameter-binding on all queries
-    # that take parameters.
-    # The rule would forbid invocation of createQuery(Criteria). However, this
-    # can be handled by adding a helper method in an exempted class to make
-    # the calls.
-    # TODO(b/179158393): enable the 'ConstantName' Java style check to ensure
-    # that non-final variables do not use the UPPER_CASE_UNDERSCORE format.
-    PresubmitCheck(
-        # Line 1: the method names we check and the opening parenthesis, which
-        #    marks the beginning of the first parameter
-        # Line 2: The first parameter is a match if is NOT any of the following:
-        #    - final variable name: \s*([A-Z_]+
-        #    - string literal: "([^"]|\\")*"
-        #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
-        # Line 3: , or the closing parenthesis, marking the end of the first
-        #    parameter
-        r'.*\.(query|createQuery|createNativeQuery)\('
-        r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
-        r'(,|\s*\))))',
-        "java",
-        # ActivityReportingQueryBuilder deals with Dremel queries
-        {"src/test", "ActivityReportingQueryBuilder.java",
-         # This class contains helper method to make queries in Beam.
-         "RegistryJpaIO.java",
-         # TODO(b/179158393): Remove everything below, which should be done
-         # using Criteria
-         "JpaTransactionManager.java",
-         "JpaTransactionManagerImpl.java",
-         # CriteriaQueryBuilder is a false positive
-         "CriteriaQueryBuilder.java",
-         "RdapDomainSearchAction.java",
-         "RdapNameserverSearchAction.java",
-         "ReadOnlyCheckingEntityManager.java",
-         "RegistryQuery",
-         },
-    ):
-        "The first String parameter to EntityManager.create(Native)Query "
-        "methods must be one of the following:\n"
-        "  - A String literal\n"
-        "  - Concatenation of String literals only\n"
-        "  - The name of a static final String variable"
 }
 
 # Note that this regex only works for one kind of Flyway file.  If we want to
@@ -308,6 +274,26 @@ def verify_flyway_index():
   return not success
 
 
+def verify_javascript_deps():
+  """Verifies that we haven't introduced any new javascript dependencies."""
+  with open('package.json') as f:
+    package = json.load(f)
+
+  deps = list(package['dependencies'].keys())
+  if deps != EXPECTED_JS_PACKAGES:
+    print('Unexpected javascript dependencies.  Was expecting '
+          '%s, got %s.' % (EXPECTED_JS_PACKAGES, deps))
+    print(textwrap.dedent("""
+        * If the new dependencies are intentional, please verify that the
+        * license is one of the allowed licenses (see
+        * config/dependency-license/allowed_licenses.json) and add an entry
+        * for the package (with the license in a comment) to the
+        * EXPECTED_JS_PACKAGES variable in config/presubmits.py.
+        """))
+    return True
+  return False
+
+
 def get_files():
   for root, dirnames, filenames in os.walk("."):
     for filename in filenames:
@@ -315,6 +301,7 @@ def get_files():
 
 
 if __name__ == "__main__":
+  print('python version is %s' % sys.version)
   failed = False
   for file in get_files():
     error_messages = []
@@ -331,5 +318,8 @@ if __name__ == "__main__":
   # when we put it here it fails fast before all of the tests are run.
   failed |= verify_flyway_index()
 
+  # Make sure we haven't introduced any javascript dependencies.
+  failed |= verify_javascript_deps()
+
   if failed:
     sys.exit(1)

config/show_upgrade_diffs.py

@@ -138,8 +138,8 @@ def main():
     if args.directory and os.path.exists(dir):
         pr(f'Reusing directory {dir}\n')
         os.chdir(dir)
-        run('git', 'fetch', 'git@github.com:google/nomulus', 'master:master')
-        run('git', 'checkout', 'master')
+        run('git', 'fetch', 'git@github.com:google/nomulus', 'master')
+        run('git', 'checkout', 'origin/master')
     else:
         run('git', 'clone', 'git@github.com:google/nomulus', dir)
         os.chdir(dir)
@@ -175,7 +175,7 @@ def main():
         # Print the list of packages that were removed.
         for package in old_packages:
             if package not in new_packages:
-                pr('removed ', b':'.join(package))
+                pr('removed ', b':'.join(package), '\n')
     else:
         pr('Package versions not updated!\n')
 

core/build.gradle

@@ -257,13 +257,12 @@ dependencies {
   compile deps['org.apache.beam:beam-sdks-java-core']
   compile deps['org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core']
   compile deps['org.apache.beam:beam-sdks-java-io-google-cloud-platform']
-  testCompile deps['org.apache.commons:commons-lang3']
+  compile deps['org.apache.commons:commons-lang3']
   testCompile deps['org.apache.commons:commons-text']
   testCompile deps['org.apache.ftpserver:ftplet-api']
   testCompile deps['org.apache.ftpserver:ftpserver-core']
   compile deps['org.apache.httpcomponents:httpclient']
   compile deps['org.apache.httpcomponents:httpcore']
-  runtime deps['org.apache.logging.log4j:log4j-core']
   testCompile deps['org.apache.sshd:sshd-core']
   testCompile deps['org.apache.sshd:sshd-scp']
   testCompile deps['org.apache.sshd:sshd-sftp']
@@ -457,6 +456,22 @@ task soyToJava {
               "--srcs", "${soyFiles.join(',')}",
               "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
     }
+
+    // Replace the "@link" tags after the "@deprecated" tags in the generated
+    // files.  The soy compiler doesn't generate imports for these, causing
+    // us to get warnings when we generate javadocs.
+    // TODO(b/200296387): To be fair, the deprecations are accurate: we're
+    // using the old "SoyInfo" classes instead of the new "Templates" files.
+    // When we convert to the new classes, this hack can go away.
+    def outputs = fileTree(outputDirectory) {
+      include '**/*.java'
+    }
+
+    outputs.each { file ->
+      exec {
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
+      }
+    }
   }
 
   doLast {
@@ -661,9 +676,9 @@ Optional<List<String>> getToolArgsList() {
 
 // To run the nomulus tools with these command line tokens:
 // "--foo", "bar baz", "--qux=quz"
-//   gradle registryTool --args="--foo 'bar baz' --qux=quz"
+//   gradle core:registryTool --args="--foo 'bar baz' --qux=quz"
 // or:
-//   gradle registryTool --PtoolArgs="--foo|bar baz|--qux=quz"
+//   gradle core:registryTool -PtoolArgs="--foo|bar baz|--qux=quz"
 // Note that the delimiting pipe can be backslash escaped if it is part of a
 // parameter.
 ext.createToolTask = {
@@ -687,33 +702,22 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
+createToolTask(
+    'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
 
 createToolTask(
-        'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
 
-project.tasks.create('initSqlPipeline', JavaExec) {
-  main = 'google.registry.beam.initsql.InitSqlPipeline'
+createToolTask(
+    'validateDatastorePipeline', 'google.registry.beam.comparedb.ValidateDatastorePipeline')
 
-  doFirst {
-    getToolArgsList().ifPresent {
-      args it
-    }
 
-    def isDirectRunner =
-        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
-    // The dependency containing DirectRunner is intentionally excluded from the
-    // production binary, so that it won't be chosen by mistake: we definitely do
-    // not want to use it for the real jobs, yet DirectRunner is the default if
-    // the user forgets to override it.
-    // DirectRunner is required for tests and is already on testRuntimeClasspath.
-    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
-    // of defining a new configuration just for the DirectRunner dependency.
-    classpath =
-        isDirectRunner
-            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
-            : sourceSets.main.runtimeClasspath
-  }
-}
+createToolTask(
+    'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+
+createToolTask(
+    'createSyntheticHistoryEntries',
+    'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
 // Caller must provide projectId, GCP region, runner, and the kinds to delete
 // (comma-separated kind names or '*' for all). E.g.:
@@ -756,50 +760,67 @@ createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
 // This packages more code and dependency than necessary. However, without
 // restructuring the source tree it is difficult to generate leaner jars.
 createUberJar(
-    'beam_pipeline_common',
+    'beamPipelineCommon',
     'beam_pipeline_common',
     '')
 
-// Create beam staging task if environment is alpha or crash.
-// All other environments use formally released pipelines through CloudBuild.
+// Create beam staging task if the environment is alpha. Production, sandbox and
+// qa use formally released pipelines through CloudBuild, whereas crash and
+// alpha use the pipelines staged on alpha deployment project.
 //
 // User should install gcloud and login to GCP before invoking this tasks.
-if (environment in ['alpha', 'crash']) {
+if (environment == 'alpha') {
   def pipelines = [
-      [
-          mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
-          metaData: 'google/registry/beam/init_sql_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
-          metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
-          metaData: 'google/registry/beam/spec11_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
-          metaData: 'google/registry/beam/invoicing_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.rde.RdePipeline',
-          metaData: 'google/registry/beam/rde_pipeline_metadata.json'
-      ],
+      initSql            :
+          [
+              mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
+              metaData : 'google/registry/beam/init_sql_pipeline_metadata.json'
+          ],
+      bulkDeleteDatastore:
+          [
+              mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
+              metaData : 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
+          ],
+      spec11             :
+          [
+              mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
+              metaData : 'google/registry/beam/spec11_pipeline_metadata.json'
+          ],
+      invoicing          :
+          [
+              mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
+              metaData : 'google/registry/beam/invoicing_pipeline_metadata.json'
+          ],
+      rde                :
+          [
+              mainClass: 'google.registry.beam.rde.RdePipeline',
+              metaData : 'google/registry/beam/rde_pipeline_metadata.json'
+          ],
+      validateDatastore  :
+          [
+              mainClass: 'google.registry.beam.comparedb.ValidateDatastorePipeline',
+              metaData: 'google/registry/beam/validate_datastore_pipeline_metadata.json'
+          ],
+      validateSql  :
+          [
+              mainClass: 'google.registry.beam.comparedb.ValidateSqlPipeline',
+              metaData: 'google/registry/beam/validate_sql_pipeline_metadata.json'
+          ],
   ]
-  project.tasks.create("stage_beam_pipelines") {
+  project.tasks.create("stageBeamPipelines") {
     doLast {
       pipelines.each {
-        def mainClass = it['mainClass']
-        def metaData = it['metaData']
-        def pipelineName = CaseFormat.UPPER_CAMEL.to(
-                CaseFormat.LOWER_UNDERSCORE,
-                mainClass.substring(mainClass.lastIndexOf('.') + 1))
-        def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
-        def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
-        def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath
+        if (rootProject.pipeline == ''|| rootProject.pipeline == it.key) {
+          def mainClass = it.value['mainClass']
+          def metaData = it.value['metaData']
+          def pipelineName = CaseFormat.UPPER_CAMEL.to(
+              CaseFormat.LOWER_UNDERSCORE,
+              mainClass.substring(mainClass.lastIndexOf('.') + 1))
+          def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
+          def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
+          def uberJarName = tasks.beamPipelineCommon.outputs.files.asPath
 
-        def command = "\
+          def command = "\
             gcloud dataflow flex-template build \
             gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
             --image-gcr-path ${imageName}:live \
@@ -809,10 +830,11 @@ if (environment in ['alpha', 'crash']) {
             --jar ${uberJarName} \
             --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
             --project ${gcpProject}".toString()
-        rootProject.ext.execInBash(command, '/tmp')
+          rootProject.ext.execInBash(command, '/tmp')
+        }
       }
     }
-  }.dependsOn(tasks.beam_pipeline_common)
+  }.dependsOn(tasks.beamPipelineCommon)
 }
 
 // A jar with classes and resources from main sourceSet, excluding internal
@@ -1087,6 +1109,10 @@ test {
   // TODO(weiminyu): Remove dependency on sqlIntegrationTest
 }.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
 
+// When we override tests, we also break the cleanTest command.
+cleanTest.dependsOn(cleanFragileTest, cleanOutcastTest, cleanStandardTest,
+                    cleanRegistryToolIntegrationTest, cleanSqlIntegrationTest)
+
 project.build.dependsOn devtool
 project.build.dependsOn buildToolImage
 project.build.dependsOn ':stage'

core/gradle/dependency-locks/compile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,111 +24,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,7 +143,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -153,32 +159,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -195,38 +205,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -259,6 +272,8 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -266,8 +281,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,109 +24,114 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -135,8 +142,8 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
@@ -151,30 +158,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -190,37 +201,40 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -252,6 +266,8 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -259,8 +275,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/default.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,112 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -142,7 +148,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -161,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -203,41 +213,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -273,6 +285,8 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -281,8 +295,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,112 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -142,7 +148,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -161,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -203,40 +213,42 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -272,6 +284,8 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -280,8 +294,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,111 +24,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,7 +143,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -153,32 +159,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -195,38 +205,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -259,6 +272,8 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -266,8 +281,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,109 +24,114 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -135,8 +142,8 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.3
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.template:soy:2021-02-01
@@ -151,30 +158,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -190,38 +201,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -253,6 +267,8 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -260,8 +276,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,111 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -141,7 +147,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -160,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -202,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -271,6 +283,8 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -279,8 +293,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,111 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -141,7 +147,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -160,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -202,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -271,6 +283,8 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -279,8 +293,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/runtime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,111 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -141,7 +147,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -160,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -202,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -271,6 +283,8 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -279,8 +293,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,112 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.79.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.94.1
-com.google.cloud:google-cloud-core:1.94.3
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.2
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -142,7 +148,7 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
+com.google.oauth-client:google-oauth-client:1.31.5
 com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
@@ -161,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -203,40 +213,42 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.8.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -272,6 +284,8 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
@@ -280,8 +294,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testCompile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,61 +24,60 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -87,38 +88,43 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -126,10 +132,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -162,32 +168,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -204,30 +214,34 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -236,7 +250,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -247,7 +260,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -307,6 +320,8 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -316,8 +331,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,60 +24,59 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -86,37 +87,43 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -124,10 +131,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -160,30 +167,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -199,30 +210,34 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -231,7 +246,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -242,7 +256,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -301,6 +315,8 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -310,8 +326,8 @@ org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testRuntime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,61 +28,60 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -91,39 +92,44 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -131,10 +137,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -170,33 +176,37 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -213,31 +223,35 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -246,8 +260,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -258,7 +270,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -320,6 +332,8 @@ org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -330,8 +344,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.3
-com.fasterxml.jackson.core:jackson-core:2.12.3
-com.fasterxml.jackson.core:jackson-databind:2.12.3
-com.fasterxml.jackson:jackson-bom:2.12.3
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,61 +28,60 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
 com.google.api-client:google-api-client:1.32.1
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
-com.google.api.grpc:proto-google-common-protos:2.3.2
-com.google.api.grpc:proto-google-iam-v1:1.0.14
-com.google.api:api-common:1.10.4
-com.google.api:gax-grpc:1.66.0
-com.google.api:gax-httpjson:0.83.0
-com.google.api:gax:1.66.0
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
 com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
@@ -91,39 +92,44 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.26.0
-com.google.auth:google-auth-library-oauth2-http:0.26.0
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.8.1
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.95.4
-com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
 com.google.cloud:google-cloud-nio:0.123.4
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
 com.google.cloud:google-cloud-storage:1.118.0
 com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.7
+com.google.code.gson:gson:2.8.8
 com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.7.1
+com.google.errorprone:error_prone_annotations:2.9.0
 com.google.escapevelocity:escapevelocity:0.9.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava-testlib:30.1.1-jre
 com.google.guava:guava:30.1.1-jre
@@ -131,10 +137,10 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.2
 com.google.http-client:google-http-client-appengine:1.39.2
-com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
 com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.2
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -170,33 +176,37 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.39.0
-io.grpc:grpc-api:1.39.0
-io.grpc:grpc-auth:1.39.0
-io.grpc:grpc-context:1.39.0
-io.grpc:grpc-core:1.39.0
-io.grpc:grpc-grpclb:1.39.0
-io.grpc:grpc-netty-shaded:1.39.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.39.0
-io.grpc:grpc-protobuf:1.39.0
-io.grpc:grpc-stub:1.39.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -213,31 +223,35 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -246,8 +260,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -258,7 +270,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -321,6 +333,8 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
 org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:junit-jupiter:1.15.2
@@ -331,8 +345,8 @@ org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -21,6 +21,7 @@ import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.ImmutableList;
 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,6 +29,7 @@ import java.io.OutputStream;
 import java.util.Iterator;
 
 /** Utilities for working with backups. */
+@DeleteAfterMigration
 public class BackupUtils {
 
   /** Keys for user metadata fields on commit log files in GCS. */
@@ -41,11 +43,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnlyWithoutBackup().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -14,21 +14,22 @@
 
 package google.registry.backup;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -48,6 +49,7 @@ import org.joda.time.DateTime;
     method = Action.Method.GET,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public final class CommitLogCheckpointAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -56,7 +58,8 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Inject Clock clock;
   @Inject CommitLogCheckpointStrategy strategy;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject CommitLogCheckpointAction() {}
 
   @Override
@@ -64,7 +67,8 @@ public final class CommitLogCheckpointAction implements Runnable {
     final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
     logger.atInfo().log(
         "Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
-    tm().transact(
+    ofyTm()
+        .transact(
             () -> {
               DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
               if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
@@ -73,16 +77,20 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveWithoutBackup()
+                  .saveIgnoringReadOnlyWithoutBackup()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.
-              taskQueueUtils.enqueue(
-                  getQueue(QUEUE_NAME),
-                  withUrl(ExportCommitLogDiffAction.PATH)
-                      .param(LOWER_CHECKPOINT_TIME_PARAM, lastWrittenTime.toString())
-                      .param(
-                          UPPER_CHECKPOINT_TIME_PARAM, checkpoint.getCheckpointTime().toString()));
+              cloudTasksUtils.enqueue(
+                  QUEUE_NAME,
+                  CloudTasksUtils.createPostTask(
+                      ExportCommitLogDiffAction.PATH,
+                      Service.BACKEND.toString(),
+                      ImmutableMultimap.of(
+                          LOWER_CHECKPOINT_TIME_PARAM,
+                          lastWrittenTime.toString(),
+                          UPPER_CHECKPOINT_TIME_PARAM,
+                          checkpoint.getCheckpointTime().toString())));
             });
   }
 }

core/src/main/java/google/registry/backup/CommitLogCheckpointStrategy.java

@@ -23,6 +23,7 @@ import static google.registry.util.DateTimeUtils.earliestOf;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -36,14 +37,14 @@ import org.joda.time.DateTime;
 /**
  * Implementation of the procedure for determining point-in-time consistent commit log checkpoint.
  *
- * <p>This algorithm examines the recently written commit log data and uses a dual-read approach
- * to determine a point-in-time consistent set of checkpoint times for the commit log buckets.  By
+ * <p>This algorithm examines the recently written commit log data and uses a dual-read approach to
+ * determine a point-in-time consistent set of checkpoint times for the commit log buckets. By
  * "consistent" we mean, generally speaking, that if the Datastore were restored by replaying all
  * the commit logs up to the checkpoint times of the buckets, the result would be transactionally
  * correct; there must be no "holes" where restored state depends on non-restored state.
  *
  * <p>The consistency guarantee really has two parts, only one of which is provided by this
- * algorithm.  The procedure below guarantees only that if the resulting checkpoint includes any
+ * algorithm. The procedure below guarantees only that if the resulting checkpoint includes any
  * given commit log, it will also include all the commit logs that were both 1) actually written
  * before that commit log "in real life", and 2) have an earlier timestamp than that commit log.
  * (These criteria do not necessarily imply each other, due to the lack of a global shared clock.)
@@ -51,8 +52,8 @@ import org.joda.time.DateTime;
  * that depends on state from a previous transaction does indeed have a later timestamp.
  *
  * <h2>Procedure description</h2>
- * <pre>
- * {@code
+ *
+ * <pre>{@code
  * ComputeCheckpoint() -> returns a set consisting of a timestamp c(b_i) for every bucket b_i
  *
  * 1) read off the latest commit timestamp t(b_i) for every bucket b_i
@@ -63,35 +64,29 @@ import org.joda.time.DateTime;
  *   a) if S is empty, let T* = +∞ (or the "end of time")
  *   b) else, let T* = T - Δ, for T = min(S) and some small Δ > 0
  * 4) return the set given by: min(t(b_i), T*) for all b_i
- * }
- * </pre>
+ * }</pre>
  *
  * <h2>Correctness proof of algorithm</h2>
  *
- * <p>{@literal
- * As described above, the algorithm is correct as long as it can ensure the following: given a
- * commit log X written at time t(X) to bucket b_x, and another commit log Y that was written "in
- * real life" before X and for which t(Y) < t(X), then if X is included in the checkpoint, so is Y;
- * that is, t(X) <= c(b_x) implies t(Y) <= c(b_y).
- * }
+ * <p>{@literal As described above, the algorithm is correct as long as it can ensure the following:
+ * given a commit log X written at time t(X) to bucket b_x, and another commit log Y that was
+ * written "in real life" before X and for which t(Y) < t(X), then if X is included in the
+ * checkpoint, so is Y; that is, t(X) <= c(b_x) implies t(Y) <= c(b_y). }
  *
- * <p>{@literal
- * To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e. every commit
- * log included in the checkpoint must have been seen in the first pass.  Hence if X was included,
- * then X must have been written by the time we started the second pass.  But since Y was written
- * "in real life" prior to X, we must have seen Y by the second pass too.
- * }
+ * <p>{@literal To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e.
+ * every commit log included in the checkpoint must have been seen in the first pass. Hence if X was
+ * included, then X must have been written by the time we started the second pass. But since Y was
+ * written "in real life" prior to X, we must have seen Y by the second pass too. }
  *
- * <p>{@literal
- * Now assume towards a contradiction that X is indeed included but Y is not, i.e. that we have
- * t(X) <= c(b_x) but t(Y) > c(b_y).  If Y was seen in the first pass, i.e. t(Y) <= t(b_y), then by
- * our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y).  By the definition of
- * c(b_y) it must then equal T*, so we have T* < t(Y).  However, this is a contradiction since
- * t(Y) < t(X) and t(X) <= c(b_x) <= T*.  If instead Y was seen in the second pass but not the
- * first, t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by
- * definition, we again reach the contradiction T* < t(Y).
- * }
+ * <p>{@literal Now assume towards a contradiction that X is indeed included but Y is not, i.e. that
+ * we have t(X) <= c(b_x) but t(Y) > c(b_y). If Y was seen in the first pass, i.e. t(Y) <= t(b_y),
+ * then by our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y). By the definition
+ * of c(b_y) it must then equal T*, so we have T* < t(Y). However, this is a contradiction since
+ * t(Y) < t(X) and t(X) <= c(b_x) <= T*. If instead Y was seen in the second pass but not the first,
+ * t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by definition,
+ * we again reach the contradiction T* < t(Y). }
  */
+@DeleteAfterMigration
 class CommitLogCheckpointStrategy {
 
   @Inject Ofy ofy;

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -20,6 +20,7 @@ import static google.registry.backup.BackupUtils.createDeserializingIterator;
 
 import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
@@ -38,6 +39,7 @@ import java.util.Iterator;
  * <p>This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial
  * population of the Cloud SQL database.
  */
+@DeleteAfterMigration
 public final class CommitLogImports {
 
   private CommitLogImports() {}
@@ -55,10 +57,9 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+  static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
       InputStream inputStream) {
-    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
-        InputStream input = new BufferedInputStream(inputStream)) {
+    try (InputStream input = new BufferedInputStream(inputStream)) {
       Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
       checkState(commitLogs.hasNext());
       checkState(commitLogs.next() instanceof CommitLogCheckpoint);
@@ -105,7 +106,7 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+  static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
     return loadEntitiesByTransaction(inputStream).stream()
         .flatMap(ImmutableList::stream)
         .collect(toImmutableList());

core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static java.lang.Boolean.FALSE;
 import static java.lang.Boolean.TRUE;
 
@@ -35,6 +35,7 @@ import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.CommitLogManifestInput;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
@@ -68,6 +69,7 @@ import org.joda.time.Duration;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 // No longer needed in SQL. Subject to future removal.
 @Deprecated
+@DeleteAfterMigration
 public final class DeleteOldCommitLogsAction implements Runnable {
 
   private static final int NUM_MAP_SHARDS = 20;
@@ -93,7 +95,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
   public void run() {
     DateTime deletionThreshold = clock.nowUtc().minus(maxAge);
     logger.atInfo().log(
-        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s",
+        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s.",
         deletionThreshold);
 
     mrRunner
@@ -208,7 +210,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       getContext().incrementCounter("EPP resources missing pre-threshold revision (SEE LOGS)");
       logger.atSevere().log(
           "EPP resource missing old enough revision: "
-              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s",
+              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s.",
           Key.create(eppResource),
           eppResource.getCreationTime(),
           eppResource.getRevisions().size(),
@@ -286,7 +288,8 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       }
 
       DeletionResult deletionResult =
-          tm().transactNew(
+          ofyTm()
+              .transactNew(
                   () -> {
                     CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
                     // It is possible that the same manifestKey was run twice, if a shard had to be

core/src/main/java/google/registry/backup/EntityImports.java

@@ -20,10 +20,12 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
 import com.google.storage.onestore.v3.OnestoreEntity.Path;
 import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
 import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
 /** Utilities for handling imported Datastore entities. */
+@DeleteAfterMigration
 public class EntityImports {
 
   /**

core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java

@@ -40,6 +40,7 @@ import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -63,6 +64,7 @@ import org.joda.time.DateTime;
     method = Action.Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public final class ExportCommitLogDiffAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -100,7 +102,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
 
     // Load the keys of all the manifests to include in this diff.
     List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
-    logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
+    logger.atInfo().log("Found %d manifests to export.", sortedKeys.size());
     // Open an output channel to GCS, wrapped in a stream for convenience.
     try (OutputStream gcsStream =
         gcsUtils.openOutputStream(
@@ -124,7 +126,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
       for (int i = 0; i < keyChunks.size(); i++) {
         // Force the async load to finish.
         Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
-        logger.atInfo().log("Loaded %d manifests", chunkValues.size());
+        logger.atInfo().log("Loaded %d manifests.", chunkValues.size());
         // Since there is no hard bound on how much data this might be, take care not to let the
         // Objectify session cache fill up and potentially run out of memory. This is the only safe
         // point to do this since at this point there is no async load in progress.
@@ -134,12 +136,12 @@ public final class ExportCommitLogDiffAction implements Runnable {
           nextChunkToExport = auditedOfy().load().keys(keyChunks.get(i + 1));
         }
         exportChunk(gcsStream, chunkValues);
-        logger.atInfo().log("Exported %d manifests", chunkValues.size());
+        logger.atInfo().log("Exported %d manifests.", chunkValues.size());
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    logger.atInfo().log("Exported %d manifests in total", sortedKeys.size());
+    logger.atInfo().log("Exported %d total manifests.", sortedKeys.size());
   }
 
   /**

core/src/main/java/google/registry/backup/GcsDiffFileLister.java

@@ -33,6 +33,7 @@ import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import google.registry.backup.BackupModule.Backups;
 import google.registry.gcs.GcsUtils;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Map;
@@ -44,6 +45,7 @@ import javax.inject.Provider;
 import org.joda.time.DateTime;
 
 /** Utility class to list commit logs diff files stored on GCS. */
+@DeleteAfterMigration
 class GcsDiffFileLister {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -91,17 +93,17 @@ class GcsDiffFileLister {
 
         // If we hit a gap, quit.
         if (blobInfo == null) {
-          logger.atInfo().log(
+          logger.atWarning().log(
               "Gap discovered in sequence terminating at %s, missing file: %s",
               sequence.lastKey(), filename);
-          logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+          logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
           return false;
         }
       }
       sequence.put(checkpointTime, blobInfo);
       checkpointTime = getLowerBoundTime(blobInfo);
     }
-    logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+    logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
     return true;
   }
 
@@ -140,7 +142,7 @@ class GcsDiffFileLister {
         }
       }
       if (upperBoundTimesToBlobInfo.isEmpty()) {
-        logger.atInfo().log("No files found");
+        logger.atInfo().log("No files found.");
         return ImmutableList.of();
       }
 
@@ -167,6 +169,10 @@ class GcsDiffFileLister {
             break;
           }
           if (!sequence.containsKey(key)) {
+            // Recalculate the sequence for purely informational purposes.
+            logger.atWarning().log(
+                "Fork found in commit log history.  The following sequence "
+                    + "is disconnected from the sequence of the final commit:");
             constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
             checkForMoreExtraDiffs = true;
             inconsistentFileSet = true;
@@ -185,7 +191,7 @@ class GcsDiffFileLister {
 
     logger.atInfo().log(
         "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
-    logger.atInfo().log("Found %d files to restore", sequence.size());
+    logger.atInfo().log("Found %d files to restore.", sequence.size());
     return ImmutableList.copyOf(sequence.values());
   }
 

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -34,6 +34,7 @@ import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
 import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
@@ -68,6 +69,7 @@ import org.joda.time.Seconds;
     method = Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class ReplayCommitLogsToSqlAction implements Runnable {
 
   static final String PATH = "/_dr/task/replayCommitLogsToSql";
@@ -112,7 +114,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
@@ -140,7 +142,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 
@@ -264,7 +266,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             .ifPresent(
                 sqlEntity -> {
                   sqlEntity.beforeSqlSaveOnReplay();
-                  jpaTm().putIgnoringReadOnly(sqlEntity);
+                  jpaTm().putIgnoringReadOnlyWithoutBackup(sqlEntity);
                 });
       } else {
         // this should never happen, but we shouldn't fail on it
@@ -273,7 +275,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             ofyPojo.getClass());
       }
     } catch (Throwable t) {
-      logger.atSevere().log("Error when replaying object %s", ofyPojo);
+      logger.atSevere().log("Error when replaying object %s.", ofyPojo);
       throw t;
     }
   }
@@ -297,10 +299,10 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
           && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
           && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
         ReplaySpecializer.beforeSqlDelete(entityVKey);
-        jpaTm().deleteIgnoringReadOnly(entityVKey);
+        jpaTm().deleteIgnoringReadOnlyWithoutBackup(entityVKey);
       }
     } catch (Throwable t) {
-      logger.atSevere().log("Error when deleting key %s", entityVKey);
+      logger.atSevere().log("Error when deleting key %s.", entityVKey);
       throw t;
     }
   }

core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java

@@ -37,6 +37,7 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
@@ -64,6 +65,7 @@ import org.joda.time.DateTime;
     method = Action.Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class RestoreCommitLogsAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -103,13 +105,13 @@ public class RestoreCommitLogsAction implements Runnable {
         !FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
         "DO NOT RUN IN PRODUCTION OR SANDBOX.");
     if (dryRun) {
-      logger.atInfo().log("Running in dryRun mode");
+      logger.atInfo().log("Running in dry-run mode.");
     }
     String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
     logger.atInfo().log("Restoring from %s.", gcsBucket);
     List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
     if (diffFiles.isEmpty()) {
-      logger.atInfo().log("Nothing to restore");
+      logger.atInfo().log("Nothing to restore.");
       return;
     }
     Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
@@ -143,7 +145,7 @@ public class RestoreCommitLogsAction implements Runnable {
                                 .build()),
                 Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
             .collect(toImmutableList()));
-    logger.atInfo().log("Restore complete");
+    logger.atInfo().log("Restore complete.");
   }
 
   /**

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -19,6 +19,7 @@ import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.appengine.api.datastore.Key;
 import com.google.auto.value.AutoValue;
 import com.google.auto.value.extension.memoized.Memoized;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
 import java.io.Serializable;
@@ -76,6 +77,7 @@ import javax.annotation.Nullable;
  * property type in this class.
  */
 @AutoValue
+@DeleteAfterMigration
 public abstract class VersionedEntity implements Serializable {
 
   private static final long serialVersionUID = 1L;
@@ -105,29 +107,29 @@ public abstract class VersionedEntity implements Serializable {
    * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
    * information.
    */
-  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+  static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
     long commitTimeMillis = manifest.getCommitTime().getMillis();
     return manifest.getDeletions().stream()
         .map(com.googlecode.objectify.Key::getRaw)
-        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+        .map(key -> newBuilder().commitTimeMills(commitTimeMillis).key(key).build());
   }
 
   /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
-  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+  static VersionedEntity fromMutation(CommitLogMutation mutation) {
     return from(
         com.googlecode.objectify.Key.create(mutation).getParent().getId(),
         mutation.getEntityProtoBytes());
   }
 
   public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
-    return builder()
+    return newBuilder()
         .entityProtoBytes(entityProtoBytes)
         .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
         .commitTimeMills(commitTimeMillis)
         .build();
   }
 
-  static Builder builder() {
+  private static Builder newBuilder() {
     return new AutoValue_VersionedEntity.Builder();
   }
 
@@ -142,7 +144,7 @@ public abstract class VersionedEntity implements Serializable {
 
     public abstract VersionedEntity build();
 
-    public Builder entityProtoBytes(byte[] bytes) {
+    Builder entityProtoBytes(byte[] bytes) {
       return entityProtoBytes(new ImmutableBytes(bytes));
     }
   }

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,10 +24,8 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
@@ -84,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -96,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -114,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -129,14 +125,13 @@ public final class AsyncTaskEnqueuer {
       String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingRegistrarId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().stringify())
             .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))
@@ -153,7 +148,7 @@ public final class AsyncTaskEnqueuer {
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,
         TaskOptions.Builder.withMethod(Method.PULL)
-            .param(PARAM_HOST_KEY, hostKey.getOfyKey().getString())
+            .param(PARAM_HOST_KEY, hostKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString()));
   }
 

core/src/main/java/google/registry/batch/BatchModule.java

@@ -24,6 +24,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.request.RequestParameters.extractIntParameter;
 import static google.registry.request.RequestParameters.extractLongParameter;
 import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
+import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter;
 import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
@@ -32,10 +33,8 @@ import static google.registry.request.RequestParameters.extractSetOfDatetimePara
 
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.model.ImmutableObject;
 import google.registry.request.Parameter;
 import java.util.Optional;
 import javax.inject.Named;
@@ -78,8 +77,8 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
-  static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
-    return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
+  static String provideResourceKey(HttpServletRequest req) {
+    return extractRequiredParameter(req, PARAM_RESOURCE_KEY);
   }
 
   @Provides
@@ -106,6 +105,13 @@ public class BatchModule {
     return extractIntParameter(req, RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM);
   }
 
+  @Provides
+  @Parameter(ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME)
+  static Optional<DateTime> provideCursorTime(HttpServletRequest req) {
+    return extractOptionalDatetimeParameter(
+        req, ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME);
+  }
+
   @Provides
   @Named(QUEUE_ASYNC_ACTIONS)
   static Queue provideAsyncActionsPushQueue() {

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -73,6 +73,7 @@ import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.mapreduce.inputs.NullInput;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
@@ -114,6 +115,7 @@ import org.joda.time.Duration;
     service = Action.Service.BACKEND,
     path = "/_dr/task/deleteContactsAndHosts",
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class DeleteContactsAndHostsAction implements Runnable {
 
   static final String KIND_CONTACT = getKind(ContactResource.class);
@@ -311,7 +313,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
     @Override
     public void reduce(final DeletionRequest deletionRequest, ReducerInput<Boolean> values) {
       final boolean hasNoActiveReferences = !Iterators.contains(values, true);
-      logger.atInfo().log("Processing async deletion request for %s", deletionRequest.key());
+      logger.atInfo().log("Processing async deletion request for %s.", deletionRequest.key());
       DeletionResult result =
           tm()
               .transactNew(
@@ -523,18 +525,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(
@@ -605,12 +608,12 @@ public class DeleteContactsAndHostsAction implements Runnable {
   static boolean doesResourceStateAllowDeletion(EppResource resource, DateTime now) {
     Key<EppResource> key = Key.create(resource);
     if (isDeleted(resource, now)) {
-      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted", key);
+      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted.", key);
       return false;
     }
     if (!resource.getStatusValues().contains(PENDING_DELETE)) {
       logger.atWarning().log(
-          "Cannot asynchronously delete %s because it is not in PENDING_DELETE", key);
+          "Cannot asynchronously delete %s because it is not in PENDING_DELETE.", key);
       return false;
     }
     return true;

core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java

@@ -167,7 +167,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
   /** Runs the actual domain delete flow and returns whether the deletion was successful. */
   private boolean runDomainDeleteFlow(DomainBase domain) {
-    logger.atInfo().log("Attempting to delete domain %s", domain.getDomainName());
+    logger.atInfo().log("Attempting to delete domain '%s'.", domain.getDomainName());
     // Create a new transaction that the flow's execution will be enlisted in that loads the domain
     // transactionally. This way we can ensure that nothing else has modified the domain in question
     // in the intervening period since the query above found it.
@@ -203,7 +203,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
     if (eppOutput.isPresent()) {
       if (eppOutput.get().isSuccess()) {
-        logger.atInfo().log("Successfully deleted domain %s", domain.getDomainName());
+        logger.atInfo().log("Successfully deleted domain '%s'.", domain.getDomainName());
       } else {
         logger.atSevere().log(
             "Failed to delete domain %s; EPP response:\n\n%s",

core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java

@@ -142,7 +142,7 @@ public class DeleteLoadTestDataAction implements Runnable {
     // that are linked to domains (since it would break the foreign keys)
     if (EppResourceUtils.isLinked(contact.createVKey(), clock.nowUtc())) {
       logger.atWarning().log(
-          "Cannot delete contact with repo ID %s since it is referenced from a domain",
+          "Cannot delete contact with repo ID %s since it is referenced from a domain.",
           contact.getRepoId());
       return;
     }
@@ -177,7 +177,7 @@ public class DeleteLoadTestDataAction implements Runnable {
         HistoryEntryDao.loadHistoryObjectsForResource(eppResource.createVKey());
     if (isDryRun) {
       logger.atInfo().log(
-          "Would delete repo ID %s along with %d history objects",
+          "Would delete repo ID %s along with %d history objects.",
           eppResource.getRepoId(), historyObjects.size());
     } else {
       historyObjects.forEach(tm()::delete);

core/src/main/java/google/registry/batch/DeleteProberDataAction.java

@@ -228,7 +228,7 @@ public class DeleteProberDataAction implements Runnable {
     if (EppResourceUtils.isActive(domain, tm().getTransactionTime())) {
       if (isDryRun) {
         logger.atInfo().log(
-            "Would soft-delete the active domain: %s (%s)",
+            "Would soft-delete the active domain: %s (%s).",
             domain.getDomainName(), domain.getRepoId());
       } else {
         softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue);
@@ -237,7 +237,7 @@ public class DeleteProberDataAction implements Runnable {
     } else {
       if (isDryRun) {
         logger.atInfo().log(
-            "Would hard-delete the non-active domain: %s (%s) and its dependents",
+            "Would hard-delete the non-active domain: %s (%s) and its dependents.",
             domain.getDomainName(), domain.getRepoId());
       } else {
         domainRepoIdsToHardDelete.add(domain.getRepoId());
@@ -331,7 +331,7 @@ public class DeleteProberDataAction implements Runnable {
           getContext().incrementCounter("skipped, non-prober data");
         }
       } catch (Throwable t) {
-        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s", key);
+        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s.", key);
         getContext().incrementCounter(String.format("error, kind %s", key.getKind()));
       }
     }
@@ -372,7 +372,7 @@ public class DeleteProberDataAction implements Runnable {
       if (EppResourceUtils.isActive(domain, now)) {
         if (isDryRun) {
           logger.atInfo().log(
-              "Would soft-delete the active domain: %s (%s)", domainName, domainKey);
+              "Would soft-delete the active domain: %s (%s).", domainName, domainKey);
         } else {
           tm().transact(() -> softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue));
         }

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -148,9 +148,9 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
               .reduce(0, Integer::sum);
 
       if (!isDryRun) {
-        logger.atInfo().log("Saved OneTime billing events", numBillingEventsSaved);
+        logger.atInfo().log("Saved OneTime billing events.", numBillingEventsSaved);
       } else {
-        logger.atInfo().log("Generated OneTime billing events (dry run)", numBillingEventsSaved);
+        logger.atInfo().log("Generated OneTime billing events (dry run).", numBillingEventsSaved);
       }
       logger.atInfo().log(
           "Recurring event expansion %s complete for billing event range [%s, %s).",

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -173,7 +173,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
                     retrier.callWithRetry(
                         () -> dnsQueue.addDomainRefreshTask(domainName),
                         TransientFailureException.class);
-                    logger.atInfo().log("Enqueued DNS refresh for domain %s.", domainName);
+                    logger.atInfo().log("Enqueued DNS refresh for domain '%s'.", domainName);
                   });
           deleteTasksWithRetry(
               refreshRequests,
@@ -353,8 +353,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
       VKey<HostResource> hostKey =
-          VKey.fromWebsafeKey(
-              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
           tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
               .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -313,7 +313,7 @@ public class RelockDomainAction implements Runnable {
         builder.add(new InternetAddress(registryLockEmailAddress));
       } catch (AddressException e) {
         // This shouldn't stop any other emails going out, so swallow it
-        logger.atWarning().log("Invalid email address %s", registryLockEmailAddress);
+        logger.atWarning().log("Invalid email address '%s'.", registryLockEmailAddress);
       }
     }
     return builder.build();

core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java

@@ -24,6 +24,7 @@ import com.googlecode.objectify.Key;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -56,6 +57,7 @@ import javax.inject.Inject;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 // No longer needed in SQL. Subject to future removal.
 @Deprecated
+@DeleteAfterMigration
 public class ResaveAllEppResourcesAction implements Runnable {
 
   @Inject MapreduceRunner mrRunner;

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -22,7 +22,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.persistence.VKey;
@@ -50,7 +49,7 @@ public class ResaveEntityAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private final Key<ImmutableObject> resourceKey;
+  private final String resourceKey;
   private final DateTime requestedTime;
   private final ImmutableSortedSet<DateTime> resaveTimes;
   private final AsyncTaskEnqueuer asyncTaskEnqueuer;
@@ -58,7 +57,7 @@ public class ResaveEntityAction implements Runnable {
 
   @Inject
   ResaveEntityAction(
-      @Parameter(PARAM_RESOURCE_KEY) Key<ImmutableObject> resourceKey,
+      @Parameter(PARAM_RESOURCE_KEY) String resourceKey,
       @Parameter(PARAM_REQUESTED_TIME) DateTime requestedTime,
       @Parameter(PARAM_RESAVE_TIMES) ImmutableSet<DateTime> resaveTimes,
       AsyncTaskEnqueuer asyncTaskEnqueuer,
@@ -76,13 +75,14 @@ public class ResaveEntityAction implements Runnable {
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
     tm().transact(
             () -> {
-              ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
+              ImmutableObject entity = tm().loadByKey(VKey.create(resourceKey));
               tm().put(
                       (entity instanceof EppResource)
                           ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
                           : entity);
               if (!resaveTimes.isEmpty()) {
-                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.create(resourceKey), requestedTime, resaveTimes);
               }
             });
     response.setPayload("Entity re-saved.");

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -55,6 +55,7 @@ import org.joda.time.format.DateTimeFormatter;
     path = SendExpiringCertificateNotificationEmailAction.PATH,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public class SendExpiringCertificateNotificationEmailAction implements Runnable {
+
   public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
   /**
    * Used as an offset when storing the last notification email sent date.
@@ -96,8 +97,13 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   public void run() {
     response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
     try {
-      sendNotificationEmails();
+      int numEmailsSent = sendNotificationEmails();
+      String message =
+          String.format(
+              "Done. Sent %d expiring certificate notification emails in total.", numEmailsSent);
+      logger.atInfo().log(message);
       response.setStatus(SC_OK);
+      response.setPayload(message);
     } catch (Exception e) {
       logger.atWarning().withCause(e).log(
           "Exception thrown when sending expiring certificate notification emails.");
@@ -107,9 +113,11 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   }
 
   /**
-   * Returns a list of registrars that should receive expiring notification emails. There are two
-   * certificates that should be considered (the main certificate and failOver certificate). The
-   * registrars should receive notifications if one of the certificate checks returns true.
+   * Returns a list of registrars that should receive expiring notification emails.
+   *
+   * <p>There are two certificates that should be considered (the main certificate and failOver
+   * certificate). The registrars should receive notifications if one of the certificate checks
+   * returns true.
    */
   @VisibleForTesting
   ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
@@ -151,15 +159,17 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
     }
     try {
       ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      ImmutableSet<InternetAddress> ccs = getEmailAddresses(registrar, Type.ADMIN);
       Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
       logger.atInfo().log(
-          "Registrar %s should receive an email that its %s SSL certificate will expire on %s.",
-          registrar.getRegistrarName(),
+          " %s SSL certificate of registrar '%s' will expire on %s.",
           certificateType.getDisplayName(),
+          registrar.getRegistrarName(),
           expirationDate.toString());
-      if (recipients.isEmpty()) {
+      if (recipients.isEmpty() && ccs.isEmpty()) {
         logger.atWarning().log(
-            "Registrar %s contains no email addresses to receive notification email.",
+            "Registrar %s contains no TECH nor ADMIN email addresses to receive notification"
+                + " email.",
             registrar.getRegistrarName());
         return false;
       }
@@ -174,7 +184,7 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
                       expirationDate,
                       registrar.getRegistrarId()))
               .setRecipients(recipients)
-              .setCcs(getEmailAddresses(registrar, Type.ADMIN))
+              .setCcs(ccs)
               .build());
       /*
        * A duration time offset is used here to ensure that date comparison between two
@@ -243,32 +253,32 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   /** Sends notification emails to registrars with expiring certificates. */
   @VisibleForTesting
   int sendNotificationEmails() {
-    int emailsSent = 0;
+    int numEmailsSent = 0;
     for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
       Registrar registrar = registrarInfo.registrar();
-      if (registrarInfo.isCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringCertNotificationSentDate(),
-            CertificateType.PRIMARY,
-            registrar.getClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringCertNotificationSentDate(),
+              CertificateType.PRIMARY,
+              registrar.getClientCertificate())) {
+        numEmailsSent++;
       }
-      if (registrarInfo.isFailOverCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringFailoverCertNotificationSentDate(),
-            CertificateType.FAILOVER,
-            registrar.getFailoverClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isFailOverCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringFailoverCertNotificationSentDate(),
+              CertificateType.FAILOVER,
+              registrar.getFailoverClientCertificate())) {
+        numEmailsSent++;
       }
     }
-    logger.atInfo().log(
-        "Attempted to send %d expiring certificate notification emails.", emailsSent);
-    return emailsSent;
+    return numEmailsSent;
   }
 
-  /** Returns a list of email addresses of the registrar that should receive a notification email */
+  /**
+   * Returns a list of email addresses of the registrar that should receive a notification email.
+   */
   @VisibleForTesting
   ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
     ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
@@ -327,6 +337,7 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
 
   @AutoValue
   public abstract static class RegistrarInfo {
+
     static RegistrarInfo create(
         Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
       return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(

core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java

@@ -0,0 +1,138 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.contact.ContactHistory;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * An action that wipes out Personal Identifiable Information (PII) fields of {@link ContactHistory}
+ * entities.
+ *
+ * <p>ContactHistory entities should be retained in the database for only certain amount of time.
+ * This periodic wipe out action only applies to SQL.
+ */
+@Action(
+    service = Service.BACKEND,
+    path = WipeOutContactHistoryPiiAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeOutContactHistoryPiiAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/wipeOutContactHistoryPii";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private final Clock clock;
+  private final Response response;
+  private final int minMonthsBeforeWipeOut;
+  private final int wipeOutQueryBatchSize;
+
+  @Inject
+  public WipeOutContactHistoryPiiAction(
+      Clock clock,
+      @Config("minMonthsBeforeWipeOut") int minMonthsBeforeWipeOut,
+      @Config("wipeOutQueryBatchSize") int wipeOutQueryBatchSize,
+      Response response) {
+    this.clock = clock;
+    this.response = response;
+    this.minMonthsBeforeWipeOut = minMonthsBeforeWipeOut;
+    this.wipeOutQueryBatchSize = wipeOutQueryBatchSize;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int totalNumOfWipedEntities = 0;
+      DateTime wipeOutTime = clock.nowUtc().minusMonths(minMonthsBeforeWipeOut);
+      logger.atInfo().log(
+          "About to wipe out all PII of contact history entities prior to %s.", wipeOutTime);
+
+      int numOfWipedEntities = 0;
+      do {
+        numOfWipedEntities =
+            jpaTm()
+                .transact(
+                    () ->
+                        wipeOutContactHistoryData(
+                            getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime)));
+        totalNumOfWipedEntities += numOfWipedEntities;
+      } while (numOfWipedEntities > 0);
+      String msg =
+          String.format(
+              "Done. Wiped out PII of %d ContactHistory entities in total.",
+              totalNumOfWipedEntities);
+      logger.atInfo().log(msg);
+      response.setPayload(msg);
+      response.setStatus(SC_OK);
+
+    } catch (Exception e) {
+      logger.atSevere().withCause(e).log(
+          "Exception thrown during the process of wiping out contact history PII.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(
+          String.format(
+              "Exception thrown during the process of wiping out contact history PII with cause"
+                  + ": %s",
+              e));
+    }
+  }
+
+  /**
+   * Returns a stream of up to {@link #wipeOutQueryBatchSize} {@link ContactHistory} entities
+   * containing PII that are prior to @param wipeOutTime.
+   */
+  @VisibleForTesting
+  Stream<ContactHistory> getNextContactHistoryEntitiesWithPiiBatch(DateTime wipeOutTime) {
+    // email is one of the required fields in EPP, meaning it's initially not null.
+    // Therefore, checking if it's null is one way to avoid processing contact history entities
+    // that have been processed previously. Refer to RFC 5733 for more information.
+    return jpaTm()
+        .query(
+            "FROM ContactHistory WHERE modificationTime < :wipeOutTime " + "AND email IS NOT NULL",
+            ContactHistory.class)
+        .setParameter("wipeOutTime", wipeOutTime)
+        .setMaxResults(wipeOutQueryBatchSize)
+        .getResultStream();
+  }
+
+  /** Wipes out the PII of each of the {@link ContactHistory} entities in the stream. */
+  @VisibleForTesting
+  int wipeOutContactHistoryData(Stream<ContactHistory> contactHistoryEntities) {
+    AtomicInteger numOfEntities = new AtomicInteger(0);
+    contactHistoryEntities.forEach(
+        contactHistoryEntity -> {
+          jpaTm().update(contactHistoryEntity.asBuilder().wipeOutPii().build());
+          numOfEntities.incrementAndGet();
+        });
+    logger.atInfo().log(
+        "Wiped out all PII fields of %d ContactHistory entities.", numOfEntities.get());
+    return numOfEntities.get();
+  }
+}

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
@@ -45,6 +46,7 @@ import javax.inject.Inject;
     service = Action.Service.BACKEND,
     path = "/_dr/task/wipeOutDatastore",
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class WipeoutDatastoreAction implements Runnable {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -92,7 +94,12 @@ public class WipeoutDatastoreAction implements Runnable {
               .setJobName(createJobName("bulk-delete-datastore-", clock))
               .setContainerSpecGcsPath(
                   String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+              .setParameters(
+                  ImmutableMap.of(
+                      "kindsToDelete",
+                      "*",
+                      "registryEnvironment",
+                      RegistryEnvironment.get().name()));
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -0,0 +1,88 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import java.util.List;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+
+/**
+ * A database snapshot shareable by concurrent queries from multiple database clients. A snapshot is
+ * uniquely identified by its {@link #getSnapshotId snapshotId}, and must stay open until all
+ * concurrent queries to this snapshot have attached to it by calling {@link
+ * google.registry.persistence.transaction.JpaTransactionManager#setDatabaseSnapshot}. However, it
+ * can be closed before those queries complete.
+ *
+ * <p>This feature is <em>Postgresql-only</em>.
+ *
+ * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to avoid
+ * exhausting predicate locks at the SERIALIZABLE level.
+ */
+// TODO(b/193662898): vendor-independent support for richer transaction semantics.
+public class DatabaseSnapshot implements AutoCloseable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private String snapshotId;
+  private EntityManager entityManager;
+  private EntityTransaction transaction;
+
+  private DatabaseSnapshot() {}
+
+  public String getSnapshotId() {
+    checkState(entityManager != null, "Snapshot not opened yet.");
+    checkState(entityManager.isOpen(), "Snapshot already closed.");
+    return snapshotId;
+  }
+
+  private DatabaseSnapshot open() {
+    entityManager = jpaTm().getStandaloneEntityManager();
+    transaction = entityManager.getTransaction();
+    transaction.setRollbackOnly();
+    transaction.begin();
+
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+
+    List<?> snapshotIds =
+        entityManager.createNativeQuery("SELECT pg_export_snapshot();").getResultList();
+    checkState(snapshotIds.size() == 1, "Unexpected number of snapshots: %s", snapshotIds.size());
+    snapshotId = (String) snapshotIds.get(0);
+    return this;
+  }
+
+  @Override
+  public void close() {
+    if (transaction != null && transaction.isActive()) {
+      try {
+        transaction.rollback();
+      } catch (Exception e) {
+        logger.atWarning().withCause(e).log("Failed to close a Database Snapshot");
+      }
+    }
+    if (entityManager != null && entityManager.isOpen()) {
+      entityManager.close();
+    }
+  }
+
+  public static DatabaseSnapshot createSnapshot() {
+    return new DatabaseSnapshot().open();
+  }
+}

core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java

@@ -17,7 +17,6 @@ package google.registry.beam.common;
 import static com.google.common.base.Verify.verify;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.model.contact.ContactResource;
 import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -59,18 +58,16 @@ public class JpaDemoPipeline implements Serializable {
                   public void processElement() {
                     // AppEngineEnvironment is needed as long as JPA entity classes still depends
                     // on Objectify.
-                    try (AppEngineEnvironment allowOfyEntity = new AppEngineEnvironment()) {
-                      int result =
-                          (Integer)
-                              jpaTm()
-                                  .transact(
-                                      () ->
-                                          jpaTm()
-                                              .getEntityManager()
-                                              .createNativeQuery("select 1;")
-                                              .getSingleResult());
-                      verify(result == 1, "Expecting 1, got %s.", result);
-                    }
+                    int result =
+                        (Integer)
+                            jpaTm()
+                                .transact(
+                                    () ->
+                                        jpaTm()
+                                            .getEntityManager()
+                                            .createNativeQuery("select 1;")
+                                            .getSingleResult());
+                    verify(result == 1, "Expecting 1, got %s.", result);
                     counter.inc();
                   }
                 }));

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -20,11 +20,9 @@ import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
-import google.registry.model.ofy.ObjectifyService;
 import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -140,6 +138,9 @@ public final class RegistryJpaIO {
 
     abstract Coder<T> coder();
 
+    @Nullable
+    abstract String snapshotId();
+
     abstract Builder<R, T> toBuilder();
 
     @Override
@@ -147,7 +148,9 @@ public final class RegistryJpaIO {
     public PCollection<T> expand(PBegin input) {
       return input
           .apply("Starting " + name(), Create.of((Void) null))
-          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
+          .apply(
+              "Run query for " + name(),
+              ParDo.of(new QueryRunner<>(query(), resultMapper(), snapshotId())))
           .setCoder(coder())
           .apply("Reshuffle", Reshuffle.viaRandomKey());
     }
@@ -164,6 +167,18 @@ public final class RegistryJpaIO {
       return toBuilder().coder(coder).build();
     }
 
+    /**
+     * Specifies the database snapshot to use for this query.
+     *
+     * <p>This feature is <em>Postgresql-only</em>. User is responsible for keeping the snapshot
+     * available until all JVM workers have started using it by calling {@link
+     * JpaTransactionManager#setDatabaseSnapshot}.
+     */
+    // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+    public Read<R, T> withSnapshot(@Nullable String snapshotId) {
+      return toBuilder().snapshotId(snapshotId).build();
+    }
+
     static <R, T> Builder<R, T> builder() {
       return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
           .name(DEFAULT_NAME)
@@ -181,6 +196,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<R, T> coder(Coder coder);
 
+      abstract Builder<R, T> snapshotId(@Nullable String sharedSnapshotId);
+
       abstract Read<R, T> build();
 
       Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
@@ -203,22 +220,28 @@ public final class RegistryJpaIO {
     static class QueryRunner<R, T> extends DoFn<Void, T> {
       private final RegistryQuery<R> query;
       private final SerializableFunction<R, T> resultMapper;
+      // java.util.Optional is not serializable. Use of Guava Optional is discouraged.
+      @Nullable private final String snapshotId;
 
-      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+      QueryRunner(
+          RegistryQuery<R> query,
+          SerializableFunction<R, T> resultMapper,
+          @Nullable String snapshotId) {
         this.query = query;
         this.resultMapper = resultMapper;
+        this.snapshotId = snapshotId;
       }
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
-        // AppEngineEnvironment is need for handling VKeys, which involve Ofy keys. Unlike
-        // SqlBatchWriter, it is unnecessary to initialize ObjectifyService in this class.
-        try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-          // TODO(b/187210388): JpaTransactionManager should support non-transactional query.
-          jpaTm()
-              .transactNoRetry(
-                  () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
-        }
+        jpaTm()
+            .transactNoRetry(
+                () -> {
+                  if (snapshotId != null) {
+                    jpaTm().setDatabaseSnapshot(snapshotId);
+                  }
+                  query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
+                });
       }
     }
   }
@@ -364,16 +387,6 @@ public final class RegistryJpaIO {
       this.withAutoTimestamp = withAutoTimestamp;
     }
 
-    @Setup
-    public void setup() {
-      // AppEngineEnvironment is needed as long as Objectify keys are still involved in the handling
-      // of SQL entities (e.g., in VKeys). ObjectifyService needs to be initialized when conversion
-      // between Ofy entity and Datastore entity is needed.
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ObjectifyService.initOfy();
-      }
-    }
-
     @ProcessElement
     public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
       if (withAutoTimestamp) {
@@ -386,19 +399,17 @@ public final class RegistryJpaIO {
     }
 
     private void actuallyProcessElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ImmutableList<Object> entities =
-            Streams.stream(kv.getValue())
-                .map(this.jpaConverter::apply)
-                // TODO(b/177340730): post migration delete the line below.
-                .filter(Objects::nonNull)
-                .collect(ImmutableList.toImmutableList());
-        try {
-          jpaTm().transact(() -> jpaTm().putAll(entities));
-          counter.inc(entities.size());
-        } catch (RuntimeException e) {
-          processSingly(entities);
-        }
+      ImmutableList<Object> entities =
+          Streams.stream(kv.getValue())
+              .map(this.jpaConverter::apply)
+              // TODO(b/177340730): post migration delete the line below.
+              .filter(Objects::nonNull)
+              .collect(ImmutableList.toImmutableList());
+      try {
+        jpaTm().transact(() -> jpaTm().putAll(entities));
+        counter.inc(entities.size());
+      } catch (RuntimeException e) {
+        processSingly(entities);
       }
     }
 

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -21,7 +21,9 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
+import google.registry.persistence.PersistenceModule.BeamReadOnlyReplicaJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.privileges.secretmanager.SecretManagerModule;
@@ -45,9 +47,26 @@ public interface RegistryPipelineComponent {
   @Config("projectId")
   String getProjectId();
 
+  /** Returns the regular {@link JpaTransactionManager} for general use. */
   @BeamJpaTm
   Lazy<JpaTransactionManager> getJpaTransactionManager();
 
+  /**
+   * Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
+   * ({@link google.registry.model.domain.DomainBase} and {@link
+   * google.registry.model.domain.DomainHistory}). Please refer to {@link
+   * google.registry.model.bulkquery.BulkQueryEntities} for more information.
+   */
+  @BeamBulkQueryJpaTm
+  Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
+
+  /**
+   * A {@link JpaTransactionManager} that uses the Postgres read-only replica if configured (uses
+   * the standard DB otherwise).
+   */
+  @BeamReadOnlyReplicaJpaTm
+  Lazy<JpaTransactionManager> getReadOnlyReplicaJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -34,7 +35,6 @@ import org.apache.beam.sdk.options.Description;
 public interface RegistryPipelineOptions extends GcpOptions {
 
   @Description("The Registry environment.")
-  @Nullable
   RegistryEnvironment getRegistryEnvironment();
 
   void setRegistryEnvironment(RegistryEnvironment environment);
@@ -45,6 +45,12 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setIsolationOverride(TransactionIsolationLevel isolationOverride);
 
+  @Description("The JPA Transaction Manager to use.")
+  @Default.Enum(value = "REGULAR")
+  JpaTransactionManagerType getJpaTransactionManagerType();
+
+  void setJpaTransactionManagerType(JpaTransactionManagerType jpaTransactionManagerType);
+
   @Description("The number of entities to write to the SQL database in one operation.")
   @Default.Integer(20)
   int getSqlWriteBatchSize();

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -20,6 +20,8 @@ import com.google.auto.service.AutoService;
 import com.google.common.flogger.FluentLogger;
 import dagger.Lazy;
 import google.registry.config.RegistryEnvironment;
+import google.registry.config.SystemPropertySetter;
+import google.registry.model.AppEngineEnvironment;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import org.apache.beam.sdk.harness.JvmInitializer;
@@ -35,18 +37,40 @@ import org.apache.beam.sdk.options.PipelineOptions;
 @AutoService(JvmInitializer.class)
 public class RegistryPipelineWorkerInitializer implements JvmInitializer {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  public static final String PROPERTY = "google.registry.beam";
 
   @Override
   public void beforeProcessing(PipelineOptions options) {
     RegistryPipelineOptions registryOptions = options.as(RegistryPipelineOptions.class);
     RegistryEnvironment environment = registryOptions.getRegistryEnvironment();
     if (environment == null || environment.equals(RegistryEnvironment.UNITTEST)) {
-      return;
+      throw new RuntimeException(
+          "A registry environment must be specified in the pipeline options.");
     }
-    logger.atInfo().log("Setting up RegistryEnvironment: %s", environment);
+    logger.atInfo().log("Setting up RegistryEnvironment %s.", environment);
     environment.setup();
-    Lazy<JpaTransactionManager> transactionManagerLazy =
-        toRegistryPipelineComponent(registryOptions).getJpaTransactionManager();
+    RegistryPipelineComponent registryPipelineComponent =
+        toRegistryPipelineComponent(registryOptions);
+    Lazy<JpaTransactionManager> transactionManagerLazy;
+    switch (registryOptions.getJpaTransactionManagerType()) {
+      case BULK_QUERY:
+        transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
+        break;
+      case READ_ONLY_REPLICA:
+        transactionManagerLazy =
+            registryPipelineComponent.getReadOnlyReplicaJpaTransactionManager();
+        break;
+      case REGULAR:
+      default:
+        transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
+    }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
+    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // loads all ofy entities.
+    new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
+        .setEnvironmentForAllThreads();
+    // Set the system property so that we can call IdService.allocateId() without access to
+    // datastore.
+    SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
   }
 }

core/src/main/java/google/registry/beam/comparedb/DatastoreSnapshots.java

@@ -0,0 +1,193 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.initsql.Transforms.createTagForKind;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.Transforms;
+import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.joda.time.DateTime;
+
+/** Utilities for loading Datastore snapshots. */
+@DeleteAfterMigration
+public final class DatastoreSnapshots {
+
+  private DatastoreSnapshots() {}
+
+  /**
+   * Datastore kinds eligible for validation. This set must be consistent with {@link
+   * SqlSnapshots#ALL_SQL_ENTITIES}.
+   */
+  @VisibleForTesting
+  static final ImmutableSet<Class<?>> ALL_DATASTORE_KINDS =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          HistoryEntry.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class);
+
+  /**
+   * Returns the Datastore snapshot right before {@code commitLogToTime} for the user specified
+   * {@code kinds}. The resulting snapshot has all changes that happened before {@code
+   * commitLogToTime}, and none at or after {@code commitLogToTime}.
+   *
+   * <p>If {@code HistoryEntry} is included in {@code kinds}, the result will contain {@code
+   * PCollections} for the child entities, {@code DomainHistory}, {@code ContactHistory}, and {@code
+   * HostHistory}.
+   */
+  static PCollectionTuple loadDatastoreSnapshotByKind(
+      Pipeline pipeline,
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<Class<?>> kinds,
+      Optional<DateTime> compareStartTime) {
+    PCollectionTuple snapshot =
+        pipeline.apply(
+            "Load Datastore snapshot.",
+            Transforms.loadDatastoreSnapshot(
+                exportDir,
+                commitLogDir,
+                commitLogFromTime,
+                commitLogToTime,
+                kinds.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet())));
+
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<?> kind : kinds) {
+      PCollection<VersionedEntity> perKindSnapshot =
+          snapshot.get(createTagForKind(Key.getKind(kind)));
+      if (SqlEntity.class.isAssignableFrom(kind)) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag((Class<? extends SqlEntity>) kind),
+                datastoreEntityToPojo(perKindSnapshot, kind.getSimpleName(), compareStartTime));
+        continue;
+      }
+      Verify.verify(kind == HistoryEntry.class, "Unexpected Non-SqlEntity class: %s", kind);
+      PCollectionTuple historyEntriesByType = splitHistoryEntry(perKindSnapshot, compareStartTime);
+      for (Map.Entry<TupleTag<?>, PCollection<?>> entry :
+          historyEntriesByType.getAll().entrySet()) {
+        perTypeSnapshots = perTypeSnapshots.and(entry.getKey().getId(), entry.getValue());
+      }
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Splits a {@link PCollection} of {@link HistoryEntry HistoryEntries} into three collections of
+   * its child entities by type.
+   */
+  static PCollectionTuple splitHistoryEntry(
+      PCollection<VersionedEntity> historyEntries, Optional<DateTime> compareStartTime) {
+    DateTime nullableStartTime = compareStartTime.orElse(null);
+    return historyEntries.apply(
+        "Split HistoryEntry by Resource Type",
+        ParDo.of(
+                new DoFn<VersionedEntity, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity historyEntry, MultiOutputReceiver out) {
+                    Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(historyEntry))
+                        .filter(e -> isEntityIncludedForComparison(e, nullableStartTime))
+                        .ifPresent(
+                            sqlEntity ->
+                                out.get(createSqlEntityTupleTag(sqlEntity.getClass()))
+                                    .output(sqlEntity));
+                  }
+                })
+            .withOutputTags(
+                createSqlEntityTupleTag(DomainHistory.class),
+                TupleTagList.of(createSqlEntityTupleTag(ContactHistory.class))
+                    .and(createSqlEntityTupleTag(HostHistory.class))));
+  }
+
+  /**
+   * Transforms a {@link PCollection} of {@link VersionedEntity VersionedEntities} to Ofy Java
+   * objects.
+   */
+  static PCollection<SqlEntity> datastoreEntityToPojo(
+      PCollection<VersionedEntity> entities, String desc, Optional<DateTime> compareStartTime) {
+    DateTime nullableStartTime = compareStartTime.orElse(null);
+    return entities.apply(
+        "Datastore Entity to Pojo " + desc,
+        ParDo.of(
+            new DoFn<VersionedEntity, SqlEntity>() {
+              @ProcessElement
+              public void processElement(
+                  @Element VersionedEntity entity, OutputReceiver<SqlEntity> out) {
+                Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(entity))
+                    .filter(e -> isEntityIncludedForComparison(e, nullableStartTime))
+                    .ifPresent(out::output);
+              }
+            }));
+  }
+
+  static boolean isEntityIncludedForComparison(
+      SqlEntity entity, @Nullable DateTime compareStartTime) {
+    if (compareStartTime == null) {
+      return true;
+    }
+    if (entity instanceof HistoryEntry) {
+      return compareStartTime.isBefore(((HistoryEntry) entity).getModificationTime());
+    }
+    if (entity instanceof EppResource) {
+      return compareStartTime.isBefore(((EppResource) entity).getUpdateTimestamp().getTimestamp());
+    }
+    return true;
+  }
+}

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -0,0 +1,154 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import dagger.Component;
+import google.registry.config.CloudTasksUtilsModule;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.util.Clock;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.joda.time.DateTime;
+import org.joda.time.Instant;
+import org.joda.time.Interval;
+
+/** Finds the necessary information for loading the most recent Datastore snapshot. */
+@DeleteAfterMigration
+public class LatestDatastoreSnapshotFinder {
+  private final String projectId;
+  private final GcsUtils gcsUtils;
+  private final Clock clock;
+
+  @Inject
+  LatestDatastoreSnapshotFinder(
+      @Config("projectId") String projectId, GcsUtils gcsUtils, Clock clock) {
+    this.projectId = projectId;
+    this.gcsUtils = gcsUtils;
+    this.clock = clock;
+  }
+
+  /**
+   * Finds information of the most recent Datastore snapshot that ends strictly before {@code
+   * exportEndTimeUpperBound}, including the GCS folder of the exported data files and the start and
+   * stop times of the export. The folder of the CommitLogs is also included in the return.
+   */
+  public DatastoreSnapshotInfo getSnapshotInfo(Instant exportEndTimeUpperBound) {
+    String bucketName = RegistryConfig.getDatastoreBackupsBucket().substring("gs://".length());
+    /**
+     * Find the bucket-relative path to the overall metadata file of the last Datastore export.
+     * Since Datastore export is saved daily, we may need to look back to yesterday. If found, the
+     * return value is like
+     * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
+     */
+    Optional<String> metaFilePathOptional =
+        findNewestExportMetadataFileBeforeTime(bucketName, exportEndTimeUpperBound, 2);
+    if (!metaFilePathOptional.isPresent()) {
+      throw new NoSuchElementException("No exports found over the past 2 days.");
+    }
+    String metaFilePath = metaFilePathOptional.get();
+    String metaFileFolder = metaFilePath.substring(0, metaFilePath.indexOf('/'));
+    Instant exportStartTime = Instant.parse(metaFileFolder.replace('_', '.') + 'Z');
+    BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath));
+    Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+    return DatastoreSnapshotInfo.create(
+        String.format("gs://%s/%s", bucketName, metaFileFolder),
+        getCommitLogDir(),
+        new Interval(exportStartTime, exportEndTime));
+  }
+
+  public String getCommitLogDir() {
+    return "gs://" + projectId + "-commits";
+  }
+
+  /**
+   * Finds the latest Datastore export that ends strictly before {@code endTimeUpperBound} and
+   * returns the bucket-relative path of the overall export metadata file, in the given bucket. The
+   * search goes back for up to {@code lookBackDays} days in time, including today.
+   *
+   * <p>The overall export metadata file is the last file created during a Datastore export. All
+   * data has been exported by the creation time of this file. The name of this file, like that of
+   * all files in the same export, begins with the timestamp when the export starts.
+   *
+   * <p>An example return value: {@code
+   * 2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata}.
+   */
+  private Optional<String> findNewestExportMetadataFileBeforeTime(
+      String bucketName, Instant endTimeUpperBound, int lookBackDays) {
+    DateTime today = clock.nowUtc();
+    for (int day = 0; day < lookBackDays; day++) {
+      String dateString = today.minusDays(day).toString("yyyy-MM-dd");
+      try {
+        Optional<String> metaFilePath =
+            gcsUtils.listFolderObjects(bucketName, dateString).stream()
+                .filter(s -> s.endsWith("overall_export_metadata"))
+                .map(s -> dateString + s)
+                .sorted(Comparator.<String>naturalOrder().reversed())
+                .findFirst();
+        if (metaFilePath.isPresent()) {
+          BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath.get()));
+          Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+          if (exportEndTime.isBefore(endTimeUpperBound)) {
+            return metaFilePath;
+          }
+        }
+      } catch (IOException ioe) {
+        throw new RuntimeException(ioe);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /** Holds information about a Datastore snapshot. */
+  @AutoValue
+  abstract static class DatastoreSnapshotInfo {
+    abstract String exportDir();
+
+    abstract String commitLogDir();
+
+    abstract Interval exportInterval();
+
+    static DatastoreSnapshotInfo create(
+        String exportDir, String commitLogDir, Interval exportOperationInterval) {
+      return new AutoValue_LatestDatastoreSnapshotFinder_DatastoreSnapshotInfo(
+          exportDir, commitLogDir, exportOperationInterval);
+    }
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
+  interface LatestDatastoreSnapshotFinderFinderComponent {
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotInfoFinder();
+  }
+}

core/src/main/java/google/registry/beam/comparedb/SqlSnapshots.java

@@ -0,0 +1,539 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.comparedb.ValidateSqlUtils.getMedianIdForHistoryTable;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Strings;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Streams;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryJpaIO.Read;
+import google.registry.model.EppResource;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.model.bulkquery.DomainBaseLite;
+import google.registry.model.bulkquery.DomainHistoryHost;
+import google.registry.model.bulkquery.DomainHistoryLite;
+import google.registry.model.bulkquery.DomainHost;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import google.registry.util.DateTimeUtils;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.persistence.Entity;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.joda.time.DateTime;
+
+/**
+ * Utilities for loading SQL snapshots.
+ *
+ * <p>For {@link DomainBase} and {@link DomainHistory}, this class assumes the presence of the
+ * {@link google.registry.persistence.PersistenceModule.JpaTransactionManagerType#BULK_QUERY
+ * bulk-query-capable JpaTransactionManager}, and takes advantage of it for higher throughput.
+ *
+ * <p>For now this class is meant for use during the database migration period only. Therefore, it
+ * contains optimizations specifically for the production database at the current size, e.g.,
+ * parallel queries for select tables.
+ */
+@DeleteAfterMigration
+public final class SqlSnapshots {
+
+  private SqlSnapshots() {}
+
+  /**
+   * SQL entity types that are eligible for validation. This set must be consistent with {@link
+   * DatastoreSnapshots#ALL_DATASTORE_KINDS}.
+   */
+  static final ImmutableSet<Class<? extends SqlEntity>> ALL_SQL_ENTITIES =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class,
+          ContactHistory.class,
+          HostHistory.class,
+          DomainHistory.class);
+
+  /**
+   * Loads a SQL snapshot for the given {@code sqlEntityTypes}.
+   *
+   * <p>If {@code snapshotId} is present, all queries use the specified database snapshot,
+   * guaranteeing a consistent result.
+   */
+  public static PCollectionTuple loadCloudSqlSnapshotByType(
+      Pipeline pipeline,
+      ImmutableSet<Class<? extends SqlEntity>> sqlEntityTypes,
+      Optional<String> snapshotId,
+      Optional<DateTime> compareStartTime) {
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<? extends SqlEntity> clazz : sqlEntityTypes) {
+      if (clazz == DomainBase.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainBase.class),
+                loadAndAssembleDomainBase(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == DomainHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainHistory.class),
+                loadAndAssembleDomainHistory(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == ContactHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(ContactHistory.class),
+                loadContactHistory(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == HostHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(HostHistory.class),
+                loadHostHistory(
+                    pipeline, snapshotId, compareStartTime.orElse(DateTimeUtils.START_OF_TIME)));
+        continue;
+      }
+      if (EppResource.class.isAssignableFrom(clazz) && compareStartTime.isPresent()) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(clazz),
+                pipeline.apply(
+                    "SQL Load " + clazz.getSimpleName(),
+                    buildEppResourceQueryWithTimeFilter(
+                            clazz, SqlEntity.class, snapshotId, compareStartTime.get())
+                        .withSnapshot(snapshotId.orElse(null))));
+        continue;
+      }
+      perTypeSnapshots =
+          perTypeSnapshots.and(
+              createSqlEntityTupleTag(clazz),
+              pipeline.apply(
+                  "SQL Load " + clazz.getSimpleName(),
+                  RegistryJpaIO.read(
+                          () -> CriteriaQueryBuilder.create(clazz).build(), SqlEntity.class::cast)
+                      .withSnapshot(snapshotId.orElse(null))));
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Bulk-loads parts of {@link DomainBase} and assembles them in the pipeline.
+   *
+   * @see BulkQueryEntities
+   */
+  public static PCollection<SqlEntity> loadAndAssembleDomainBase(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PCollection<KV<String, Serializable>> baseObjects =
+        readAllAndAssignKey(
+            pipeline,
+            DomainBaseLite.class,
+            DomainBaseLite::getRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriod.class,
+            GracePeriod::getDomainRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DelegationSignerData.class,
+            DelegationSignerData::getDomainRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline, DomainHost.class, DomainHost::getDomainRepoId, snapshotId, compareStartTime);
+
+    DateTime nullableCompareStartTime = compareStartTime.orElse(null);
+    return PCollectionList.of(
+            ImmutableList.of(baseObjects, gracePeriods, delegationSigners, domainHosts))
+        .apply("SQL Merge DomainBase parts", Flatten.pCollections())
+        .apply("Group by Domain Parts by RepoId", GroupByKey.create())
+        .apply(
+            "Assemble DomainBase",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainBaseLite> baseObjects =
+                        partsByType.getAllOf(DomainBaseLite.class);
+                    if (nullableCompareStartTime != null) {
+                      Verify.verify(
+                          baseObjects.size() <= 1,
+                          "Found duplicate DomainBaseLite object per repoId: " + kv.getKey());
+                      if (baseObjects.isEmpty()) {
+                        return;
+                      }
+                    }
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainBaseLite object per repoId: " + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainBase(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(GracePeriod.class),
+                            partsByType.getAllOf(DelegationSignerData.class),
+                            partsByType.getAllOf(DomainHost.class).stream()
+                                .map(DomainHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet())));
+                  }
+                }));
+  }
+
+  /**
+   * Loads all {@link ContactHistory} entities from the database.
+   *
+   * <p>This method uses two queries to load data in parallel. This is a performance optimization
+   * specifically for the production database.
+   */
+  static PCollection<SqlEntity> loadContactHistory(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PartitionedQuery partitionedQuery =
+        buildPartitonedHistoryQuery(ContactHistory.class, compareStartTime);
+    PCollection<SqlEntity> part1 =
+        pipeline.apply(
+            "SQL Load ContactHistory first half",
+            RegistryJpaIO.read(
+                    partitionedQuery.firstHalfQuery(),
+                    partitionedQuery.parameters(),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    PCollection<SqlEntity> part2 =
+        pipeline.apply(
+            "SQL Load ContactHistory second half",
+            RegistryJpaIO.read(
+                    partitionedQuery.secondHalfQuery(),
+                    partitionedQuery.parameters(),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    return PCollectionList.of(part1)
+        .and(part2)
+        .apply("Combine ContactHistory parts", Flatten.pCollections());
+  }
+
+  /** Loads all {@link HostHistory} entities from the database. */
+  static PCollection<SqlEntity> loadHostHistory(
+      Pipeline pipeline, Optional<String> snapshotId, DateTime compareStartTime) {
+    return pipeline.apply(
+        "SQL Load HostHistory",
+        RegistryJpaIO.read(
+                "select c from HostHistory c where :compareStartTime  < modificationTime",
+                ImmutableMap.of("compareStartTime", compareStartTime),
+                false,
+                SqlEntity.class::cast)
+            .withSnapshot(snapshotId.orElse(null)));
+  }
+
+  /**
+   * Bulk-loads all parts of {@link DomainHistory} and assembles them in the pipeline.
+   *
+   * <p>This method uses two queries to load {@link DomainBaseLite} in parallel. This is a
+   * performance optimization specifically for the production database.
+   *
+   * @see BulkQueryEntities
+   */
+  static PCollection<SqlEntity> loadAndAssembleDomainHistory(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PartitionedQuery partitionedQuery =
+        buildPartitonedHistoryQuery(DomainHistoryLite.class, compareStartTime);
+    PCollection<KV<String, Serializable>> baseObjectsPart1 =
+        queryAndAssignKey(
+            pipeline,
+            "first half",
+            partitionedQuery.firstHalfQuery(),
+            partitionedQuery.parameters(),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> baseObjectsPart2 =
+        queryAndAssignKey(
+            pipeline,
+            "second half",
+            partitionedQuery.secondHalfQuery(),
+            partitionedQuery.parameters(),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriodHistory.class,
+            compose(GracePeriodHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DomainDsDataHistory.class,
+            compose(DomainDsDataHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline,
+            DomainHistoryHost.class,
+            compose(DomainHistoryHost::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> transactionRecords =
+        readAllAndAssignKey(
+            pipeline,
+            DomainTransactionRecord.class,
+            compose(DomainTransactionRecord::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+
+    DateTime nullableCompareStartTime = compareStartTime.orElse(null);
+    return PCollectionList.of(
+            ImmutableList.of(
+                baseObjectsPart1,
+                baseObjectsPart2,
+                gracePeriods,
+                delegationSigners,
+                domainHosts,
+                transactionRecords))
+        .apply("Merge DomainHistory parts", Flatten.pCollections())
+        .apply("Group by DomainHistory Parts by DomainHistoryId string", GroupByKey.create())
+        .apply(
+            "Assemble DomainHistory",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainHistoryLite> baseObjects =
+                        partsByType.getAllOf(DomainHistoryLite.class);
+                    if (nullableCompareStartTime != null) {
+                      Verify.verify(
+                          baseObjects.size() <= 1,
+                          "Found duplicate DomainHistoryLite object per domainHistoryId: "
+                              + kv.getKey());
+                      if (baseObjects.isEmpty()) {
+                        return;
+                      }
+                    }
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainHistoryLite object per domainHistoryId: "
+                            + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainHistory(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(DomainDsDataHistory.class),
+                            partsByType.getAllOf(DomainHistoryHost.class).stream()
+                                .map(DomainHistoryHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet()),
+                            partsByType.getAllOf(GracePeriodHistory.class),
+                            partsByType.getAllOf(DomainTransactionRecord.class)));
+                  }
+                }));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> readAllAndAssignKey(
+      Pipeline pipeline,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId,
+      Optional<DateTime> compareStartTime) {
+    Read<R, R> queryObject;
+    if (compareStartTime.isPresent() && EppResource.class.isAssignableFrom(type)) {
+      queryObject =
+          buildEppResourceQueryWithTimeFilter(type, type, snapshotId, compareStartTime.get());
+    } else {
+      queryObject =
+          RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(type).build())
+              .withSnapshot(snapshotId.orElse(null));
+    }
+    return pipeline
+        .apply("SQL Load " + type.getSimpleName(), queryObject)
+        .apply(
+            "Assign Key to " + type.getSimpleName(),
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> queryAndAssignKey(
+      Pipeline pipeline,
+      String diffrentiator,
+      String jplQuery,
+      ImmutableMap<String, Object> queryParameters,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName() + " " + diffrentiator,
+            RegistryJpaIO.read(jplQuery, queryParameters, false, type::cast)
+                .withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName() + " " + diffrentiator,
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  // TODO(b/205988530): don't use beam serializablefunction, make one that extends Java's Function.
+  private static <R, I, T> SerializableFunction<R, T> compose(
+      SerializableFunction<R, I> f1, SerializableFunction<I, T> f2) {
+    return r -> f2.apply(f1.apply(r));
+  }
+
+  static <R, T> Read<R, T> buildEppResourceQueryWithTimeFilter(
+      Class<R> entityType,
+      Class<T> castOutputAsType,
+      Optional<String> snapshotId,
+      DateTime compareStartTime) {
+    String tableName = getJpaEntityName(entityType);
+    String jpql =
+        String.format("select c from %s c where :compareStartTime < updateTimestamp", tableName);
+    return RegistryJpaIO.read(
+            jpql,
+            ImmutableMap.of("compareStartTime", UpdateAutoTimestamp.create(compareStartTime)),
+            false,
+            (R x) -> castOutputAsType.cast(x))
+        .withSnapshot(snapshotId.orElse(null));
+  }
+
+  static PartitionedQuery buildPartitonedHistoryQuery(
+      Class<?> entityType, Optional<DateTime> compareStartTime) {
+    String tableName = getJpaEntityName(entityType);
+    Verify.verify(
+        !Strings.isNullOrEmpty(tableName), "Invalid entity type %s", entityType.getSimpleName());
+    long medianId =
+        getMedianIdForHistoryTable(tableName)
+            .orElseThrow(() -> new IllegalStateException("Not a valid database: no " + tableName));
+    String firstHalfQuery = String.format("select c from %s c where id <= :historyId", tableName);
+    String secondHalfQuery = String.format("select c from %s c where id > :historyId", tableName);
+    if (compareStartTime.isPresent()) {
+      String timeFilter = "  and :compareStartTime  < modificationTime";
+      firstHalfQuery += timeFilter;
+      secondHalfQuery += timeFilter;
+      return PartitionedQuery.createPartitionedQuery(
+          firstHalfQuery,
+          secondHalfQuery,
+          ImmutableMap.of("historyId", medianId, "compareStartTime", compareStartTime.get()));
+    } else {
+      return PartitionedQuery.createPartitionedQuery(
+          firstHalfQuery, secondHalfQuery, ImmutableMap.of("historyId", medianId));
+    }
+  }
+
+  private static String getJpaEntityName(Class entityType) {
+    Entity entityAnnotation = (Entity) entityType.getAnnotation(Entity.class);
+    checkState(
+        entityAnnotation != null, "Unexpected non-entity type %s", entityType.getSimpleName());
+    return Strings.isNullOrEmpty(entityAnnotation.name())
+        ? entityType.getSimpleName()
+        : entityAnnotation.name();
+  }
+
+  /** Contains two queries that partition the target table in two. */
+  @AutoValue
+  abstract static class PartitionedQuery {
+    abstract String firstHalfQuery();
+
+    abstract String secondHalfQuery();
+
+    abstract ImmutableMap<String, Object> parameters();
+
+    public static PartitionedQuery createPartitionedQuery(
+        String firstHalfQuery, String secondHalfQuery, ImmutableMap<String, Object> parameters) {
+      return new AutoValue_SqlSnapshots_PartitionedQuery(
+          firstHalfQuery, secondHalfQuery, parameters);
+    }
+  }
+
+  /** Container that receives mixed-typed data and groups them by {@link Class}. */
+  static class TypedClassifier {
+    private final ImmutableSetMultimap<Class<?>, Object> classifiedEntities;
+
+    TypedClassifier(Iterable<Serializable> inputs) {
+      this.classifiedEntities =
+          Streams.stream(inputs)
+              .collect(ImmutableSetMultimap.toImmutableSetMultimap(Object::getClass, x -> x));
+    }
+
+    <T> ImmutableSet<T> getAllOf(Class<T> clazz) {
+      return classifiedEntities.get(clazz).stream()
+          .map(clazz::cast)
+          .collect(ImmutableSet.toImmutableSet());
+    }
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipeline.java

@@ -0,0 +1,83 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.joda.time.DateTime;
+
+/**
+ * Validates the asynchronous data replication process from Cloud SQL (primary) to Datastore
+ * (secondary).
+ *
+ * <p>This pipeline simply compares the snapshots provided by an invoker, which is responsible for
+ * obtaining two consistent snapshots for the same point of time.
+ */
+// TODO(weiminyu): Implement the invoker action in a followup PR.
+@DeleteAfterMigration
+public class ValidateDatastorePipeline {
+
+  private final ValidateDatastorePipelineOptions options;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+
+  public ValidateDatastorePipeline(
+      ValidateDatastorePipelineOptions options,
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
+    this.options = options;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+  }
+
+  void run(Pipeline pipeline) {
+    DateTime latestCommitLogTime = DateTime.parse(options.getLatestCommitLogTimestamp());
+    DatastoreSnapshotInfo mostRecentExport =
+        datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
+
+    ValidateSqlPipeline.setupPipeline(
+        pipeline,
+        Optional.ofNullable(options.getSqlSnapshotId()),
+        mostRecentExport,
+        latestCommitLogTime,
+        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
+
+    pipeline.run();
+  }
+
+  public static void main(String[] args) {
+    ValidateDatastorePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args)
+            .withValidation()
+            .as(ValidateDatastorePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+    new ValidateDatastorePipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipelineOptions.java

@@ -0,0 +1,39 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.model.annotations.DeleteAfterMigration;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Validation;
+
+/** BEAM pipeline options for {@link ValidateDatastorePipelineOptions}. */
+@DeleteAfterMigration
+public interface ValidateDatastorePipelineOptions extends ValidateSqlPipelineOptions {
+
+  @Description(
+      "The id of the SQL snapshot to be compared with Datastore. "
+          + "If null, the current state of the SQL database is used.")
+  @Nullable
+  String getSqlSnapshotId();
+
+  void setSqlSnapshotId(String snapshotId);
+
+  @Description("The latest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getLatestCommitLogTimestamp();
+
+  void setLatestCommitLogTimestamp(String commitLogEndTimestamp);
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -0,0 +1,264 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.model.server.Lock;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import google.registry.util.RequestStatusChecker;
+import google.registry.util.SystemClock;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
+ * (secondary storage).
+ */
+@DeleteAfterMigration
+public class ValidateSqlPipeline {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final Duration COMMITLOG_START_TIME_MARGIN = Duration.standardMinutes(10);
+
+  /**
+   * Name of the lock used by the commitlog replay process.
+   *
+   * <p>See {@link google.registry.backup.ReplayCommitLogsToSqlAction} for more information.
+   */
+  private static final String COMMITLOG_REPLAY_LOCK_NAME = "ReplayCommitLogsToSqlAction";
+
+  private static final Duration REPLAY_LOCK_LEASE_LENGTH = Duration.standardHours(1);
+  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_TIMEOUT =
+      java.time.Duration.ofMinutes(6);
+  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_DELAY =
+      java.time.Duration.ofSeconds(30);
+
+  private final ValidateSqlPipelineOptions options;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+
+  public ValidateSqlPipeline(
+      ValidateSqlPipelineOptions options, LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
+    this.options = options;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    Optional<Lock> lock = acquireCommitLogReplayLock();
+    if (lock.isPresent()) {
+      logger.atInfo().log("Acquired CommitLog Replay lock.");
+    } else {
+      throw new RuntimeException("Failed to acquire CommitLog Replay lock.");
+    }
+
+    try {
+      DateTime latestCommitLogTime =
+          TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+      DatastoreSnapshotInfo mostRecentExport =
+          datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
+      Preconditions.checkState(
+          latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
+          "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
+      try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+        // Eagerly release the commitlog replay lock so that replay can resume.
+        lock.ifPresent(Lock::releaseSql);
+        lock = Optional.empty();
+
+        logger.atInfo().log(
+            "Starting comparison with export at %s and latestCommitLogTime at %s",
+            mostRecentExport.exportDir(), latestCommitLogTime);
+
+        setupPipeline(
+            pipeline,
+            Optional.of(databaseSnapshot.getSnapshotId()),
+            mostRecentExport,
+            latestCommitLogTime,
+            Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
+        State state = pipeline.run().waitUntilFinish();
+        if (!State.DONE.equals(state)) {
+          throw new IllegalStateException("Unexpected pipeline state: " + state);
+        }
+      }
+    } finally {
+      lock.ifPresent(Lock::releaseSql);
+    }
+  }
+
+  static void setupPipeline(
+      Pipeline pipeline,
+      Optional<String> sqlSnapshotId,
+      DatastoreSnapshotInfo mostRecentExport,
+      DateTime latestCommitLogTime,
+      Optional<DateTime> compareStartTime) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minus(COMMITLOG_START_TIME_MARGIN),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS,
+            compareStartTime);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId, compareStartTime);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      PCollectionList.of(datastoreSnapshot.get(tag))
+          .and(cloudSqlSnapshot.get(tag))
+          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
+          .apply(
+              "Assign primary key to merged " + clazz.getSimpleName(),
+              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
+          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  private static Optional<Lock> acquireCommitLogReplayLock() {
+    Stopwatch stopwatch = Stopwatch.createStarted();
+    while (stopwatch.elapsed().minus(REPLAY_LOCK_ACQUIRE_TIMEOUT).isNegative()) {
+      Optional<Lock> lock = tryAcquireCommitLogReplayLock();
+      if (lock.isPresent()) {
+        return lock;
+      }
+      logger.atInfo().log("Failed to acquired CommitLog Replay lock. Will retry...");
+      try {
+        Thread.sleep(REPLAY_LOCK_ACQUIRE_DELAY.toMillis());
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new RuntimeException("Interrupted.");
+      }
+    }
+    return Optional.empty();
+  }
+
+  private static Optional<Lock> tryAcquireCommitLogReplayLock() {
+    return Lock.acquireSql(
+        COMMITLOG_REPLAY_LOCK_NAME,
+        null,
+        REPLAY_LOCK_LEASE_LENGTH,
+        getLockingRequestStatusChecker(),
+        false);
+  }
+
+  /**
+   * Returns a fake implementation of {@link RequestStatusChecker} that is required for lock
+   * acquisition. The default implementation is AppEngine-specific and is unusable on GCE.
+   */
+  private static RequestStatusChecker getLockingRequestStatusChecker() {
+    return new RequestStatusChecker() {
+      @Override
+      public String getLogId() {
+        return "ValidateSqlPipeline";
+      }
+
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+          DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+              .datastoreSnapshotInfoFinder();
+
+      @Override
+      public boolean isRunning(String requestLogId) {
+        return true;
+      }
+    };
+  }
+
+  public static void main(String[] args) {
+    ValidateSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    MigrationState state =
+        DatabaseMigrationStateSchedule.getValueAtTime(new SystemClock().nowUtc());
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      throw new IllegalStateException("This pipeline is not designed for migration phase " + state);
+    }
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+
+    new ValidateSqlPipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -0,0 +1,34 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.model.annotations.DeleteAfterMigration;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.options.Description;
+
+/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
+@DeleteAfterMigration
+public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {
+
+  @Description(
+      "For history entries and EPP resources, only those modified strictly after this time are "
+          + "included in comparison. Value is in ISO8601 format. "
+          + "Other entity types are not affected.")
+  @Nullable
+  String getComparisonStartTimestamp();
+
+  void setComparisonStartTimestamp(String comparisonStartTimestamp);
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -0,0 +1,387 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.initsql.Transforms;
+import google.registry.config.RegistryEnvironment;
+import google.registry.model.BackupGroupRoot;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.contact.ContactBase;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.eppcommon.AuthInfo;
+import google.registry.model.host.HostHistory;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import java.lang.reflect.Field;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Helpers for use by {@link ValidateSqlPipeline}. */
+@DeleteAfterMigration
+final class ValidateSqlUtils {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ValidateSqlUtils() {}
+
+  private static final ImmutableSet<String> PROBER_CELLS = ImmutableSet.of("IQ", "LG", "TL");
+  private static final ImmutableSet<String> PROBER_TYPES =
+      ImmutableSet.of("ANYT", "ANYTES", "CANARY");
+
+  /**
+   * Query template for finding the median value of the {@code history_revision_id} column in one of
+   * the History tables.
+   *
+   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * history tables. Although the {@code repo_id} column is the leading column in the primary keys
+   * of these tables, in practice and with production data, division by {@code history_revision_id}
+   * works slightly faster for unknown reasons.
+   */
+  private static final String MEDIAN_ID_QUERY_TEMPLATE =
+      "SELECT history_revision_id FROM (                                                        "
+          + "  SELECT"
+          + "    ROW_NUMBER() OVER (ORDER BY history_revision_id ASC) AS rownumber,"
+          + "    history_revision_id"
+          + "  FROM \"%TABLE%\""
+          + ") AS foo\n"
+          + "WHERE rownumber in (select count(*) / 2 + 1 from \"%TABLE%\")";
+
+  static Optional<Long> getMedianIdForHistoryTable(String tableName) {
+    Preconditions.checkArgument(
+        tableName.endsWith("History"), "Table must be one of the History tables.");
+    String sqlText = MEDIAN_ID_QUERY_TEMPLATE.replace("%TABLE%", tableName);
+    List results =
+        jpaTm()
+            .transact(() -> jpaTm().getEntityManager().createNativeQuery(sqlText).getResultList());
+    verify(results.size() < 2, "MidPoint query should have at most one result.");
+    if (results.isEmpty()) {
+      return Optional.empty();
+    }
+    return Optional.of(((BigInteger) results.get(0)).longValue());
+  }
+
+  static TupleTag<SqlEntity> createSqlEntityTupleTag(Class<? extends SqlEntity> actualType) {
+    return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
+  }
+
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+    private final HashMap<String, Counter> totalCounters = new HashMap<>();
+    private final HashMap<String, Counter> missingCounters = new HashMap<>();
+    private final HashMap<String, Counter> unequalCounters = new HashMap<>();
+    private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
+
+    private volatile boolean logPrinted = false;
+
+    private String getCounterKey(Class<?> clazz) {
+      return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
+    }
+
+    private synchronized void ensureCounterExists(String counterKey) {
+      if (totalCounters.containsKey(counterKey)) {
+        return;
+      }
+      totalCounters.put(counterKey, Metrics.counter("CompareDB", "Total Compared: " + counterKey));
+      missingCounters.put(
+          counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
+      unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
+      badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
+    }
+
+    /**
+     * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
+     * This will be removed when we start exporting such entities to GCS.
+     */
+    void logDiff(String key, Object entry0, Object entry1) {
+      if (logPrinted) {
+        return;
+      }
+      logPrinted = true;
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      StringBuilder sb = new StringBuilder();
+      fields0.forEach(
+          (field, value) -> {
+            if (fields1.containsKey(field)) {
+              if (!Objects.equals(value, fields1.get(field))) {
+                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
+              }
+            } else {
+              sb.append(field + "Not found in entity 2\n");
+            }
+          });
+      fields1.forEach(
+          (field, value) -> {
+            if (!fields0.containsKey(field)) {
+              sb.append(field + "Not found in entity 1\n");
+            }
+          });
+      logger.atWarning().log(key + "  " + sb.toString());
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+      ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
+
+      verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
+      verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
+
+      String counterKey = getCounterKey(entities.get(0).getClass());
+      ensureCounterExists(counterKey);
+      totalCounters.get(counterKey).inc();
+
+      if (entities.size() == 1) {
+        if (isSpecialCaseProberEntity(entities.get(0))) {
+          return;
+        }
+        missingCounters.get(counterKey).inc();
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
+        }
+        return;
+      }
+      SqlEntity entity0;
+      SqlEntity entity1;
+
+      try {
+        entity0 = normalizeEntity(entities.get(0));
+        entity1 = normalizeEntity(entities.get(1));
+      } catch (Exception e) {
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          badEntityCounters.get(counterKey).inc();
+        }
+        return;
+      }
+
+      if (!Objects.equals(entity0, entity1)) {
+        unequalCounters.get(counterKey).inc();
+        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+      }
+    }
+  }
+
+  static SqlEntity normalizeEntity(SqlEntity sqlEntity) {
+    if (sqlEntity instanceof EppResource) {
+      return normalizeEppResource(sqlEntity);
+    }
+    if (sqlEntity instanceof HistoryEntry) {
+      return (SqlEntity) normalizeHistoryEntry((HistoryEntry) sqlEntity);
+    }
+    return sqlEntity;
+  }
+
+  /**
+   * Normalizes an {@link EppResource} instance for comparison.
+   *
+   * <p>This method may modify the input object using reflection instead of making a copy with
+   * {@code eppResource.asBuilder().build()}, because when {@code eppResource} is a {@link
+   * google.registry.model.domain.DomainBase}, the {@code build} method accesses the Database, which
+   * we want to avoid.
+   */
+  static SqlEntity normalizeEppResource(SqlEntity eppResource) {
+    try {
+      if (isSpecialCaseProberEntity(eppResource)) {
+        // Clearing some timestamps. See isSpecialCaseProberEntity() for reasons.
+        Field lastUpdateTime = BackupGroupRoot.class.getDeclaredField("updateTimestamp");
+        lastUpdateTime.setAccessible(true);
+        lastUpdateTime.set(eppResource, null);
+        Field deletionTime = EppResource.class.getDeclaredField("deletionTime");
+        deletionTime.setAccessible(true);
+        deletionTime.set(eppResource, null);
+      }
+      Field authField =
+          eppResource instanceof DomainContent
+              ? DomainContent.class.getDeclaredField("authInfo")
+              : eppResource instanceof ContactBase
+                  ? ContactBase.class.getDeclaredField("authInfo")
+                  : null;
+      if (authField != null) {
+        authField.setAccessible(true);
+        AuthInfo authInfo = (AuthInfo) authField.get(eppResource);
+        // When AuthInfo is missing, the authInfo field is null if the object is loaded from
+        // Datastore, or a PasswordAuth with null properties if loaded from SQL. In the second case
+        // we set the authInfo field to null.
+        if (authInfo != null
+            && authInfo.getPw() != null
+            && authInfo.getPw().getRepoId() == null
+            && authInfo.getPw().getValue() == null) {
+          authField.set(eppResource, null);
+        }
+      }
+
+      Field field = EppResource.class.getDeclaredField("revisions");
+      field.setAccessible(true);
+      field.set(eppResource, null);
+      return eppResource;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Normalizes a {@link HistoryEntry} for comparison.
+   *
+   * <p>This method modifies the input using reflection because relevant builder methods performs
+   * unwanted checks and changes.
+   */
+  static HistoryEntry normalizeHistoryEntry(HistoryEntry historyEntry) {
+    // History objects from Datastore do not have details of their EppResource objects
+    // (domainContent, contactBase, hostBase).
+    try {
+      if (historyEntry instanceof DomainHistory) {
+        Field domainContent = DomainHistory.class.getDeclaredField("domainContent");
+        domainContent.setAccessible(true);
+        domainContent.set(historyEntry, null);
+        // Convert empty domainTransactionRecords to null for comparison.
+        Field domainTransactionRecords =
+            HistoryEntry.class.getDeclaredField("domainTransactionRecords");
+        domainTransactionRecords.setAccessible(true);
+        Set<?> domainTransactionRecordsValue = (Set<?>) domainTransactionRecords.get(historyEntry);
+        if (domainTransactionRecordsValue != null && domainTransactionRecordsValue.isEmpty()) {
+          domainTransactionRecords.set(historyEntry, null);
+        }
+        // DomainHistory in Datastore does not have the following properties either:
+        Field nsHosts = DomainHistory.class.getDeclaredField("nsHosts");
+        nsHosts.setAccessible(true);
+        nsHosts.set(historyEntry, null);
+        Field dsDataHistories = DomainHistory.class.getDeclaredField("dsDataHistories");
+        dsDataHistories.setAccessible(true);
+        dsDataHistories.set(historyEntry, null);
+        Field gracePeriodHistories = DomainHistory.class.getDeclaredField("gracePeriodHistories");
+        gracePeriodHistories.setAccessible(true);
+        gracePeriodHistories.set(historyEntry, null);
+      } else if (historyEntry instanceof ContactHistory) {
+        Field contactBase = ContactHistory.class.getDeclaredField("contactBase");
+        contactBase.setAccessible(true);
+        contactBase.set(historyEntry, null);
+      } else if (historyEntry instanceof HostHistory) {
+        Field hostBase = HostHistory.class.getDeclaredField("hostBase");
+        hostBase.setAccessible(true);
+        hostBase.set(historyEntry, null);
+      }
+      return historyEntry;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Returns {@code true} if {@code entity} is created by the prober and needs special treatment.
+   *
+   * <p>{@link EppResource} entities created by the prober are deleted by a cron job that bypasses
+   * the CommitLog mechanism. As a result, their deletions are not propagated to SQL, creating two
+   * types of mismatches: an entity exists in both databases but differs in lastUpdateTime and
+   * deletionTime; an entity only exists in the SQL database.
+   *
+   * <p>In production, there are few placeholder {@link Registrar registrars} that do not exist in
+   * Datastore. They were manually created to in SQL to solve a one-time problem (see b/187946868
+   * for details). They can be ignored in the database comparison.
+   */
+  static boolean isSpecialCaseProberEntity(Object entity) {
+    if (entity instanceof EppResource) {
+      EppResource host = (EppResource) entity;
+      if (host.getPersistedCurrentSponsorRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (entity instanceof HistoryEntry) {
+      HistoryEntry historyEntry = (HistoryEntry) entity;
+      if (historyEntry.getRegistrarId().startsWith("prober-")) {
+        // Not all prober entities have "prober-" as registrar prefix.
+        return true;
+      }
+      if (Objects.equals(historyEntry.getReason(), "Deletion of prober data")) {
+        // Soft-delete event in Datastore that is not propagated to SQL.
+        return true;
+      }
+    }
+    if (entity instanceof DomainHistory) {
+      DomainHistory domainHistory = (DomainHistory) entity;
+      if (domainHistory.getDomainContent().isPresent()
+          && domainHistory.getDomainContent().get().getDomainName().startsWith("prober-")) {
+        // Asynchronously replicated event in SQL.
+        return true;
+      }
+      if (domainHistory.getDomainRepoId() != null) {
+        // Some synthetic events only have domainRepoId.
+        String repoId = domainHistory.getDomainRepoId();
+        if (Transforms.IGNORED_DOMAINS.contains(repoId)) {
+          return true;
+        }
+        String suffix = repoId.substring(repoId.indexOf('-') + 1);
+        String cell = suffix.substring(0, 2);
+        suffix = suffix.substring(2);
+        if (PROBER_CELLS.contains(cell) && PROBER_TYPES.contains(suffix)) {
+          return true;
+        }
+      }
+    }
+    if (entity instanceof ContactHistory) {
+      if (Transforms.IGNORED_CONTACTS.contains(((ContactHistory) entity).getContactRepoId())) {
+        return true;
+      }
+    }
+    if (entity instanceof HostHistory) {
+      if (Transforms.IGNORED_HOSTS.contains(((HostHistory) entity).getHostRepoId())) {
+        return true;
+      }
+    }
+    if (entity instanceof BillingEvent) {
+      BillingEvent event = (BillingEvent) entity;
+      if (event.getRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (entity instanceof PollMessage) {
+      if (((PollMessage) entity).getRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (RegistryEnvironment.get().equals(RegistryEnvironment.PRODUCTION)
+        && entity instanceof Registrar) {
+      Registrar registrar = (Registrar) entity;
+      if (registrar.getRegistrarId().startsWith("prober-wj-")) {
+        return true;
+      }
+    }
+    return false;
+  }
+}

core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java

@@ -25,6 +25,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.datastore.v1.Entity;
+import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.util.Iterator;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
@@ -78,6 +80,7 @@ import org.apache.beam.sdk.values.TupleTagList;
  * types in the Datastore using the {@code --numOfKindsHint} argument. If the default value for this
  * parameter is too low, performance will suffer.
  */
+@DeleteAfterMigration
 public class BulkDeleteDatastorePipeline {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -308,6 +311,11 @@ public class BulkDeleteDatastorePipeline {
 
   public interface BulkDeletePipelineOptions extends GcpOptions {
 
+    @Description("The Registry environment.")
+    RegistryEnvironment getRegistryEnvironment();
+
+    void setRegistryEnvironment(RegistryEnvironment environment);
+
     @Description(
         "The Datastore KINDs to be deleted. The format may be:\n"
             + "\t- The list of kinds to be deleted as a comma-separated string, or\n"

core/src/main/java/google/registry/beam/datastore/DatastoreV1.java

@@ -53,6 +53,7 @@ import com.google.datastore.v1.client.DatastoreOptions;
 import com.google.datastore.v1.client.QuerySplitter;
 import com.google.protobuf.Int32Value;
 import com.google.rpc.Code;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.Serializable;
 import java.util.List;
 import java.util.NoSuchElementException;
@@ -80,6 +81,7 @@ import org.joda.time.Duration;
  * Contains an adaptation of {@link org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read}. See
  * {@link MultiRead} for details.
  */
+@DeleteAfterMigration
 public class DatastoreV1 {
 
   // A package-private constructor to prevent direct instantiation from outside of this package
@@ -169,14 +171,15 @@ public class DatastoreV1 {
       int numSplits;
       try {
         long estimatedSizeBytes = getEstimatedSizeBytes(datastore, query, namespace);
-        logger.atInfo().log("Estimated size bytes for the query is: %s", estimatedSizeBytes);
+        logger.atInfo().log("Estimated size for the query is %d bytes.", estimatedSizeBytes);
         numSplits =
             (int)
                 Math.min(
                     NUM_QUERY_SPLITS_MAX,
                     Math.round(((double) estimatedSizeBytes) / DEFAULT_BUNDLE_SIZE_BYTES));
       } catch (Exception e) {
-        logger.atWarning().log("Failed the fetch estimatedSizeBytes for query: %s", query, e);
+        logger.atWarning().withCause(e).log(
+            "Failed the fetch estimatedSizeBytes for query: %s", query);
         // Fallback in case estimated size is unavailable.
         numSplits = NUM_QUERY_SPLITS_MIN;
       }
@@ -215,7 +218,7 @@ public class DatastoreV1 {
     private static Entity getLatestTableStats(
         String ourKind, @Nullable String namespace, Datastore datastore) throws DatastoreException {
       long latestTimestamp = queryLatestStatisticsTimestamp(datastore, namespace);
-      logger.atInfo().log("Latest stats timestamp for kind %s is %s", ourKind, latestTimestamp);
+      logger.atInfo().log("Latest stats timestamp for kind %s is %s.", ourKind, latestTimestamp);
 
       Query.Builder queryBuilder = Query.newBuilder();
       if (Strings.isNullOrEmpty(namespace)) {
@@ -234,7 +237,7 @@ public class DatastoreV1 {
       long now = System.currentTimeMillis();
       RunQueryResponse response = datastore.runQuery(request);
       logger.atFine().log(
-          "Query for per-kind statistics took %sms", System.currentTimeMillis() - now);
+          "Query for per-kind statistics took %d ms.", System.currentTimeMillis() - now);
 
       QueryResultBatch batch = response.getBatch();
       if (batch.getEntityResultsCount() == 0) {
@@ -330,7 +333,7 @@ public class DatastoreV1 {
           logger.atWarning().log(
               "Failed to translate Gql query '%s': %s", gqlQueryWithZeroLimit, e.getMessage());
           logger.atWarning().log(
-              "User query might have a limit already set, so trying without zero limit");
+              "User query might have a limit already set, so trying without zero limit.");
           // Retry without the zero limit.
           return translateGqlQuery(gql, datastore, namespace);
         } else {
@@ -514,10 +517,10 @@ public class DatastoreV1 {
       @ProcessElement
       public void processElement(ProcessContext c) throws Exception {
         String gqlQuery = c.element();
-        logger.atInfo().log("User query: '%s'", gqlQuery);
+        logger.atInfo().log("User query: '%s'.", gqlQuery);
         Query query =
             translateGqlQueryWithLimitCheck(gqlQuery, datastore, v1Options.getNamespace());
-        logger.atInfo().log("User gql query translated to Query(%s)", query);
+        logger.atInfo().log("User gql query translated to Query(%s).", query);
         c.output(query);
       }
     }
@@ -573,7 +576,7 @@ public class DatastoreV1 {
           estimatedNumSplits = numSplits;
         }
 
-        logger.atInfo().log("Splitting the query into %s splits", estimatedNumSplits);
+        logger.atInfo().log("Splitting the query into %d splits.", estimatedNumSplits);
         List<Query> querySplits;
         try {
           querySplits =
@@ -647,7 +650,7 @@ public class DatastoreV1 {
               throw exception;
             }
             if (!BackOffUtils.next(sleeper, backoff)) {
-              logger.atSevere().log("Aborting after %s retries.", MAX_RETRIES);
+              logger.atSevere().log("Aborting after %d retries.", MAX_RETRIES);
               throw exception;
             }
           }

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -21,12 +21,14 @@ import static com.google.common.base.Strings.isNullOrEmpty;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
+import google.registry.model.annotations.DeleteAfterMigration;
 import org.joda.time.DateTime;
 
 /**
  * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
  * Datastore export and Nomulus CommitLogs that overlap with the export.
  */
+@DeleteAfterMigration
 public final class BackupPaths {
 
   private BackupPaths() {}

core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java

@@ -18,9 +18,11 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.appengine.api.datastore.Entity;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.util.Objects;
 
 /** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */
+@DeleteAfterMigration
 final class DomainBaseUtil {
 
   private DomainBaseUtil() {}

core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java

@@ -20,11 +20,12 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.backup.VersionedEntity;
 import google.registry.beam.common.RegistryJpaIO;
 import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken;
@@ -62,6 +63,7 @@ import org.joda.time.DateTime;
  * <ol>
  *   <li>{@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set
  *       up in the SQL database.
+ *   <li>{@link Cursor}: Logically can depend on {@code Registry}, but without foreign key.
  *   <li>{@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet.
  *   <li>{@link ContactResource}: references {@code Registrar}
  *   <li>{@link RegistrarContact}: references {@code Registrar}.
@@ -93,6 +95,7 @@ import org.joda.time.DateTime;
  * may start writing {@code DomainBase} entities before all {@code Registry}, {@code Registrar} and
  * {@code ContactResource} entities have been persisted.
  */
+@DeleteAfterMigration
 public class InitSqlPipeline implements Serializable {
 
   /**
@@ -101,7 +104,11 @@ public class InitSqlPipeline implements Serializable {
    */
   private static final ImmutableList<Class<?>> PHASE_ONE_ORDERED =
       ImmutableList.of(
-          Registry.class, Registrar.class, ContactResource.class, RegistrarContact.class);
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class);
 
   /**
    * Datastore kinds to be written to the SQL database after the cleansed version of {@link
@@ -224,9 +231,7 @@ public class InitSqlPipeline implements Serializable {
   }
 
   private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
-    try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-      return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
-    }
+    return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
   }
 
   public static void main(String[] args) {

core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java

@@ -15,10 +15,12 @@
 package google.registry.beam.initsql;
 
 import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.model.annotations.DeleteAfterMigration;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.Validation;
 
 /** Pipeline options for {@link InitSqlPipeline} */
+@DeleteAfterMigration
 public interface InitSqlPipelineOptions extends RegistryPipelineOptions {
 
   @Description("The root directory of the export to load.")

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -22,6 +22,7 @@ import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.util.Comparator.comparing;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -36,6 +37,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.DomainBase;
@@ -79,6 +81,7 @@ import org.joda.time.DateTime;
  * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export
  * files and Nomulus CommitLog files.
  */
+@DeleteAfterMigration
 public final class Transforms {
 
   private Transforms() {}
@@ -100,8 +103,9 @@ public final class Transforms {
   }
 
   /**
-   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
-   * commitLogToTime} for caller specified {@code kinds}.
+   * Composite {@link PTransform transform} that loads the Datastore snapshot right before {@code
+   * commitLogToTime} for caller specified {@code kinds}. The resulting snapshot has all changes
+   * that happened before {@code commitLogToTime}, and none at or after {@code commitLogToTime}.
    *
    * <p>Caller must provide the location of a Datastore export that started AFTER {@code
    * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
@@ -259,16 +263,19 @@ public final class Transforms {
                     .iterator()));
   }
 
-  // Production data repair configs go below. See b/185954992.
+  // Production data repair configs go below. See b/185954992. Note that the CommitLog replay
+  // process does not filter out the ignored entities listed below, a mistake that we do not fix
+  // for operational convenience. Instead, the Database comparison tool will filter them out. See
+  // ValidateSqlUtils.java for more information.
 
-  // Prober domains in bad state, without associated contacts, hosts, billings, and history.
-  // They can be safely ignored.
-  private static final ImmutableSet<String> IGNORED_DOMAINS =
+  // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized
+  // history. They can be safely ignored.
+  public static final ImmutableSet<String> IGNORED_DOMAINS =
       ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT");
 
   // Prober hosts referencing phantom registrars. They and their associated history entries can be
   // safely ignored.
-  private static final ImmutableSet<String> IGNORED_HOSTS =
+  public static final ImmutableSet<String> IGNORED_HOSTS =
       ImmutableSet.of(
           "4E21_WJ0TEST-GOOGLE",
           "4E21_WJ1TEST-GOOGLE",
@@ -277,7 +284,7 @@ public final class Transforms {
 
   // Prober contacts referencing phantom registrars. They and their associated history entries can
   // be safely ignored.
-  private static final ImmutableSet IGNORED_CONTACTS =
+  public static final ImmutableSet<String> IGNORED_CONTACTS =
       ImmutableSet.of(
           "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE");
 
@@ -298,7 +305,14 @@ public final class Transforms {
       return !IGNORED_HOSTS.contains(roid);
     }
     if (entity.getKind().equals("HistoryEntry")) {
-      // Remove production bad data: History of the contacts to be ignored:
+      // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated.
+      // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot
+      // be deserialized.
+      if (Objects.equals(entity.getProperty("type"), "DOMAIN_APPLICATION_CREATE")) {
+        return false;
+      }
+
+      // Remove production bad data: Histories of ignored EPP resources:
       com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent();
       if (parentKey.getKind().equals("ContactResource")) {
         String contactRoid = parentKey.getName();
@@ -308,19 +322,16 @@ public final class Transforms {
         String hostRoid = parentKey.getName();
         return !IGNORED_HOSTS.contains(hostRoid);
       }
-    }
-    // End of production-specific checks.
-
-    if (entity.getKind().equals("HistoryEntry")) {
-      // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated.
-      // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot
-      // be deserialized.
-      return !Objects.equals(entity.getProperty("type"), "DOMAIN_APPLICATION_CREATE");
+      if (parentKey.getKind().equals("DomainBase")) {
+        String domainRoid = parentKey.getName();
+        return !IGNORED_DOMAINS.contains(domainRoid);
+      }
     }
     return true;
   }
 
-  private static Entity repairBadData(Entity entity) {
+  @VisibleForTesting
+  static Entity repairBadData(Entity entity) {
     if (entity.getKind().equals("Cancellation")
         && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) {
       // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the
@@ -328,6 +339,15 @@ public final class Transforms {
       // instead of append. See b/185954992.
       entity.setUnindexedProperty("reason", Reason.RENEW.name());
       entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name()));
+    } else if (entity.getKind().equals("DomainBase")) {
+      // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this.
+      entity.setIndexedProperty(
+          "fullyQualifiedDomainName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedDomainName")));
+    } else if (entity.getKind().equals("HostResource")) {
+      entity.setIndexedProperty(
+          "fullyQualifiedHostName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedHostName")));
     }
     return entity;
   }
@@ -348,7 +368,7 @@ public final class Transforms {
    *     to make Optional work with BEAM)
    */
   @Nullable
-  public static Object convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
+  public static SqlEntity convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
     return dsEntity
         .getEntity()
         .filter(Transforms::isMigratable)
@@ -365,7 +385,8 @@ public final class Transforms {
    * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
    * given {@link Iterable}.
    */
-  static PTransform<PBegin, PCollection<String>> toStringPCollection(Iterable<String> strings) {
+  private static PTransform<PBegin, PCollection<String>> toStringPCollection(
+      Iterable<String> strings) {
     return Create.of(strings).withCoder(StringUtf8Coder.of());
   }
 
@@ -373,7 +394,7 @@ public final class Transforms {
    * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
    * caller-provided {@code transformer}.
    */
-  static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+  private static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
       DoFn<ReadableFile, VersionedEntity> transformer) {
     return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
       @Override
@@ -389,7 +410,7 @@ public final class Transforms {
     private final DateTime fromTime;
     private final DateTime toTime;
 
-    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+    FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
       checkNotNull(fromTime, "fromTime");
       checkNotNull(toTime, "toTime");
       checkArgument(

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -57,7 +57,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=invoicing}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -19,10 +19,13 @@ import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static google.registry.rde.RdeModule.BRDA_QUEUE;
+import static google.registry.rde.RdeModule.RDE_UPLOAD_QUEUE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.auto.value.AutoValue;
 import com.google.cloud.storage.BlobId;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.PgpHelper;
@@ -31,14 +34,20 @@ import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
 import google.registry.model.tld.Registry;
+import google.registry.rde.BrdaCopyAction;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.Ghostryde;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.RdeCounter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.rde.RdeModule;
 import google.registry.rde.RdeResourceType;
+import google.registry.rde.RdeUploadAction;
 import google.registry.rde.RdeUtil;
+import google.registry.request.Action.Service;
+import google.registry.request.RequestParameters;
 import google.registry.tldconfig.idn.IdnTableEnum;
+import google.registry.util.CloudTasksUtils;
 import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
 import google.registry.xml.ValidationMode;
@@ -66,8 +75,12 @@ public class RdeIO {
   abstract static class Write
       extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
 
+    private static final long serialVersionUID = 3334807737227087760L;
+
     abstract GcsUtils gcsUtils();
 
+    abstract CloudTasksUtils cloudTasksUtils();
+
     abstract String rdeBucket();
 
     // It's OK to return a primitive array because we are only using it to construct the
@@ -83,7 +96,9 @@ public class RdeIO {
 
     @AutoValue.Builder
     abstract static class Builder {
-      abstract Builder setGcsUtils(GcsUtils gcsUtils);
+      abstract Builder setGcsUtils(GcsUtils value);
+
+      abstract Builder setCloudTasksUtils(CloudTasksUtils value);
 
       abstract Builder setRdeBucket(String value);
 
@@ -100,7 +115,9 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()));
+          .apply(
+              "Update cursor and enqueue next action",
+              ParDo.of(new CursorUpdater(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -109,6 +126,7 @@ public class RdeIO {
       extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5496375923068400382L;
 
     private final GcsUtils gcsUtils;
     private final String rdeBucket;
@@ -149,13 +167,13 @@ public class RdeIO {
           Optional.ofNullable(key.revision())
               .orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
       String id = RdeUtil.timestampToId(watermark);
-      String prefix = options.getJobName();
-      String basename = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
+      String prefix =
+          options.getJobName()
+              + '/'
+              + RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
       if (key.manual()) {
         checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
-        prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
-      } else {
-        prefix = prefix + "/" + basename;
+        prefix = "manual/" + key.directoryWithTrailingSlash() + prefix;
       }
       BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
       // This file will contain the byte length (ASCII) of the raw unencrypted XML.
@@ -172,7 +190,7 @@ public class RdeIO {
 
       // Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
 
-      logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
+      logger.atInfo().log("Writing files '%s' and '%s'.", xmlFilename, xmlLengthFilename);
       try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
           OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
           OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
@@ -219,7 +237,7 @@ public class RdeIO {
       //
       // This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
       if (mode == RdeMode.FULL) {
-        logger.atInfo().log("Writing %s", reportFilename);
+        logger.atInfo().log("Writing file '%s'.", reportFilename);
         try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
             OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
           counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
@@ -229,7 +247,10 @@ public class RdeIO {
       }
       // Now that we're done, output roll the cursor forward.
       if (key.manual()) {
-        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
+        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task.");
+        // Temporary measure to run RDE in beam in parallel with the daily MapReduce based RDE runs.
+      } else if (tm().isOfy()) {
+        logger.atInfo().log("Ofy is primary TM; not advancing cursor or enqueuing upload task.");
       } else {
         outputReceiver.output(KV.of(key, revision));
       }
@@ -237,10 +258,19 @@ public class RdeIO {
   }
 
   private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5822176227753327224L;
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private CursorUpdater(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
 
     @ProcessElement
-    public void processElement(@Element KV<PendingDeposit, Integer> input) {
+    public void processElement(
+        @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -265,8 +295,33 @@ public class RdeIO {
                     key);
                 tm().put(Cursor.create(key.cursor(), newPosition, registry));
                 logger.atInfo().log(
-                    "Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
+                    "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                if (key.mode() == RdeMode.FULL) {
+                  cloudTasksUtils.enqueue(
+                      RDE_UPLOAD_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          RdeUploadAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                } else {
+                  cloudTasksUtils.enqueue(
+                      BRDA_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          BrdaCopyAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_WATERMARK,
+                              key.watermark().toString(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                }
               });
     }
   }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -14,36 +14,53 @@
 
 package google.registry.beam.rde;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.beam.rde.RdePipeline.TupleTags.DOMAIN_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.EXTERNAL_HOST_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.HOST_TO_PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_CONTACTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
+import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
+import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
-import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.util.CloudTasksUtils;
+import google.registry.util.UtilsModule;
 import google.registry.xml.ValidationMode;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -51,73 +68,158 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.persistence.Entity;
+import javax.persistence.IdClass;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.DateTime;
 
 /**
  * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=rde}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
+ * <p>This pipeline only works for pending deposits with the same watermark, the {@link
+ * google.registry.rde.RdeStagingAction} will batch such pending deposits together and launch
+ * multiple pipelines if multiple watermarks exist.
+ *
+ * <p>The pipeline is broadly divided into two parts -- creating the {@link DepositFragment}s, and
+ * processing them.
+ *
+ * <h1>Creating {@link DepositFragment}</h1>
+ *
+ * <h2>{@link Registrar}</h2>
+ *
+ * Non-test registrar entities are loaded from Cloud SQL and marshalled into deposit fragments. They
+ * are <b>NOT</b> rewound to the watermark.
+ *
+ * <h2>{@link EppResource}</h2>
+ *
+ * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
+ * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
+ * revision ID) from the SQL query.
+ *
+ * <h3>{@link DomainBase}</h3>
+ *
+ * After the most recent (live) domain resources are loaded from the corresponding history objects,
+ * we marshall them to deposit fragments and emit the (pending deposit: deposit fragment) pairs for
+ * further processing. We also find all the contacts and hosts referenced by a given domain and emit
+ * pairs of (contact/host repo ID: pending deposit) for all RDE pending deposits for further
+ * processing.
+ *
+ * <h3>{@link ContactResource}</h3>
+ *
+ * We first join most recent contact histories, represented by (contact repo ID: contact history
+ * revision ID) pairs, with referenced contacts, represented by (contact repo ID: pending deposit)
+ * pairs, on the contact repo ID, to remove unreferenced contact histories. Contact resources are
+ * then loaded from the remaining referenced contact histories, and marshalled into (pending
+ * deposit: deposit fragment) pairs.
+ *
+ * <h3>{@link HostResource}</h3>
+ *
+ * Similar to {@link ContactResource}, we join the most recent host history with referenced hosts to
+ * find most recent referenced hosts. For external hosts we do the same treatment as we did on
+ * contacts and obtain the (pending deposit: deposit fragment) pairs. For subordinate hosts, we need
+ * to find the superordinate domain in order to properly handle pending transfer in the deposit as
+ * well. So we first find the superordinate domain repo ID from the host and join the (superordinate
+ * domain repo ID: (subordinate host repo ID: (pending deposit: revision ID))) pair with the (domain
+ * repo ID: revision ID) pair obtained from the domain history query in order to map the host at
+ * watermark to the domain at watermark. We then proceed to create the (pending deposit: deposit
+ * fragment) pair for subordinate hosts using the added domain information.
+ *
+ * <h1>Processing {@link DepositFragment}</h1>
+ *
+ * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
+ * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
+ * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
+ * pending deposit is then rolled forward, and the next action is enqueued. The latter two
+ * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
+ *
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
 @Singleton
 public class RdePipeline implements Serializable {
 
+  private static final long serialVersionUID = -4866795928854754666L;
   private final transient RdePipelineOptions options;
   private final ValidationMode mode;
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final ImmutableSet<PendingDeposit> pendingDeposits;
+  private final DateTime watermark;
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
+  private final CloudTasksUtils cloudTasksUtils;
+  private final RdeMarshaller marshaller;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
   private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
       Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
 
-  private static final String EPP_RESOURCE_QUERY =
-      "SELECT id FROM %entity% "
-          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
-
-  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
-    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
-        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
-  }
+  // The field name of the EPP resource embedded in its corresponding history entry.
+  private static final ImmutableMap<Class<? extends HistoryEntry>, String> EPP_RESOURCE_FIELD_NAME =
+      ImmutableMap.of(
+          DomainHistory.class,
+          "domainContent",
+          ContactHistory.class,
+          "contactBase",
+          HostHistory.class,
+          "hostBase");
 
   @Inject
-  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
+  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
-    this.pendings = decodePendings(options.getPendings());
-    this.rdeBucket = options.getGcsBucket();
+    this.pendingDeposits = decodePendingDeposits(options.getPendings());
+    ImmutableSet<DateTime> potentialWatermarks =
+        pendingDeposits.stream()
+            .map(PendingDeposit::watermark)
+            .distinct()
+            .collect(toImmutableSet());
+    checkArgument(
+        potentialWatermarks.size() == 1,
+        String.format(
+            "RDE pipeline should only work on pending deposits "
+                + "with the same watermark, but %d were given: %s",
+            potentialWatermarks.size(), potentialWatermarks));
+    this.watermark = potentialWatermarks.asList().get(0);
+    this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
+    this.marshaller = new RdeMarshaller(mode);
   }
 
   PipelineResult run() {
@@ -129,152 +231,452 @@ public class RdePipeline implements Serializable {
   }
 
   PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
-    return PCollectionList.of(processRegistrars(pipeline))
-        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
-        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
-        .and(processNonRegistrarEntities(pipeline, HostResource.class))
-        .apply(Flatten.pCollections())
+    PCollection<KV<PendingDeposit, DepositFragment>> registrarFragments =
+        processRegistrars(pipeline);
+
+    PCollection<KV<String, Long>> domainHistories =
+        getMostRecentHistoryEntries(pipeline, DomainHistory.class);
+
+    PCollection<KV<String, Long>> contactHistories =
+        getMostRecentHistoryEntries(pipeline, ContactHistory.class);
+
+    PCollection<KV<String, Long>> hostHistories =
+        getMostRecentHistoryEntries(pipeline, HostHistory.class);
+
+    PCollectionTuple processedDomainHistories = processDomainHistories(domainHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> domainFragments =
+        processedDomainHistories.get(DOMAIN_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> contactFragments =
+        processContactHistories(
+            processedDomainHistories.get(REFERENCED_CONTACTS), contactHistories);
+
+    PCollectionTuple processedHosts =
+        processHostHistories(processedDomainHistories.get(REFERENCED_HOSTS), hostHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> externalHostFragments =
+        processedHosts.get(EXTERNAL_HOST_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> subordinateHostFragments =
+        processSubordinateHosts(processedHosts.get(SUPERORDINATE_DOMAINS), domainHistories);
+
+    return PCollectionList.of(registrarFragments)
+        .and(domainFragments)
+        .and(contactFragments)
+        .and(externalHostFragments)
+        .and(subordinateHostFragments)
+        .apply(
+            "Combine PendingDeposit:DepositFragment pairs from all entities",
+            Flatten.pCollections())
         .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply("Group by PendingDeposit", GroupByKey.create());
+        .apply("Group DepositFragment by PendingDeposit", GroupByKey.create());
   }
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
     input.apply(
-        "Write to GCS and update cursors",
+        "Write to GCS, update cursors, and enqueue upload tasks",
         RdeIO.Write.builder()
             .setRdeBucket(rdeBucket)
             .setGcsUtils(gcsUtils)
+            .setCloudTasksUtils(cloudTasksUtils)
             .setValidationMode(mode)
             .setStagingKeyBytes(stagingKeyBytes)
             .build());
   }
 
-  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+  private PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    // Note that the namespace in the metric is not being used by Stackdriver, it just has to be
+    // non-empty.
+    // See:
+    // https://stackoverflow.com/questions/48530496/google-dataflow-custom-metrics-not-showing-on-stackdriver
+    Counter includedRegistrarCounter = Metrics.counter("RDE", "IncludedRegistrar");
+    Counter registrarFragmentCounter = Metrics.counter("RDE", "RegistrarFragment");
     return pipeline
         .apply(
-            "Read all production Registrar entities",
+            "Read all production Registrars",
             RegistryJpaIO.read(
                 "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
                 ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
                 String.class,
-                // TODO: consider adding coders for entities and pass them directly instead of using
-                // VKeys.
                 id -> VKey.createSql(Registrar.class, id)))
         .apply(
-            "Marshal Registrar into DepositFragment",
+            "Marshall Registrar into DepositFragment",
             FlatMapElements.into(
-                    TypeDescriptors.kvs(
+                    kvs(
                         TypeDescriptor.of(PendingDeposit.class),
                         TypeDescriptor.of(DepositFragment.class)))
                 .via(
                     (VKey<Registrar> key) -> {
+                      includedRegistrarCounter.inc();
                       Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
-                      DepositFragment fragment =
-                          new RdeMarshaller(mode).marshalRegistrar(registrar);
-                      return pendings.values().stream()
-                          .map(pending -> KV.of(pending, fragment))
-                          .collect(toImmutableSet());
+                      DepositFragment fragment = marshaller.marshalRegistrar(registrar);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          pendingDeposits.stream()
+                              .map(pending -> KV.of(pending, fragment))
+                              .collect(toImmutableSet());
+                      registrarFragmentCounter.inc(fragments.size());
+                      return fragments;
                     }));
   }
 
-  @SuppressWarnings("deprecation") // Reshuffle is still recommended by Dataflow.
-  <T extends EppResource>
-      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
-          Pipeline pipeline, Class<T> clazz) {
-    return createInputs(pipeline, clazz)
-        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
+  /**
+   * Load the most recent history entry before the watermark for a given history entry type.
+   *
+   * <p>Note that deleted and non-production resources are not included.
+   *
+   * @return A KV pair of (repoId, revisionId), used to reconstruct the composite key for the
+   *     history entry.
+   */
+  private <T extends HistoryEntry> PCollection<KV<String, Long>> getMostRecentHistoryEntries(
+      Pipeline pipeline, Class<T> historyClass) {
+    String repoIdFieldName = HistoryEntryDao.REPO_ID_FIELD_NAMES.get(historyClass);
+    String resourceFieldName = EPP_RESOURCE_FIELD_NAME.get(historyClass);
+    return pipeline
         .apply(
-            "Reshuffle KV<PendingDeposit, DepositFragment> of "
-                + clazz.getSimpleName()
-                + " to prevent fusion",
-            Reshuffle.of());
+            String.format("Load most recent %s", historyClass.getSimpleName()),
+            RegistryJpaIO.read(
+                ("SELECT %repoIdField%, id FROM %entity% WHERE (%repoIdField%, modificationTime)"
+                     + " IN (SELECT %repoIdField%, MAX(modificationTime) FROM %entity% WHERE"
+                     + " modificationTime <= :watermark GROUP BY %repoIdField%) AND"
+                     + " %resourceField%.deletionTime > :watermark AND"
+                     + " COALESCE(%resourceField%.creationClientId, '') NOT LIKE 'prober-%' AND"
+                     + " COALESCE(%resourceField%.currentSponsorClientId, '') NOT LIKE 'prober-%'"
+                     + " AND COALESCE(%resourceField%.lastEppUpdateClientId, '') NOT LIKE"
+                     + " 'prober-%' "
+                        + (historyClass == DomainHistory.class
+                            ? "AND %resourceField%.tld IN "
+                                + "(SELECT id FROM Tld WHERE tldType = 'REAL')"
+                            : ""))
+                    .replace("%entity%", historyClass.getSimpleName())
+                    .replace("%repoIdField%", repoIdFieldName)
+                    .replace("%resourceField%", resourceFieldName),
+                ImmutableMap.of("watermark", watermark),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1])))
+        .setCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));
   }
 
-  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
-    return pipeline.apply(
-        "Read all production " + clazz.getSimpleName() + " entities",
-        RegistryJpaIO.read(
-            createEppResourceQuery(clazz),
-            clazz.equals(DomainBase.class)
-                ? ImmutableMap.of("tlds", pendings.keySet())
-                : ImmutableMap.of(),
-            String.class,
-            // TODO: consider adding coders for entities and pass them directly instead of using
-            // VKeys.
-            x -> VKey.create(clazz, x)));
+  private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
+      Class<T> historyEntryClazz, String repoId, long revisionId) {
+    try {
+      Class<?> idClazz = historyEntryClazz.getAnnotation(IdClass.class).value();
+      Serializable idObject =
+          (Serializable)
+              idClazz.getConstructor(String.class, long.class).newInstance(repoId, revisionId);
+      return jpaTm()
+          .transact(() -> jpaTm().loadByKey(VKey.createSql(historyEntryClazz, idObject)))
+          .getResourceAtPointInTime()
+          .map(resource -> resource.cloneProjectedAtTime(watermark))
+          .get();
+    } catch (NoSuchMethodException
+        | InvocationTargetException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot load resource from %s with repoId %s and revisionId %s",
+              historyEntryClazz.getSimpleName(), repoId, revisionId),
+          e);
+    }
   }
 
-  <T extends EppResource>
-      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
-    return FlatMapElements.into(
-            TypeDescriptors.kvs(
-                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
-        .via(
-            (VKey<T> key) -> {
-              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
-              // The set of all TLDs to which this resource should be emitted.
-              ImmutableSet<String> tlds =
-                  clazz.equals(DomainBase.class)
-                      ? ImmutableSet.of(((DomainBase) resource).getTld())
-                      : pendings.keySet();
-              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
-              ImmutableSet<DateTime> dates =
-                  tlds.stream()
-                      .map(pendings::get)
-                      .flatMap(ImmutableSet::stream)
-                      .map(PendingDeposit::watermark)
-                      .collect(toImmutableSet());
-              // Launch asynchronous fetches of point-in-time representations of resource.
-              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
-                  ImmutableMap.copyOf(
-                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
-              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
-              // the result.
-              RdeFragmenter fragmenter =
-                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
-              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
-              for (String tld : tlds) {
-                for (PendingDeposit pending : pendings.get(tld)) {
-                  // Hosts and contacts don't get included in BRDA deposits.
-                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
-                    continue;
+  /**
+   * Remove unreferenced resources by joining the (repoId, pendingDeposit) pair with the (repoId,
+   * revisionId) on the repoId.
+   *
+   * <p>The (repoId, pendingDeposit) pairs denote resources (contact, host) that are referenced from
+   * a domain, that are to be included in the corresponding pending deposit.
+   *
+   * <p>The (repoId, revisionId) paris come from the most recent history entry query, which can be
+   * used to load the embedded resources themselves.
+   *
+   * @return a pair of (repoId, ([pendingDeposit], [revisionId])) where neither the pendingDeposit
+   *     nor the revisionId list is empty.
+   */
+  private static PCollection<KV<String, CoGbkResult>> removeUnreferencedResource(
+      PCollection<KV<String, PendingDeposit>> referencedResources,
+      PCollection<KV<String, Long>> historyEntries,
+      Class<? extends EppResource> resourceClazz) {
+    String resourceName = resourceClazz.getSimpleName();
+    Class<? extends HistoryEntry> historyEntryClazz =
+        RESOURCE_TYPES_TO_HISTORY_TYPES.get(resourceClazz);
+    String historyEntryName = historyEntryClazz.getSimpleName();
+    Counter referencedResourceCounter = Metrics.counter("RDE", "Referenced" + resourceName);
+    return KeyedPCollectionTuple.of(PENDING_DEPOSIT, referencedResources)
+        .and(REVISION_ID, historyEntries)
+        .apply(
+            String.format(
+                "Join PendingDeposit with %s revision ID on %s", historyEntryName, resourceName),
+            CoGroupByKey.create())
+        .apply(
+            String.format("Remove unreferenced %s", resourceName),
+            Filter.by(
+                (KV<String, CoGbkResult> kv) -> {
+                  boolean toInclude =
+                      // If a resource does not have corresponding pending deposit, it is not
+                      // referenced and should not be included.
+                      kv.getValue().getAll(PENDING_DEPOSIT).iterator().hasNext()
+                          // If a resource does not have revision id (this should not happen, as
+                          // every referenced resource must be valid at watermark time, therefore
+                          // be embedded in a history entry valid at watermark time, otherwise
+                          // the domain cannot reference it), there is no way for us to find the
+                          // history entry and load the embedded resource. So we ignore the resource
+                          // to keep the downstream process simple.
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedResourceCounter.inc();
                   }
-                  Optional<DepositFragment> fragment =
-                      fragmenter.marshal(pending.watermark(), pending.mode());
-                  fragment.ifPresent(
-                      depositFragment -> results.add(KV.of(pending, depositFragment)));
-                }
-              }
-              return results;
-            });
+                  return toInclude;
+                }));
+  }
+
+  private PCollectionTuple processDomainHistories(PCollection<KV<String, Long>> domainHistories) {
+    Counter activeDomainCounter = Metrics.counter("RDE", "ActiveDomainBase");
+    Counter domainFragmentCounter = Metrics.counter("RDE", "DomainFragment");
+    Counter referencedContactCounter = Metrics.counter("RDE", "ReferencedContactResource");
+    Counter referencedHostCounter = Metrics.counter("RDE", "ReferencedHostResource");
+    return domainHistories.apply(
+        "Map DomainHistory to DepositFragment "
+            + "and emit referenced ContactResource and HostResource",
+        ParDo.of(
+                new DoFn<KV<String, Long>, KV<PendingDeposit, DepositFragment>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Long> kv, MultiOutputReceiver receiver) {
+                    activeDomainCounter.inc();
+                    DomainBase domain =
+                        (DomainBase)
+                            loadResourceByHistoryEntryId(
+                                DomainHistory.class, kv.getKey(), kv.getValue());
+                    pendingDeposits.stream()
+                        .filter(pendingDeposit -> pendingDeposit.tld().equals(domain.getTld()))
+                        .forEach(
+                            pendingDeposit -> {
+                              // Domains are always deposited in both modes.
+                              domainFragmentCounter.inc();
+                              receiver
+                                  .get(DOMAIN_FRAGMENTS)
+                                  .output(
+                                      KV.of(
+                                          pendingDeposit,
+                                          marshaller.marshalDomain(domain, pendingDeposit.mode())));
+                              // Contacts and hosts are only deposited in RDE, not BRDA.
+                              if (pendingDeposit.mode() == RdeMode.FULL) {
+                                HashSet<Serializable> contacts = new HashSet<>();
+                                contacts.add(domain.getAdminContact().getSqlKey());
+                                contacts.add(domain.getTechContact().getSqlKey());
+                                contacts.add(domain.getRegistrant().getSqlKey());
+                                // Billing contact is not mandatory.
+                                if (domain.getBillingContact() != null) {
+                                  contacts.add(domain.getBillingContact().getSqlKey());
+                                }
+                                referencedContactCounter.inc(contacts.size());
+                                contacts.forEach(
+                                    contactRepoId ->
+                                        receiver
+                                            .get(REFERENCED_CONTACTS)
+                                            .output(KV.of((String) contactRepoId, pendingDeposit)));
+                                if (domain.getNsHosts() != null) {
+                                  referencedHostCounter.inc(domain.getNsHosts().size());
+                                  domain
+                                      .getNsHosts()
+                                      .forEach(
+                                          hostKey ->
+                                              receiver
+                                                  .get(REFERENCED_HOSTS)
+                                                  .output(
+                                                      KV.of(
+                                                          (String) hostKey.getSqlKey(),
+                                                          pendingDeposit)));
+                                }
+                              }
+                            });
+                  }
+                })
+            .withOutputTags(
+                DOMAIN_FRAGMENTS, TupleTagList.of(REFERENCED_CONTACTS).and(REFERENCED_HOSTS)));
+  }
+
+  private PCollection<KV<PendingDeposit, DepositFragment>> processContactHistories(
+      PCollection<KV<String, PendingDeposit>> referencedContacts,
+      PCollection<KV<String, Long>> contactHistories) {
+    Counter contactFragmentCounter = Metrics.counter("RDE", "ContactFragment");
+    return removeUnreferencedResource(referencedContacts, contactHistories, ContactResource.class)
+        .apply(
+            "Map ContactResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      ContactResource contact =
+                          (ContactResource)
+                              loadResourceByHistoryEntryId(
+                                  ContactHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      DepositFragment fragment = marshaller.marshalContact(contact);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same contact could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .map(pendingDeposit -> KV.of(pendingDeposit, fragment))
+                              .collect(toImmutableSet());
+                      contactFragmentCounter.inc(fragments.size());
+                      return fragments;
+                    }));
+  }
+
+  private PCollectionTuple processHostHistories(
+      PCollection<KV<String, PendingDeposit>> referencedHosts,
+      PCollection<KV<String, Long>> hostHistories) {
+    Counter subordinateHostCounter = Metrics.counter("RDE", "SubordinateHostResource");
+    Counter externalHostCounter = Metrics.counter("RDE", "ExternalHostResource");
+    Counter externalHostFragmentCounter = Metrics.counter("RDE", "ExternalHostFragment");
+    return removeUnreferencedResource(referencedHosts, hostHistories, HostResource.class)
+        .apply(
+            "Map external DomainResource to DepositFragment and process subordinate domains",
+            ParDo.of(
+                    new DoFn<KV<String, CoGbkResult>, KV<PendingDeposit, DepositFragment>>() {
+                      @ProcessElement
+                      public void processElement(
+                          @Element KV<String, CoGbkResult> kv, MultiOutputReceiver receiver) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    kv.getKey(),
+                                    kv.getValue().getOnly(REVISION_ID));
+                        // When a host is subordinate, we need to find it's superordinate domain and
+                        // include it in the deposit as well.
+                        if (host.isSubordinate()) {
+                          subordinateHostCounter.inc();
+                          receiver
+                              .get(SUPERORDINATE_DOMAINS)
+                              .output(
+                                  // The output are pairs of
+                                  // (superordinateDomainRepoId,
+                                  //   (subordinateHostRepoId, (pendingDeposit, revisionId))).
+                                  KV.of((String) host.getSuperordinateDomain().getSqlKey(), kv));
+                        } else {
+                          externalHostCounter.inc();
+                          DepositFragment fragment = marshaller.marshalExternalHost(host);
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same host could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .forEach(
+                                  pendingDeposit -> {
+                                    externalHostFragmentCounter.inc();
+                                    receiver
+                                        .get(EXTERNAL_HOST_FRAGMENTS)
+                                        .output(KV.of(pendingDeposit, fragment));
+                                  });
+                        }
+                      }
+                    })
+                .withOutputTags(EXTERNAL_HOST_FRAGMENTS, TupleTagList.of(SUPERORDINATE_DOMAINS)));
+  }
+
+  /**
+   * Process subordinate hosts by making a deposit fragment with pending transfer information
+   * obtained from its superordinate domain.
+   *
+   * @param superordinateDomains Pairs of (superordinateDomainRepoId, (subordinateHostRepoId,
+   *     (pendingDeposit, revisionId))). This collection maps the subordinate host and the pending
+   *     deposit to include it to its superordinate domain.
+   * @param domainHistories Pairs of (domainRepoId, revisionId). This collection helps us find the
+   *     historical superordinate domain from its history entry and is obtained from calling {@link
+   *     #getMostRecentHistoryEntries} for domains.
+   */
+  private PCollection<KV<PendingDeposit, DepositFragment>> processSubordinateHosts(
+      PCollection<KV<String, KV<String, CoGbkResult>>> superordinateDomains,
+      PCollection<KV<String, Long>> domainHistories) {
+    Counter subordinateHostFragmentCounter = Metrics.counter("RDE", "SubordinateHostFragment");
+    Counter referencedSubordinateHostCounter = Metrics.counter("RDE", "ReferencedSubordinateHost");
+    return KeyedPCollectionTuple.of(HOST_TO_PENDING_DEPOSIT, superordinateDomains)
+        .and(REVISION_ID, domainHistories)
+        .apply(
+            "Join HostResource:PendingDeposits with DomainHistory on DomainResource",
+            CoGroupByKey.create())
+        .apply(
+            " Remove unreferenced DomainResource",
+            Filter.by(
+                kv -> {
+                  boolean toInclude =
+                      kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT).iterator().hasNext()
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedSubordinateHostCounter.inc();
+                  }
+                  return toInclude;
+                }))
+        .apply(
+            "Map subordinate HostResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      DomainBase superordinateDomain =
+                          (DomainBase)
+                              loadResourceByHistoryEntryId(
+                                  DomainHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      ImmutableSet.Builder<KV<PendingDeposit, DepositFragment>> results =
+                          new ImmutableSet.Builder<>();
+                      for (KV<String, CoGbkResult> hostToPendingDeposits :
+                          kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT)) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    hostToPendingDeposits.getKey(),
+                                    hostToPendingDeposits.getValue().getOnly(REVISION_ID));
+                        DepositFragment fragment =
+                            marshaller.marshalSubordinateHost(host, superordinateDomain);
+                        Streams.stream(hostToPendingDeposits.getValue().getAll(PENDING_DEPOSIT))
+                            .distinct()
+                            .forEach(
+                                pendingDeposit -> {
+                                  subordinateHostFragmentCounter.inc();
+                                  results.add(KV.of(pendingDeposit, fragment));
+                                });
+                      }
+                      return results.build();
+                    }));
   }
 
   /**
    * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
-   * the original TLD to pending deposit map.
+   * the original pending deposit set.
    */
   @SuppressWarnings("unchecked")
-  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+  static ImmutableSet<PendingDeposit> decodePendingDeposits(String encodedPendingDeposits) {
     try (ObjectInputStream ois =
         new ObjectInputStream(
             new ByteArrayInputStream(
-                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
-      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+                BaseEncoding.base64Url().omitPadding().decode(encodedPendingDeposits)))) {
+      return (ImmutableSet<PendingDeposit>) ois.readObject();
     } catch (IOException | ClassNotFoundException e) {
       throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
     }
   }
 
   /**
-   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
-   * worker by the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
+   * the pipeline launcher as a pipeline option.
    */
-  static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
-      oos.writeObject(pendings);
+      oos.writeObject(pendingDeposits);
       oos.flush();
       return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
     }
@@ -282,13 +684,50 @@ public class RdePipeline implements Serializable {
 
   public static void main(String[] args) throws IOException, ClassNotFoundException {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
-    RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
+    RdePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
+  /**
+   * A utility class that contains {@link TupleTag}s when {@link PCollectionTuple}s and {@link
+   * CoGbkResult}s are used.
+   */
+  protected abstract static class TupleTags {
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> DOMAIN_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_CONTACTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_HOSTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, KV<String, CoGbkResult>>> SUPERORDINATE_DOMAINS =
+        new TupleTag<KV<String, KV<String, CoGbkResult>>>() {};
+
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> EXTERNAL_HOST_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<PendingDeposit> PENDING_DEPOSIT =
+        new TupleTag<PendingDeposit>() {};
+
+    protected static final TupleTag<KV<String, CoGbkResult>> HOST_TO_PENDING_DEPOSIT =
+        new TupleTag<KV<String, CoGbkResult>>() {};
+
+    protected static final TupleTag<Long> REVISION_ID = new TupleTag<Long>() {};
+  }
+
   @Singleton
-  @Component(modules = {CredentialModule.class, ConfigModule.class})
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
   interface RdePipelineComponent {
     RdePipeline rdePipeline();
 

core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java

@@ -31,9 +31,9 @@ public interface RdePipelineOptions extends RegistryPipelineOptions {
   void setValidationMode(String value);
 
   @Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
-  String getGcsBucket();
+  String getRdeStagingBucket();
 
-  void setGcsBucket(String value);
+  void setRdeStagingBucket(String value);
 
   @Description("The Base64-encoded PGP public key to encrypt the deposits.")
   String getStagingKey();

core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java

@@ -218,7 +218,7 @@ public class SafeBrowsingTransforms {
         throws JSONException, IOException {
       int statusCode = response.getStatusLine().getStatusCode();
       if (statusCode != SC_OK) {
-        logger.atWarning().log("Got unexpected status code %s from response", statusCode);
+        logger.atWarning().log("Got unexpected status code %s from response.", statusCode);
       } else {
         // Unpack the response body
         JSONObject responseBody =
@@ -227,7 +227,7 @@ public class SafeBrowsingTransforms {
                     new InputStreamReader(response.getEntity().getContent(), UTF_8)));
         logger.atInfo().log("Got response: %s", responseBody.toString());
         if (responseBody.length() == 0) {
-          logger.atInfo().log("Response was empty, no threats detected");
+          logger.atInfo().log("Response was empty, no threats detected.");
         } else {
           // Emit all DomainNameInfos with their API results.
           JSONArray threatMatches = responseBody.getJSONArray("matches");

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -16,6 +16,7 @@ package google.registry.beam.spec11;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
@@ -30,6 +31,7 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.reporting.Spec11ThreatMatch;
 import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
 import google.registry.util.UtilsModule;
@@ -41,6 +43,7 @@ import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -58,7 +61,8 @@ import org.json.JSONObject;
 /**
  * Definition of a Dataflow Flex template, which generates a given month's spec11 report.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=spec11}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
@@ -113,15 +117,43 @@ public class Spec11Pipeline implements Serializable {
   }
 
   static PCollection<DomainNameInfo> readFromCloudSql(Pipeline pipeline) {
-    Read<Object[], DomainNameInfo> read =
+    Read<Object[], KV<String, String>> read =
         RegistryJpaIO.read(
-            "select d, r.emailAddress from Domain d join Registrar r on"
-                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
-                + " and d.deletionTime > now()",
+            "select d.repoId, r.emailAddress from Domain d join Registrar r on"
+                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL' and"
+                + " d.deletionTime > now()",
             false,
             Spec11Pipeline::parseRow);
 
-    return pipeline.apply("Read active domains from Cloud SQL", read);
+    return pipeline
+        .apply("Read active domains from Cloud SQL", read)
+        .apply(
+            "Build DomainNameInfo",
+            ParDo.of(
+                new DoFn<KV<String, String>, DomainNameInfo>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, String> input, OutputReceiver<DomainNameInfo> output) {
+                    DomainBase domainBase =
+                        jpaTm()
+                            .transact(
+                                () ->
+                                    jpaTm()
+                                        .loadByKey(
+                                            VKey.createSql(DomainBase.class, input.getKey())));
+                    String emailAddress = input.getValue();
+                    if (emailAddress == null) {
+                      emailAddress = "";
+                    }
+                    DomainNameInfo domainNameInfo =
+                        DomainNameInfo.create(
+                            domainBase.getDomainName(),
+                            domainBase.getRepoId(),
+                            domainBase.getCurrentSponsorRegistrarId(),
+                            emailAddress);
+                    output.output(domainNameInfo);
+                  }
+                }));
   }
 
   static PCollection<DomainNameInfo> readFromBigQuery(
@@ -142,17 +174,8 @@ public class Spec11Pipeline implements Serializable {
             .withTemplateCompatibility());
   }
 
-  private static DomainNameInfo parseRow(Object[] row) {
-    DomainBase domainBase = (DomainBase) row[0];
-    String emailAddress = (String) row[1];
-    if (emailAddress == null) {
-      emailAddress = "";
-    }
-    return DomainNameInfo.create(
-        domainBase.getDomainName(),
-        domainBase.getRepoId(),
-        domainBase.getCurrentSponsorRegistrarId(),
-        emailAddress);
+  private static KV<String, String> parseRow(Object[] row) {
+    return KV.of((String) row[0], (String) row[1]);
   }
 
   static void saveToSql(

core/src/main/java/google/registry/bigquery/BigqueryConnection.java

@@ -632,7 +632,7 @@ public class BigqueryConnection implements AutoCloseable {
   private static String summarizeCompletedJob(Job job) {
     JobStatistics stats = job.getStatistics();
     return String.format(
-        "Job took %,.3f seconds after a %,.3f second delay and processed %,d bytes (%s)",
+        "Job took %,.3f seconds after a %,.3f second delay and processed %,d bytes (%s).",
         (stats.getEndTime() - stats.getStartTime()) / 1000.0,
         (stats.getStartTime() - stats.getCreationTime()) / 1000.0,
         stats.getTotalBytesProcessed(),
@@ -706,17 +706,17 @@ public class BigqueryConnection implements AutoCloseable {
   }
 
   /**
-   * Helper that creates a dataset with this name if it doesn't already exist, and returns true
-   * if creation took place.
+   * Helper that creates a dataset with this name if it doesn't already exist, and returns true if
+   * creation took place.
    */
-  public boolean createDatasetIfNeeded(String datasetName) throws IOException {
+  private boolean createDatasetIfNeeded(String datasetName) throws IOException {
     if (!checkDatasetExists(datasetName)) {
       bigquery.datasets()
           .insert(getProjectId(), new Dataset().setDatasetReference(new DatasetReference()
               .setProjectId(getProjectId())
               .setDatasetId(datasetName)))
           .execute();
-      logger.atInfo().log("Created dataset: %s:%s\n", getProjectId(), datasetName);
+      logger.atInfo().log("Created dataset: %s: %s.", getProjectId(), datasetName);
       return true;
     }
     return false;
@@ -732,9 +732,8 @@ public class BigqueryConnection implements AutoCloseable {
                   .setDefaultDataset(getDataset())
                   .setDestinationTable(table))));
     } catch (BigqueryJobFailureException e) {
-      if (e.getReason().equals("duplicate")) {
-        // Table already exists.
-      } else {
+      if (!e.getReason().equals("duplicate")) {
+        // Throw if it failed for any reason other than table already existing.
         throw e;
       }
     }

core/src/main/java/google/registry/bigquery/CheckedBigquery.java

@@ -116,7 +116,7 @@ public class CheckedBigquery {
           .setTableReference(table))
           .execute();
       logger.atInfo().log(
-          "Created BigQuery table %s:%s.%s",
+          "Created BigQuery table %s:%s.%s.",
           table.getProjectId(), table.getDatasetId(), table.getTableId());
     } catch (IOException e) {
       // Swallow errors about a table that exists, and throw any other ones.

core/src/main/java/google/registry/config/CloudTasksUtilsModule.java

@@ -22,10 +22,13 @@ import dagger.Provides;
 import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.util.CloudTasksUtils;
+import google.registry.util.CloudTasksUtils.GcpCloudTasksClient;
+import google.registry.util.CloudTasksUtils.SerializableCloudTasksClient;
 import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
 import java.io.IOException;
-import javax.inject.Provider;
+import java.io.Serializable;
+import java.util.function.Supplier;
 import javax.inject.Singleton;
 
 /**
@@ -42,24 +45,35 @@ public abstract class CloudTasksUtilsModule {
   public static CloudTasksUtils provideCloudTasksUtils(
       @Config("projectId") String projectId,
       @Config("locationId") String locationId,
-      // Use a provider so that we can use try-with-resources with the client, which implements
-      // Autocloseable.
-      Provider<CloudTasksClient> clientProvider,
+      SerializableCloudTasksClient client,
       Retrier retrier) {
-    return new CloudTasksUtils(retrier, projectId, locationId, clientProvider);
+    return new CloudTasksUtils(retrier, projectId, locationId, client);
+  }
+
+  // Provides a supplier instead of using a Dagger @Provider because the latter is not serializable.
+  @Provides
+  public static Supplier<CloudTasksClient> provideCloudTasksClientSupplier(
+      @DefaultCredential GoogleCredentialsBundle credentials) {
+    return (Supplier<CloudTasksClient> & Serializable)
+        () -> {
+          CloudTasksClient client;
+          try {
+            client =
+                CloudTasksClient.create(
+                    CloudTasksSettings.newBuilder()
+                        .setCredentialsProvider(
+                            FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
+                        .build());
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+          return client;
+        };
   }
 
   @Provides
-  public static CloudTasksClient provideCloudTasksClient(
-      @DefaultCredential GoogleCredentialsBundle credentials) {
-    try {
-      return CloudTasksClient.create(
-          CloudTasksSettings.newBuilder()
-              .setCredentialsProvider(
-                  FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
-              .build());
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
+  public static SerializableCloudTasksClient provideSerializableCloudTasksClient(
+      final Supplier<CloudTasksClient> clientSupplier) {
+    return new GcpCloudTasksClient(clientSupplier);
   }
 }

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.util.TaskQueueUtils;
 import google.registry.util.YamlUtils;
 import java.lang.annotation.Documented;
@@ -392,19 +393,26 @@ public final class RegistryConfig {
 
     @Provides
     @Config("cloudSqlJdbcUrl")
-    public static String providesCloudSqlJdbcUrl(RegistryConfigSettings config) {
+    public static String provideCloudSqlJdbcUrl(RegistryConfigSettings config) {
       return config.cloudSql.jdbcUrl;
     }
 
     @Provides
     @Config("cloudSqlInstanceConnectionName")
-    public static String providesCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
+    public static String provideCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
       return config.cloudSql.instanceConnectionName;
     }
 
+    @Provides
+    @Config("cloudSqlReplicaInstanceConnectionName")
+    public static Optional<String> provideCloudSqlReplicaInstanceConnectionName(
+        RegistryConfigSettings config) {
+      return Optional.ofNullable(config.cloudSql.replicaInstanceConnectionName);
+    }
+
     @Provides
     @Config("cloudSqlDbInstanceName")
-    public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
+    public static String provideCloudSqlDbInstance(RegistryConfigSettings config) {
       // Format of instanceConnectionName: project-id:region:instance-name
       int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
       return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);
@@ -557,11 +565,23 @@ public final class RegistryConfig {
       return config.registryPolicy.requireSslCertificates;
     }
 
+    /**
+     * Returns the GCE machine type that a CPU-demanding pipeline should use.
+     *
+     * @see google.registry.beam.rde.RdePipeline
+     */
+    @Provides
+    @Config("highPerformanceMachineType")
+    public static String provideHighPerformanceMachineType(RegistryConfigSettings config) {
+      return config.beam.highPerformanceMachineType;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
      * @see google.registry.beam.invoicing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
+     * @see google.registry.beam.invoicing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")
@@ -1306,6 +1326,18 @@ public final class RegistryConfig {
     public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
       return ImmutableSet.copyOf(config.sslCertificateValidation.allowedEcdsaCurves);
     }
+
+    @Provides
+    @Config("minMonthsBeforeWipeOut")
+    public static int provideMinMonthsBeforeWipeOut(RegistryConfigSettings config) {
+      return config.contactHistory.minMonthsBeforeWipeOut;
+    }
+
+    @Provides
+    @Config("wipeOutQueryBatchSize")
+    public static int provideWipeOutQueryBatchSize(RegistryConfigSettings config) {
+      return config.contactHistory.wipeOutQueryBatchSize;
+    }
   }
 
   /** Returns the App Engine project ID, which is based off the environment name. */
@@ -1500,6 +1532,31 @@ public final class RegistryConfig {
     return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
   }
 
+  /**
+   * JDBC-specific: driver default batch size is 0, which means that every INSERT statement will be
+   * sent to the database individually. Batching allows us to group together multiple inserts into
+   * one single INSERT statement which can dramatically increase speed in situations with many
+   * inserts.
+   *
+   * <p>Hibernate docs, i.e.
+   * https://docs.jboss.org/hibernate/orm/5.6/userguide/html_single/Hibernate_User_Guide.html,
+   * recommend between 10 and 50.
+   */
+  public static String getHibernateJdbcBatchSize() {
+    return CONFIG_SETTINGS.get().hibernate.jdbcBatchSize;
+  }
+
+  /**
+   * Returns the JDBC fetch size.
+   *
+   * <p>Postgresql-specific: driver default fetch size is 0, which disables streaming result sets.
+   * Here we set a small default geared toward Nomulus server transactions. Large queries can
+   * override the defaults using {@link JpaTransactionManager#setQueryFetchSize}.
+   */
+  public static String getHibernateJdbcFetchSize() {
+    return CONFIG_SETTINGS.get().hibernate.jdbcFetchSize;
+  }
+
   /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
   public static String getContactAndHostRoidSuffix() {
     return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -41,6 +41,7 @@ public class RegistryConfigSettings {
   public Keyring keyring;
   public RegistryTool registryTool;
   public SslCertificateValidation sslCertificateValidation;
+  public ContactHistory contactHistory;
 
   /** Configuration options that apply to the entire App Engine project. */
   public static class AppEngine {
@@ -119,6 +120,8 @@ public class RegistryConfigSettings {
     public String hikariMinimumIdle;
     public String hikariMaximumPoolSize;
     public String hikariIdleTimeout;
+    public String jdbcBatchSize;
+    public String jdbcFetchSize;
   }
 
   /** Configuration for Cloud SQL. */
@@ -127,11 +130,13 @@ public class RegistryConfigSettings {
     // TODO(05012021): remove username field after it is removed from all yaml files.
     public String username;
     public String instanceConnectionName;
+    public String replicaInstanceConnectionName;
   }
 
   /** Configuration for Apache Beam (Cloud Dataflow). */
   public static class Beam {
     public String defaultJobRegion;
+    public String highPerformanceMachineType;
     public String stagingBucketUrl;
   }
 
@@ -234,4 +239,10 @@ public class RegistryConfigSettings {
     public String expirationWarningEmailBodyText;
     public String expirationWarningEmailSubjectText;
   }
+
+  /** Configuration for contact history. */
+  public static class ContactHistory {
+    public int minMonthsBeforeWipeOut;
+    public int wipeOutQueryBatchSize;
+  }
 }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -221,6 +221,17 @@ hibernate:
   hikariMinimumIdle: 1
   hikariMaximumPoolSize: 10
   hikariIdleTimeout: 300000
+  # The batch size is basically the number of insertions / updates in a single
+  # transaction that will be batched together into one INSERT/UPDATE statement.
+  # A larger batch size is useful when inserting or updating many entities in a
+  # single transaction. Hibernate docs
+  # (https://docs.jboss.org/hibernate/orm/5.6/userguide/html_single/Hibernate_User_Guide.html)
+  # recommend between 10 and 50.
+  jdbcBatchSize: 50
+  # The fetch size is the number of entities retrieved at a time from the
+  # database cursor. Here we set a small default geared toward Nomulus server
+  # transactions. Large queries can override the defaults on a per-query basis.
+  jdbcFetchSize: 20
 
 cloudSql:
   # jdbc url for the Cloud SQL database.
@@ -231,6 +242,10 @@ cloudSql:
   jdbcUrl: jdbc:postgresql://localhost
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
+  # If non-null, we will use this instance for certain read-only actions or
+  # pipelines, e.g. RDE, in order to offload some work from the primary
+  # instance. Expect any write actions on this instance to fail.
+  replicaInstanceConnectionName: null
 
 cloudDns:
   # Set both properties to null in Production.
@@ -419,6 +434,14 @@ misc:
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
   defaultJobRegion: us-east1
+  # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
+  # to check the VM CPU quota for the job region. In a massively parallel
+  # pipeline this quota can be easily reached and needs to be raised, otherwise
+  # the job will run very slowly. Also note that there is a separate quota for
+  # external IPv4 address in a region, which means that machine type with higher
+  # core count per machine may be preferable in order to preserve IP addresses.
+  # See: https://cloud.google.com/compute/quotas#cpu_quota
+  highPerformanceMachineType: n2-standard-4
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:
@@ -442,6 +465,13 @@ registryTool:
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
 
+# Configuration options for handling contact history.
+contactHistory:
+  # The number of months that a ContactHistory entity should be stored in the database.
+  minMonthsBeforeWipeOut: 18
+  # The batch size for querying ContactHistory table in the database.
+  wipeOutQueryBatchSize: 500
+
 # Configuration options for checking SSL certificates.
 sslCertificateValidation:
   # A map specifying the maximum amount of days the certificate can be valid.

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -14,18 +14,15 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
-import java.time.Duration;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import javax.inject.Inject;
 
 /** Action for fanning out cron tasks for each commit log bucket. */
@@ -38,25 +35,27 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  private static final Random random = new Random();
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
-  @Inject TaskQueueUtils taskQueueUtils;
   @Inject @Parameter("endpoint") String endpoint;
   @Inject @Parameter("queue") String queue;
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
+
+
   @Override
   public void run() {
-    Queue taskQueue = getQueue(queue);
     for (int bucketId : CommitLogBucket.getBucketIds()) {
-      long delay =
-          jitterSeconds.map(i -> random.nextInt((int) Duration.ofSeconds(i).toMillis())).orElse(0);
-      TaskOptions taskOptions =
-          TaskOptions.Builder.withUrl(endpoint)
-              .param(BUCKET_PARAM, Integer.toString(bucketId))
-              .countdownMillis(delay);
-      taskQueueUtils.enqueue(taskQueue, taskOptions);
+      cloudTasksUtils.enqueue(
+          queue,
+          CloudTasksUtils.createPostTask(
+              endpoint,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
+              clock,
+              jitterSeconds));
     }
   }
 }

core/src/main/java/google/registry/cron/TldFanoutAction.java

@@ -30,7 +30,6 @@ import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
 import static google.registry.model.tld.Registries.getTldsOfType;
 import static google.registry.model.tld.Registry.TldType.REAL;
 import static google.registry.model.tld.Registry.TldType.TEST;
-import static java.util.concurrent.TimeUnit.SECONDS;
 
 import com.google.cloud.tasks.v2.Task;
 import com.google.common.collect.ArrayListMultimap;
@@ -39,7 +38,6 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
-import com.google.protobuf.Timestamp;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
@@ -49,9 +47,7 @@ import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
-import java.time.Instant;
 import java.util.Optional;
-import java.util.Random;
 import java.util.stream.Stream;
 import javax.inject.Inject;
 
@@ -100,8 +96,6 @@ public final class TldFanoutAction implements Runnable {
           EXCLUDE_PARAM,
           JITTER_SECONDS_PARAM);
 
-  private static final Random random = new Random();
-
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   @Inject Clock clock;
@@ -141,7 +135,7 @@ public final class TldFanoutAction implements Runnable {
     StringBuilder outputPayload =
         new StringBuilder(
             String.format("OK: Launched the following %d tasks in queue %s\n", tlds.size(), queue));
-    logger.atInfo().log("Launching %d tasks in queue %s", tlds.size(), queue);
+    logger.atInfo().log("Launching %d tasks in queue %s.", tlds.size(), queue);
     if (tlds.isEmpty()) {
       logger.atWarning().log("No TLDs to fan-out!");
     }
@@ -153,7 +147,7 @@ public final class TldFanoutAction implements Runnable {
               "- Task: '%s', tld: '%s', endpoint: '%s'\n",
               createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri()));
       logger.atInfo().log(
-          "Task: '%s', tld: '%s', endpoint: '%s'",
+          "Task: '%s', tld: '%s', endpoint: '%s'.",
           createdTask.getName(), tld, createdTask.getAppEngineHttpRequest().getRelativeUri());
     }
     response.setContentType(PLAIN_TEXT_UTF_8);
@@ -165,22 +159,7 @@ public final class TldFanoutAction implements Runnable {
       params = ArrayListMultimap.create(params);
       params.put(RequestParameters.PARAM_TLD, tld);
     }
-    Instant scheduleTime =
-        Instant.ofEpochMilli(
-            clock
-                .nowUtc()
-                .plusMillis(
-                    jitterSeconds
-                        .map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
-                        .orElse(0))
-                .getMillis());
-    return Task.newBuilder(
-            CloudTasksUtils.createPostTask(endpoint, Service.BACKEND.toString(), params))
-        .setScheduleTime(
-            Timestamp.newBuilder()
-                .setSeconds(scheduleTime.getEpochSecond())
-                .setNanos(scheduleTime.getNano())
-                .build())
-        .build();
+    return CloudTasksUtils.createPostTask(
+        endpoint, Service.BACKEND.toString(), params, clock, jitterSeconds);
   }
 }

core/src/main/java/google/registry/dns/DnsQueue.java

@@ -107,7 +107,7 @@ public class DnsQueue {
   private TaskHandle addToQueue(
       TargetType targetType, String targetName, String tld, Duration countdown) {
     logger.atInfo().log(
-        "Adding task type=%s, target=%s, tld=%s to pull queue %s (%d tasks currently on queue)",
+        "Adding task type=%s, target=%s, tld=%s to pull queue %s (%d tasks currently on queue).",
         targetType, targetName, tld, DNS_PULL_QUEUE_NAME, queue.fetchStatistics().getNumTasks());
     return queue.add(
         TaskOptions.Builder.withDefaults()
@@ -166,7 +166,7 @@ public class DnsQueue {
           "There are %d tasks in the DNS queue '%s'.", numTasks, DNS_PULL_QUEUE_NAME);
       return queue.leaseTasks(leaseDuration.getMillis(), MILLISECONDS, leaseTasksBatchSize);
     } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed leasing tasks too fast");
+      logger.atSevere().withCause(e).log("Failed leasing tasks too fast.");
       return ImmutableList.of();
     }
   }
@@ -176,7 +176,7 @@ public class DnsQueue {
     try {
       queue.deleteTask(tasks);
     } catch (TransientFailureException | DeadlineExceededException e) {
-      logger.atSevere().withCause(e).log("Failed deleting tasks too fast");
+      logger.atSevere().withCause(e).log("Failed deleting tasks too fast.");
     }
   }
 }

core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java

@@ -104,7 +104,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
         new Duration(enqueuedTime, now));
     logger.atInfo().log(
         "publishDnsWriter latency statistics: TLD: %s, dnsWriter: %s, actionStatus: %s, "
-            + "numItems: %d, timeSinceCreation: %s, timeInQueue: %s",
+            + "numItems: %d, timeSinceCreation: %s, timeInQueue: %s.",
         tld,
         dnsWriter,
         status,
@@ -144,7 +144,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
 
   /** Adds all the domains and hosts in the batch back to the queue to be processed later. */
   private void requeueBatch() {
-    logger.atInfo().log("Requeueing batch for retry");
+    logger.atInfo().log("Requeueing batch for retry.");
     for (String domain : nullToEmpty(domains)) {
       dnsQueue.addDomainRefreshTask(domain);
     }
@@ -158,14 +158,14 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     // LockIndex should always be within [1, numPublishLocks]
     if (lockIndex > numPublishLocks || lockIndex <= 0) {
       logger.atSevere().log(
-          "Lock index should be within [1,%d], got %d instead", numPublishLocks, lockIndex);
+          "Lock index should be within [1,%d], got %d instead.", numPublishLocks, lockIndex);
       return false;
     }
     // Check if the Registry object's num locks has changed since this task was batched
     int registryNumPublishLocks = Registry.get(tld).getNumDnsPublishLocks();
     if (registryNumPublishLocks != numPublishLocks) {
       logger.atWarning().log(
-          "Registry numDnsPublishLocks %d out of sync with parameter %d",
+          "Registry numDnsPublishLocks %d out of sync with parameter %d.",
           registryNumPublishLocks, numPublishLocks);
       return false;
     }
@@ -179,7 +179,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     DnsWriter writer = dnsWriterProxy.getByClassNameForTld(dnsWriter, tld);
 
     if (writer == null) {
-      logger.atWarning().log("Couldn't get writer %s for TLD %s", dnsWriter, tld);
+      logger.atWarning().log("Couldn't get writer %s for TLD %s.", dnsWriter, tld);
       recordActionResult(ActionStatus.BAD_WRITER);
       requeueBatch();
       return;
@@ -190,11 +190,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     for (String domain : nullToEmpty(domains)) {
       if (!DomainNameUtils.isUnder(
           InternetDomainName.from(domain), InternetDomainName.from(tld))) {
-        logger.atSevere().log("%s: skipping domain %s not under tld", tld, domain);
+        logger.atSevere().log("%s: skipping domain %s not under TLD.", tld, domain);
         domainsRejected += 1;
       } else {
         writer.publishDomain(domain);
-        logger.atInfo().log("%s: published domain %s", tld, domain);
+        logger.atInfo().log("%s: published domain %s.", tld, domain);
         domainsPublished += 1;
       }
     }
@@ -206,11 +206,11 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
     for (String host : nullToEmpty(hosts)) {
       if (!DomainNameUtils.isUnder(
           InternetDomainName.from(host), InternetDomainName.from(tld))) {
-        logger.atSevere().log("%s: skipping host %s not under tld", tld, host);
+        logger.atSevere().log("%s: skipping host %s not under TLD.", tld, host);
         hostsRejected += 1;
       } else {
         writer.publishHost(host);
-        logger.atInfo().log("%s: published host %s", tld, host);
+        logger.atInfo().log("%s: published host %s.", tld, host);
         hostsPublished += 1;
       }
     }
@@ -233,7 +233,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
           tld, dnsWriter, commitStatus, duration, domainsPublished, hostsPublished);
       logger.atInfo().log(
           "writer.commit() statistics: TLD: %s, dnsWriter: %s, commitStatus: %s, duration: %s, "
-              + "domainsPublished: %d, domainsRejected: %d, hostsPublished: %d, hostsRejected: %d",
+              + "domainsPublished: %d, domainsRejected: %d, hostsPublished: %d, hostsRejected: %d.",
           tld,
           dnsWriter,
           commitStatus,

core/src/main/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java

@@ -180,11 +180,11 @@ public class CloudDnsWriter extends BaseDnsWriter {
 
     desiredRecords.put(absoluteDomainName, domainRecords.build());
     logger.atFine().log(
-        "Will write %d records for domain %s", domainRecords.build().size(), absoluteDomainName);
+        "Will write %d records for domain '%s'.", domainRecords.build().size(), absoluteDomainName);
   }
 
   private void publishSubordinateHost(String hostName) {
-    logger.atInfo().log("Publishing glue records for %s", hostName);
+    logger.atInfo().log("Publishing glue records for host '%s'.", hostName);
     // Canonicalize name
     String absoluteHostName = getAbsoluteHostName(hostName);
 
@@ -250,7 +250,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
 
     // Host not managed by our registry, no need to update DNS.
     if (!tld.isPresent()) {
-      logger.atSevere().log("publishHost called for invalid host %s", hostName);
+      logger.atSevere().log("publishHost called for invalid host '%s'.", hostName);
       return;
     }
 
@@ -273,7 +273,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
     ImmutableMap<String, ImmutableSet<ResourceRecordSet>> desiredRecordsCopy =
         ImmutableMap.copyOf(desiredRecords);
     retrier.callWithRetry(() -> mutateZone(desiredRecordsCopy), ZoneStateException.class);
-    logger.atInfo().log("Wrote to Cloud DNS");
+    logger.atInfo().log("Wrote to Cloud DNS.");
   }
 
   /** Returns the glue records for in-bailiwick nameservers for the given domain+records. */
@@ -329,7 +329,7 @@ public class CloudDnsWriter extends BaseDnsWriter {
    */
   private Map<String, List<ResourceRecordSet>> getResourceRecordsForDomains(
       Set<String> domainNames) {
-    logger.atFine().log("Fetching records for %s", domainNames);
+    logger.atFine().log("Fetching records for domain '%s'.", domainNames);
     // As per Concurrent.transform() - if numThreads or domainNames.size() < 2, it will not use
     // threading.
     return ImmutableMap.copyOf(
@@ -381,11 +381,11 @@ public class CloudDnsWriter extends BaseDnsWriter {
     ImmutableSet<ResourceRecordSet> intersection =
         Sets.intersection(additions, deletions).immutableCopy();
     logger.atInfo().log(
-        "There are %d common items out of the %d items in 'additions' and %d items in 'deletions'",
+        "There are %d common items out of the %d items in 'additions' and %d items in 'deletions'.",
         intersection.size(), additions.size(), deletions.size());
     // Exit early if we have nothing to update - dnsConnection doesn't work on empty changes
     if (additions.equals(deletions)) {
-      logger.atInfo().log("Returning early because additions is the same as deletions");
+      logger.atInfo().log("Returning early because additions are the same as deletions.");
       return;
     }
     Change change =

core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml

@@ -208,6 +208,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
     <description>

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -157,6 +157,12 @@
     <url-pattern>/_dr/cron/readDnsQueue</url-pattern>
   </servlet-mapping>
 
+  <!-- Replicates SQL transactions to Datastore during the Registry 3.0 migration. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/cron/replicateToDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Publishes DNS updates. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>
@@ -391,6 +397,13 @@
     <url-pattern>/_dr/task/relockDomain</url-pattern>
   </servlet-mapping>
 
+  <!-- Background action to wipe out PII fields of ContactHistory entities that
+have been in the database for a certain period of time. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutContactHistoryPii</url-pattern>
+  </servlet-mapping>
+
   <!-- Action to wipeout Cloud SQL data -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/env/crash/default/WEB-INF/cron.xml

@@ -211,6 +211,15 @@
   </cron>
   -->
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <!--
     The next two wipeout jobs are required when crash has production data.
    -->

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -36,6 +36,19 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url>/_dr/task/rdeStaging?beam=true</url>
+    <description>
+      This job generates a full RDE escrow deposit as a single gigantic XML
+      document using the Beam pipeline regardless of the current TM
+      configuration and streams it to cloud storage. It does not trigger the
+      subsequent upload tasks and is meant to run parallel with the main cron
+      job in order to compare the results from both runs.
+    </description>
+    <schedule>every 8 hours from 00:07 to 20:00</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
     <description>
@@ -348,4 +361,23 @@
     <schedule>every 3 minutes</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -120,6 +120,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
     <description>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -246,4 +246,23 @@
     <schedule>every 3 minutes</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/export/AnnotatedEntities.java

@@ -21,12 +21,14 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Ordering;
 import com.googlecode.objectify.Key;
 import google.registry.model.EntityClasses;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.annotations.VirtualEntity;
 
 /** Constants related to export code. */
+@DeleteAfterMigration
 public final class AnnotatedEntities {
 
   /** Returns the names of kinds to include in Datastore backups. */

core/src/main/java/google/registry/export/BackupDatastoreAction.java

@@ -21,6 +21,7 @@ import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig;
 import google.registry.export.datastore.DatastoreAdmin;
 import google.registry.export.datastore.Operation;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
 import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.request.Response;
@@ -46,6 +47,7 @@ import javax.inject.Inject;
     method = POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class BackupDatastoreAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -80,7 +82,7 @@ public class BackupDatastoreAction implements Runnable {
       logger.atInfo().log(message);
       response.setPayload(message);
     } catch (Throwable e) {
-      throw new InternalServerErrorException("Exception occurred while backing up datastore.", e);
+      throw new InternalServerErrorException("Exception occurred while backing up Datastore", e);
     }
   }
 }