Add a bit of documentation about the replica config (#1488)

* Add support for null or empty params

* Add Null or empty check in CollectionUtils

* Remove content type header for empty params in POST request

.gcloudignore

@@ -1,4 +1,5 @@
 python/
+node_modules/
 **/build/
 **/out/
 .*/

.gitignore

@@ -4,6 +4,7 @@
 ######################################################################
 # Java Ignores
 
+gjf.out
 *.class
 
 # Mobile Tools for Java (J2ME)
@@ -111,3 +112,7 @@ core/**/registrar_bin*.js
 core/**/registrar_dbg*.js
 core/**/registrar_bin*.css
 core/**/registrar_dbg*.css
+
+# Appengine generated files
+core/WEB-INF/appengine-generated/*.bin
+core/WEB-INF/appengine-generated/*.xml

SECURITY.md

@@ -0,0 +1,4 @@
+To report a security issue, please use http://g.co/vulnz. We use
+http://g.co/vulnz for our intake, and do coordination and disclosure here on
+GitHub (including using GitHub Security Advisory). The Google Security Team will
+respond within 5 working days of your report on g.co/vulnz.

build.gradle

@@ -20,11 +20,15 @@ buildscript {
     // Lock buildscript dependencies.
     configurations.classpath {
       resolutionStrategy.activateDependencyLocking()
+
+      // log4j has high-profile security vulnerabilities. It's a transitive
+      // dependency used by Gradle itself during build, and not strictly needed.
+      exclude group: 'org.apache.logging.log4j'
     }
   }
 
   dependencies {
-    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.0.1'
+    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.4.1'
     classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.6.1'
     classpath 'org.sonatype.aether:aether-api:1.13.1'
     classpath 'org.sonatype.aether:aether-impl:1.13.1'
@@ -46,7 +50,7 @@ plugins {
   id 'com.diffplug.gradle.spotless' version '3.25.0'
 
   id 'jacoco'
-  id 'com.dorongold.task-tree' version '1.5'
+  id 'com.dorongold.task-tree' version '2.1.0'
 }
 
 node {
@@ -182,6 +186,12 @@ allprojects {
       println "Java dependencies: Using Maven Central..."
       mavenCentral()
       google()
+      maven {
+        url "https://packages.confluent.io/maven/"
+        content {
+          includeGroup "io.confluent"
+        }
+      }
     }
   }
 
@@ -196,9 +206,41 @@ allprojects {
   }
 }
 
+rootProject.ext {
+  pyver = { exe ->
+    try {
+      ext.execInBash(
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
+    } catch (org.gradle.process.internal.ExecException e) {
+      return -1;
+    }
+  }
+
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
+    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
+    // Debian so it seems like that should be available anywhere).
+    def MIN_PY_VER = 0x3070300
+    if (pyver('python') >= MIN_PY_VER) {
+      return 'python'
+    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
+      return '/usr/bin/python3'
+    } else {
+      throw new GradleException("No usable Python version found (build " +
+                                "requires at least python 3.7.3)");
+    }
+  }
+}
+
 task runPresubmits(type: Exec) {
-  executable '/usr/bin/python3'
+
   args('config/presubmits.py')
+
+  doFirst {
+    executable getPythonExecutable()
+  }
 }
 
 def javadocSource = []
@@ -318,7 +360,7 @@ subprojects {
   // expose to users.
   if (project.name != 'docs') {
     javadocSource << project.sourceSets.main.allJava
-    javadocClasspath << project.sourceSets.main.compileClasspath
+    javadocClasspath << project.sourceSets.main.runtimeClasspath
     javadocClasspath << "${buildDir}/generated/sources/annotationProcessor/java/main"
     javadocDependentTasks << project.tasks.compileJava
   }
@@ -412,9 +454,10 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
@@ -422,18 +465,23 @@ rootProject.ext {
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
-    def checkResult = invokeJavaDiffFormatScript("check")
-    if (checkResult == 'true') {
-      throw new IllegalStateException(
-          "Some Java files need to be reformatted. You may use the "
-              + "'javaIncrementalFormatDryRun' task to review\n "
-              + "the changes, or the 'javaIncrementalFormatApply' task "
-              + "to reformat.")
-    } else if (checkResult != 'false') {
-      throw new RuntimeException(
-          "Failed to invoke format check script:\n" + checkResult)
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
+      def checkResult = invokeJavaDiffFormatScript("check")
+      if (checkResult == 'true') {
+        throw new IllegalStateException(
+            "Some Java files need to be reformatted. You may use the "
+                + "'javaIncrementalFormatDryRun' task to review\n "
+                + "the changes, or the 'javaIncrementalFormatApply' task "
+                + "to reformat.")
+      } else if (checkResult != 'false') {
+        throw new RuntimeException(
+            "Failed to invoke format check script:\n" + checkResult)
+      }
+      println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
     }
-    println("Incremental Java format check ok.")
   }
 }
 
@@ -463,7 +511,8 @@ task javadoc(type: Javadoc) {
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
   options.tags = ["type:a:Generic Type",
-                  "error:a:Expected Error"]
+                  "error:a:Expected Error",
+                  "invariant:a:Guaranteed Property"]
 }
 
 tasks.build.dependsOn(tasks.javadoc)
@@ -481,3 +530,15 @@ task coreDev {
 }
 
 javadocDependentTasks.each { tasks.javadoc.dependsOn(it) }
+
+// disable javadoc in subprojects, these will break because they don't have
+// the correct classpath (see above).
+gradle.taskGraph.whenReady { graph ->
+  graph.getAllTasks().each { task ->
+    def subprojectJavadoc = (task.path =~ /:.+:javadoc/)
+    if (subprojectJavadoc) {
+        println "Skipping ${task.path} for javadoc (only root javadoc works)"
+        task.enabled = false
+    }
+  }
+}

buildSrc/build.gradle

@@ -72,6 +72,7 @@ dependencies {
   compile deps['com.google.auth:google-auth-library-credentials']
   compile deps['com.google.auth:google-auth-library-oauth2-http']
   compile deps['com.google.auto.value:auto-value-annotations']
+  compile deps['com.google.common.html.types:types']
   compile deps['com.google.cloud:google-cloud-core']
   compile deps['com.google.cloud:google-cloud-storage']
   compile deps['com.google.guava:guava']

buildSrc/gradle/dependency-locks/compile.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/compileClasspath.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/deploy_jar.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/runtime.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/testCompile.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/testRuntime.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0

buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0

buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java

@@ -23,6 +23,7 @@ import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.html.types.TrustedResourceUrls;
 import com.google.template.soy.SoyFileSet;
 import com.google.template.soy.tofu.SoyTofu;
 import google.registry.gradle.plugin.ProjectData.TaskData;
@@ -118,7 +119,7 @@ final class CoverPageGenerator {
 
     builder.put("projectState", state.toString());
     builder.put("title", title);
-    builder.put("cssFiles", ImmutableSet.of("css/style.css"));
+    builder.put("cssFiles", ImmutableSet.of(TrustedResourceUrls.fromConstant("css/style.css")));
     builder.put("invocation", getInvocation());
     builder.put("tasksByState", getTasksByStateSoyData());
     return builder.build();

buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java

@@ -91,7 +91,7 @@ abstract class ProjectData {
       /** The task was actually run and has finished successfully. */
       SUCCESS,
       /** The task was up-to-date and successful, and hence didn't need to run again. */
-      UP_TO_DATE;
+      UP_TO_DATE
     }
 
     abstract String uniqueName();

buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy

@@ -16,7 +16,7 @@
 
 {template .coverPage}
   {@param title: string}
-  {@param cssFiles: list<string>}
+  {@param cssFiles: list<trusted_resource_uri>}
   {@param projectState: string}
   {@param invocation: string}
   {@param tasksByState: map<string, list<[uniqueName: string, description: string, log: string, reports: map<string, string>]>>}

common/gradle/dependency-locks/compile.lockfile

@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0

common/gradle/dependency-locks/compileClasspath.lockfile

@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0

common/gradle/dependency-locks/default.lockfile

@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0

common/gradle/dependency-locks/deploy_jar.lockfile

@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0

common/gradle/dependency-locks/runtime.lockfile

@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0

common/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0

common/gradle/dependency-locks/testCompile.lockfile

@@ -4,9 +4,9 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -4,9 +4,9 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/gradle/dependency-locks/testRuntime.lockfile

@@ -4,10 +4,10 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -4,10 +4,10 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/gradle/dependency-locks/testingCompile.lockfile

@@ -4,9 +4,9 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/gradle/dependency-locks/testingCompileClasspath.lockfile

@@ -4,9 +4,9 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/gradle/dependency-locks/testingRuntime.lockfile

@@ -4,10 +4,10 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/gradle/dependency-locks/testingRuntimeClasspath.lockfile

@@ -4,10 +4,10 @@
 com.google.auto.value:auto-value-annotations:1.7.4
 com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2

common/src/main/java/google/registry/util/DateTimeUtils.java

@@ -100,6 +100,15 @@ public class DateTimeUtils {
     return ZonedDateTime.ofInstant(instant, ZoneId.of(dateTime.getZone().getID()).normalized());
   }
 
+  /**
+   * Converts a Joda {@link DateTime} object to an equivalent java.time {@link ZonedDateTime}
+   * object.
+   */
+  public static ZonedDateTime toZonedDateTime(DateTime dateTime, ZoneId zoneId) {
+    java.time.Instant instant = java.time.Instant.ofEpochMilli(dateTime.getMillis());
+    return ZonedDateTime.ofInstant(instant, zoneId);
+  }
+
   /**
    * Converts a java.time {@link ZonedDateTime} object to an equivalent Joda {@link DateTime}
    * object.

common/src/testing/java/google/registry/testing/FakeClock.java

@@ -22,6 +22,7 @@ import google.registry.util.Clock;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.concurrent.ThreadSafe;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 import org.joda.time.ReadableDuration;
 import org.joda.time.ReadableInstant;
 
@@ -35,6 +36,8 @@ public final class FakeClock implements Clock {
   // threads should see a consistent flow.
   private final AtomicLong currentTimeMillis = new AtomicLong();
 
+  private volatile long autoIncrementStepMs;
+
   /** Creates a FakeClock that starts at START_OF_TIME. */
   public FakeClock() {
     this(START_OF_TIME);
@@ -48,7 +51,21 @@ public final class FakeClock implements Clock {
   /** Returns the current time. */
   @Override
   public DateTime nowUtc() {
-    return new DateTime(currentTimeMillis.get(), UTC);
+    return new DateTime(currentTimeMillis.addAndGet(autoIncrementStepMs), UTC);
+  }
+
+  /**
+   * Sets the increment applied to the clock whenever it is queried. The increment is zero by
+   * default: the clock is left unchanged when queried.
+   *
+   * <p>Passing a duration of zero to this method effectively unsets the auto increment mode.
+   *
+   * @param autoIncrementStep the new auto increment duration
+   * @return this
+   */
+  public FakeClock setAutoIncrementStep(ReadableDuration autoIncrementStep) {
+    this.autoIncrementStepMs = autoIncrementStep.getMillis();
+    return this;
   }
 
   /** Advances clock by one millisecond. */
@@ -65,4 +82,14 @@ public final class FakeClock implements Clock {
   public void setTo(ReadableInstant time) {
     currentTimeMillis.set(time.getMillis());
   }
+
+  /** Invokes {@link #setAutoIncrementStep} with one millisecond-step. */
+  public FakeClock setAutoIncrementByOneMilli() {
+    return setAutoIncrementStep(Duration.millis(1));
+  }
+
+  /** Disables the auto-increment mode. */
+  public FakeClock disableAutoIncrement() {
+    return setAutoIncrementStep(Duration.ZERO);
+  }
 }

common/src/testing/java/google/registry/testing/SystemInfo.java

@@ -39,7 +39,7 @@ public final class SystemInfo {
                     pid.getOutputStream().close();
                     pid.waitFor();
                   } catch (IOException e) {
-                    logger.atWarning().withCause(e).log("%s command not available", cmd);
+                    logger.atWarning().withCause(e).log("%s command not available.", cmd);
                     return false;
                   }
                   return true;

config/dependency-license/allowed_licenses.json

@@ -24,6 +24,9 @@
     {
       "moduleLicense": "Apache License v2.0"
     },
+    {
+      "moduleLicense": "Apache License V2.0"
+    },
     {
       "moduleLicense": "Apache License, Version 2.0"
     },

config/nom_build.py

@@ -140,6 +140,8 @@ PROPERTIES = [
              'a BEAM pipeline to image. Setting this property to empty string '
              'will disable image generation.',
              '/usr/bin/dot'),
+    Property('pipeline',
+             'The name of the Beam pipeline being staged.')
 ]
 
 GRADLE_FLAGS = [
@@ -256,6 +258,7 @@ GRADLE_FLAGS = [
                'Specify a task to be excluded from execution.',
                True),
 ]
+
 def generate_gradle_properties() -> str:
     """Returns the expected contents of gradle.properties."""
     out = io.StringIO()
@@ -270,7 +273,7 @@ def generate_gradle_properties() -> str:
 def get_root() -> str:
     """Returns the root of the nomulus build tree."""
     cur_dir = os.getcwd()
-    if not os.path.exists(os.path.join(cur_dir, '.git')) or \
+    if not os.path.exists(os.path.join(cur_dir, 'buildSrc')) or \
        not os.path.exists(os.path.join(cur_dir, 'core')) or \
        not os.path.exists(os.path.join(cur_dir, 'gradle.properties')):
         raise Exception('You must run this script from the root directory')

config/presubmits.py

@@ -17,9 +17,11 @@ These aren't built in to the static code analysis tools we use (e.g. Checkstyle,
 Error Prone) so we must write them manually.
 """
 
+import json
 import os
 from typing import List, Tuple
 import sys
+import textwrap
 import re
 
 # We should never analyze any generated files
@@ -28,6 +30,13 @@ UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/"
 FORBIDDEN = 1
 REQUIRED = 2
 
+# The list of expected json packages and their licenses.
+# These should be one of the allowed licenses in:
+# config/dependency-license/allowed_licenses.json
+EXPECTED_JS_PACKAGES = [
+    'google-closure-library',   # Owned by Google, Apache 2.0
+]
+
 
 class PresubmitCheck:
 
@@ -79,9 +88,9 @@ PRESUBMITS = {
         r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
         ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
             ".git", "/build/", "/generated/", "/generated_tests/",
-            "node_modules/", "JUnitBackports.java", "registrar_bin.",
+            "node_modules/", "LoggerConfig.java", "registrar_bin.",
             "registrar_dbg.", "google-java-format-diff.py",
-            "nomulus.golden.sql", "soyutils_usegoog.js"
+            "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
         }, REQUIRED):
         "File did not include the license header.",
 
@@ -110,9 +119,10 @@ PRESUBMITS = {
       "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
+    # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
-        "java", {}):
+        "java", {"DatabaseSnapshotTest.java"}):
       "PostgreSQLContainer instantiation must specify docker tag.",
 
     # Various Soy linting checks
@@ -177,49 +187,6 @@ PRESUBMITS = {
         {"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
     ):
         "JavaScript files should not include console logging.",
-    # SQL injection protection rule for java source file:
-    # The sql template passed to createQuery/createNativeQuery methods must be
-    # a variable name in UPPER_CASE_UNDERSCORE format, i.e., a static final
-    # String variable. This forces the use of parameter-binding on all queries
-    # that take parameters.
-    # The rule would forbid invocation of createQuery(Criteria). However, this
-    # can be handled by adding a helper method in an exempted class to make
-    # the calls.
-    # TODO(b/179158393): enable the 'ConstantName' Java style check to ensure
-    # that non-final variables do not use the UPPER_CASE_UNDERSCORE format.
-    PresubmitCheck(
-        # Line 1: the method names we check and the opening parenthesis, which
-        #    marks the beginning of the first parameter
-        # Line 2: The first parameter is a match if is NOT any of the following:
-        #    - final variable name: \s*([A-Z_]+
-        #    - string literal: "([^"]|\\")*"
-        #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
-        # Line 3: , or the closing parenthesis, marking the end of the first
-        #    parameter
-        r'.*\.(query|createQuery|createNativeQuery)\('
-        r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
-        r'(,|\s*\))))',
-        "java",
-        # ActivityReportingQueryBuilder deals with Dremel queries
-        {"src/test", "ActivityReportingQueryBuilder.java",
-         # TODO(b/179158393): Remove everything below, which should be done
-         # using Criteria
-         "ForeignKeyIndex.java",
-         "HistoryEntryDao.java",
-         "JpaTransactionManager.java",
-         "JpaTransactionManagerImpl.java",
-         # CriteriaQueryBuilder is a false positive
-         "CriteriaQueryBuilder.java",
-         "RdapDomainSearchAction.java",
-         "RdapNameserverSearchAction.java",
-         "RdapSearchActionBase.java",
-         },
-    ):
-        "The first String parameter to EntityManager.create(Native)Query "
-        "methods must be one of the following:\n"
-        "  - A String literal\n"
-        "  - Concatenation of String literals only\n"
-        "  - The name of a static final String variable"
 }
 
 # Note that this regex only works for one kind of Flyway file.  If we want to
@@ -307,6 +274,26 @@ def verify_flyway_index():
   return not success
 
 
+def verify_javascript_deps():
+  """Verifies that we haven't introduced any new javascript dependencies."""
+  with open('package.json') as f:
+    package = json.load(f)
+
+  deps = list(package['dependencies'].keys())
+  if deps != EXPECTED_JS_PACKAGES:
+    print('Unexpected javascript dependencies.  Was expecting '
+          '%s, got %s.' % (EXPECTED_JS_PACKAGES, deps))
+    print(textwrap.dedent("""
+        * If the new dependencies are intentional, please verify that the
+        * license is one of the allowed licenses (see
+        * config/dependency-license/allowed_licenses.json) and add an entry
+        * for the package (with the license in a comment) to the
+        * EXPECTED_JS_PACKAGES variable in config/presubmits.py.
+        """))
+    return True
+  return False
+
+
 def get_files():
   for root, dirnames, filenames in os.walk("."):
     for filename in filenames:
@@ -314,6 +301,7 @@ def get_files():
 
 
 if __name__ == "__main__":
+  print('python version is %s' % sys.version)
   failed = False
   for file in get_files():
     error_messages = []
@@ -330,5 +318,8 @@ if __name__ == "__main__":
   # when we put it here it fails fast before all of the tests are run.
   failed |= verify_flyway_index()
 
+  # Make sure we haven't introduced any javascript dependencies.
+  failed |= verify_javascript_deps()
+
   if failed:
     sys.exit(1)

config/show_upgrade_diffs.py

@@ -138,8 +138,8 @@ def main():
     if args.directory and os.path.exists(dir):
         pr(f'Reusing directory {dir}\n')
         os.chdir(dir)
-        run('git', 'fetch', 'git@github.com:google/nomulus', 'master:master')
-        run('git', 'checkout', 'master')
+        run('git', 'fetch', 'git@github.com:google/nomulus', 'master')
+        run('git', 'checkout', 'origin/master')
     else:
         run('git', 'clone', 'git@github.com:google/nomulus', dir)
         os.chdir(dir)
@@ -175,7 +175,7 @@ def main():
         # Print the list of packages that were removed.
         for package in old_packages:
             if package not in new_packages:
-                pr('removed ', b':'.join(package))
+                pr('removed ', b':'.join(package), '\n')
     else:
         pr('Package versions not updated!\n')
 

core/build.gradle

@@ -44,7 +44,6 @@ def outcastTestPatterns = [
     "google/registry/flows/domain/DomainCreateFlowTest.*",
     "google/registry/flows/domain/DomainUpdateFlowTest.*",
     "google/registry/tools/CreateDomainCommandTest.*",
-    "google/registry/tools/server/CreatePremiumListActionTest.*",
 ]
 
 // Tests that fail when running Gradle in a docker container, e. g. when
@@ -70,15 +69,14 @@ def dockerIncompatibleTestPatterns = [
 // Nomulus classes, e.g., threads and objects retained by frameworks.
 // TODO(weiminyu): identify cause and fix offending tests.
 def fragileTestPatterns = [
-    // Problem seems to lie with AppEngine TaskQueue for test.
-    "google/registry/cron/TldFanoutActionTest.*",
     // Test Datastore inexplicably aborts transaction.
     "google/registry/model/tmch/ClaimsListShardTest.*",
-    // Creates large object (64MBytes), occasionally throws OOM error.
-    "google/registry/model/server/KmsSecretRevisionTest.*",
     // Changes cache timeouts and for some reason appears to have contention
     // with other tests.
     "google/registry/whois/WhoisCommandFactoryTest.*",
+    // Currently changes a global configuration parameter that for some reason
+    // results in timestamp inversions for other tests.  TODO(mmuller): fix.
+    "google/registry/flows/host/HostInfoFlowTest.*",
 ] + dockerIncompatibleTestPatterns
 
 sourceSets {
@@ -184,6 +182,7 @@ dependencies {
   compile deps['com.google.monitoring-client:metrics']
   compile deps['com.google.monitoring-client:stackdriver']
   compile deps['com.google.api-client:google-api-client-java6']
+  compile deps['com.google.api.grpc:proto-google-cloud-tasks-v2']
   compile deps['com.google.apis:google-api-services-admin-directory']
   compile deps['com.google.apis:google-api-services-appengine']
   compile deps['com.google.apis:google-api-services-bigquery']
@@ -194,6 +193,7 @@ dependencies {
   compile deps['com.google.apis:google-api-services-groupssettings']
   compile deps['com.google.apis:google-api-services-monitoring']
   compile deps['com.google.apis:google-api-services-sheets']
+  compile deps['com.google.apis:google-api-services-storage']
   testCompile deps['com.google.appengine:appengine-api-stubs']
   compile deps['com.google.appengine.tools:appengine-gcs-client']
   compile deps['com.google.appengine.tools:appengine-mapreduce']
@@ -215,9 +215,13 @@ dependencies {
   compile deps['com.google.flogger:flogger']
   runtime deps['com.google.flogger:flogger-system-backend']
   compile deps['com.google.guava:guava']
+  compile deps['com.google.protobuf:protobuf-java']
   gradleLint.ignore('unused-dependency') {
     compile deps['com.google.gwt:gwt-user']
   }
+  compile deps['com.google.cloud:google-cloud-core']
+  compile deps['com.google.cloud:google-cloud-storage']
+  compile deps['com.google.cloud:google-cloud-tasks']
   compile deps['com.google.http-client:google-http-client']
   compile deps['com.google.http-client:google-http-client-appengine']
   compile deps['com.google.http-client:google-http-client-jackson2']
@@ -253,13 +257,12 @@ dependencies {
   compile deps['org.apache.beam:beam-sdks-java-core']
   compile deps['org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core']
   compile deps['org.apache.beam:beam-sdks-java-io-google-cloud-platform']
-  testCompile deps['org.apache.commons:commons-lang3']
+  compile deps['org.apache.commons:commons-lang3']
   testCompile deps['org.apache.commons:commons-text']
   testCompile deps['org.apache.ftpserver:ftplet-api']
   testCompile deps['org.apache.ftpserver:ftpserver-core']
   compile deps['org.apache.httpcomponents:httpclient']
   compile deps['org.apache.httpcomponents:httpcore']
-  runtime deps['org.apache.logging.log4j:log4j-core']
   testCompile deps['org.apache.sshd:sshd-core']
   testCompile deps['org.apache.sshd:sshd-scp']
   testCompile deps['org.apache.sshd:sshd-sftp']
@@ -312,11 +315,13 @@ dependencies {
   annotationProcessor project(':processor')
   testAnnotationProcessor project(':processor')
 
+  testCompile deps['com.google.cloud:google-cloud-nio']
   testCompile deps['com.google.appengine:appengine-testing']
   testCompile deps['com.google.guava:guava-testlib']
   testCompile deps['com.google.monitoring-client:contrib']
   testCompile deps['com.google.truth:truth']
   testCompile deps['com.google.truth.extensions:truth-java8-extension']
+  testCompile deps['org.checkerframework:checker-qual']
   testCompile deps['org.hamcrest:hamcrest']
   testCompile deps['org.hamcrest:hamcrest-core']
   testCompile deps['org.hamcrest:hamcrest-library']
@@ -423,7 +428,7 @@ task jaxbToJava {
       }
     }
     execInBash(
-        'find . -name *.java -exec sed -i /\\*\\ \\<p\\>\\$/d {} +',
+        "find . -name *.java -exec sed -i -e '/" + /\* <p>$/ + "/d' {} +",
         generatedDir)
   }
 }
@@ -432,12 +437,9 @@ task soyToJava {
   // Relative paths of soy directories.
   def spec11SoyDir = "google/registry/reporting/spec11/soy"
   def toolsSoyDir = "google/registry/tools/soy"
-  def uiSoyDir = "google/registry/ui/soy"
   def registrarSoyDir = "google/registry/ui/soy/registrar"
 
-  def soyRelativeDirs = [
-          spec11SoyDir, toolsSoyDir, uiSoyDir, registrarSoyDir,
-  ]
+  def soyRelativeDirs = [spec11SoyDir, toolsSoyDir, registrarSoyDir]
   soyRelativeDirs.each {
     inputs.dir "${resourcesSourceDir}/${it}"
     outputs.dir "${generatedDir}/${it}"
@@ -451,7 +453,24 @@ task soyToJava {
               "--outputDirectory", "${outputDirectory}",
               "--javaClassNameSource", "filename",
               "--allowExternalCalls", "true",
-              "--srcs", "${soyFiles.join(',')}"
+              "--srcs", "${soyFiles.join(',')}",
+              "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
+    }
+
+    // Replace the "@link" tags after the "@deprecated" tags in the generated
+    // files.  The soy compiler doesn't generate imports for these, causing
+    // us to get warnings when we generate javadocs.
+    // TODO(b/200296387): To be fair, the deprecations are accurate: we're
+    // using the old "SoyInfo" classes instead of the new "Templates" files.
+    // When we convert to the new classes, this hack can go away.
+    def outputs = fileTree(outputDirectory) {
+      include '**/*.java'
+    }
+
+    outputs.each { file ->
+      exec {
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
+      }
     }
   }
 
@@ -468,14 +487,6 @@ task soyToJava {
                     dir: "${resourcesSourceDir}/${registrarSoyDir}",
                     include: ['**/*.soy']))
 
-    soyToJava('google.registry.ui.soy',
-            "${generatedDir}/${uiSoyDir}",
-            files {
-              file("${resourcesSourceDir}/${uiSoyDir}").listFiles()
-            }.filter {
-              it.name.endsWith(".soy")
-            })
-
     soyToJava('google.registry.reporting.spec11.soy',
             "${generatedDir}/${spec11SoyDir}",
             fileTree(
@@ -484,42 +495,24 @@ task soyToJava {
   }
 }
 
-task soyToJS {
-  def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy"
-  def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy"
+task soyToJS(type: JavaExec) {
+  def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy/registrar"
+  def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy/registrar"
   inputs.dir rootSoyDirectory
   outputs.dir outputSoyDirectory
 
-  ext.soyToJS = { outputDirectory, soyFiles , deps->
-    javaexec {
-      main = "com.google.template.soy.SoyToJsSrcCompiler"
-      classpath configurations.soy
+  def inputSoyFiles = files {
+    file("${rootSoyDirectory}").listFiles()
+  }.filter {
+    it.name.endsWith(".soy")
+  }
 
-      args "--outputPathFormat", "${outputDirectory}/{INPUT_FILE_NAME}.js",
+  classpath configurations.soy
+  main = "com.google.template.soy.SoyToJsSrcCompiler"
+  args "--outputPathFormat", "${outputSoyDirectory}/{INPUT_FILE_NAME}.js",
           "--allowExternalCalls", "false",
-          "--srcs", "${soyFiles.join(',')}",
-          "--shouldProvideRequireSoyNamespaces", "true",
+          "--srcs", "${inputSoyFiles.join(',')}",
           "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
-      if (deps != "") {
-        args "--deps", "${deps.join(',')}"
-      }
-    }
-  }
-
-  doLast {
-    def rootSoyFiles =
-      fileTree(
-        dir: "${rootSoyDirectory}",
-        include: ['*.soy'])
-
-    soyToJS("${outputSoyDirectory}", rootSoyFiles, "")
-    soyToJS("${outputSoyDirectory}/registrar",
-      files {
-        file("${rootSoyDirectory}/registrar").listFiles()
-      }.filter {
-        it.name.endsWith(".soy")
-      }, rootSoyFiles)
-  }
 }
 
 task stylesheetsToJavascript {
@@ -602,8 +595,8 @@ task compileProdJS(type: JavaExec) {
   closureArgs << "--generate_exports"
 
   // manually include all the required js files
-  closureArgs << "--js=${nodeModulesDir}/google-closure-library/**.js"
-  closureArgs << "--js=${jsDir}/soyutils_usegoog.js"
+  closureArgs << "--js=${nodeModulesDir}/google-closure-library/**/*.js"
+  closureArgs << "--js=${jsDir}/*.js"
   closureArgs << "--js=${cssSourceDir}/registrar_bin.css.js"
   closureArgs << "--js=${jsSourceDir}/**.js"
   closureArgs << "--js=${externsDir}/json.js"
@@ -630,15 +623,6 @@ compileProdJS.dependsOn processResources
 compileProdJS.dependsOn processTestResources
 compileProdJS.dependsOn soyToJS
 
-task karmaTest(type: Exec) {
-  dependsOn ':npmInstall'
-  workingDir rootProject.projectDir
-  executable 'node_modules/karma/bin/karma'
-  args('start', "${project.projectDir}/karma.conf.js")
-}
-
-test.dependsOn karmaTest
-
 // Make testing artifacts available to be depended up on by other projects.
 // TODO: factor out google.registry.testing to be a separate project.
 task testJar(type: Jar) {
@@ -692,9 +676,9 @@ Optional<List<String>> getToolArgsList() {
 
 // To run the nomulus tools with these command line tokens:
 // "--foo", "bar baz", "--qux=quz"
-//   gradle registryTool --args="--foo 'bar baz' --qux=quz"
+//   gradle core:registryTool --args="--foo 'bar baz' --qux=quz"
 // or:
-//   gradle registryTool --PtoolArgs="--foo|bar baz|--qux=quz"
+//   gradle core:registryTool -PtoolArgs="--foo|bar baz|--qux=quz"
 // Note that the delimiting pipe can be backslash escaped if it is part of a
 // parameter.
 ext.createToolTask = {
@@ -718,33 +702,19 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
+createToolTask(
+    'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
 
 createToolTask(
-        'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
 
-project.tasks.create('initSqlPipeline', JavaExec) {
-  main = 'google.registry.beam.initsql.InitSqlPipeline'
 
-  doFirst {
-    getToolArgsList().ifPresent {
-      args it
-    }
+createToolTask(
+    'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
 
-    def isDirectRunner =
-        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
-    // The dependency containing DirectRunner is intentionally excluded from the
-    // production binary, so that it won't be chosen by mistake: we definitely do
-    // not want to use it for the real jobs, yet DirectRunner is the default if
-    // the user forgets to override it.
-    // DirectRunner is required for tests and is already on testRuntimeClasspath.
-    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
-    // of defining a new configuration just for the DirectRunner dependency.
-    classpath =
-        isDirectRunner
-            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
-            : sourceSets.main.runtimeClasspath
-  }
-}
+createToolTask(
+    'createSyntheticHistoryEntries',
+    'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
 // Caller must provide projectId, GCP region, runner, and the kinds to delete
 // (comma-separated kind names or '*' for all). E.g.:
@@ -787,38 +757,57 @@ createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
 // This packages more code and dependency than necessary. However, without
 // restructuring the source tree it is difficult to generate leaner jars.
 createUberJar(
-    'beam_pipeline_common',
+    'beamPipelineCommon',
     'beam_pipeline_common',
     '')
 
-// Create beam staging task if environment is alpha or crash.
-// All other environments use formally released pipelines through CloudBuild.
+// Create beam staging task if the environment is alpha. Production, sandbox and
+// qa use formally released pipelines through CloudBuild, whereas crash and
+// alpha use the pipelines staged on alpha deployment project.
 //
 // User should install gcloud and login to GCP before invoking this tasks.
-if (environment in ['alpha', 'crash']) {
+if (environment == 'alpha') {
   def pipelines = [
-      [
-          mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
-          metaData: 'google/registry/beam/init_sql_pipeline_metadata.json'
-      ],
-      [
-          mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
-          metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
-      ],
+      initSql            :
+          [
+              mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
+              metaData : 'google/registry/beam/init_sql_pipeline_metadata.json'
+          ],
+      bulkDeleteDatastore:
+          [
+              mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
+              metaData : 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
+          ],
+      spec11             :
+          [
+              mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
+              metaData : 'google/registry/beam/spec11_pipeline_metadata.json'
+          ],
+      invoicing          :
+          [
+              mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
+              metaData : 'google/registry/beam/invoicing_pipeline_metadata.json'
+          ],
+      rde                :
+          [
+              mainClass: 'google.registry.beam.rde.RdePipeline',
+              metaData : 'google/registry/beam/rde_pipeline_metadata.json'
+          ],
   ]
-  project.tasks.create("stage_beam_pipelines") {
+  project.tasks.create("stageBeamPipelines") {
     doLast {
       pipelines.each {
-        def mainClass = it['mainClass']
-        def metaData = it['metaData']
-        def pipelineName = CaseFormat.UPPER_CAMEL.to(
-                CaseFormat.LOWER_UNDERSCORE,
-                mainClass.substring(mainClass.lastIndexOf('.') + 1))
-        def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
-        def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
-        def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath
+        if (rootProject.pipeline == ''|| rootProject.pipeline == it.key) {
+          def mainClass = it.value['mainClass']
+          def metaData = it.value['metaData']
+          def pipelineName = CaseFormat.UPPER_CAMEL.to(
+              CaseFormat.LOWER_UNDERSCORE,
+              mainClass.substring(mainClass.lastIndexOf('.') + 1))
+          def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
+          def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
+          def uberJarName = tasks.beamPipelineCommon.outputs.files.asPath
 
-        def command = "\
+          def command = "\
             gcloud dataflow flex-template build \
             gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
             --image-gcr-path ${imageName}:live \
@@ -828,10 +817,11 @@ if (environment in ['alpha', 'crash']) {
             --jar ${uberJarName} \
             --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
             --project ${gcpProject}".toString()
-        rootProject.ext.execInBash(command, '/tmp')
+          rootProject.ext.execInBash(command, '/tmp')
+        }
       }
     }
-  }.dependsOn(tasks.beam_pipeline_common)
+  }.dependsOn(tasks.beamPipelineCommon)
 }
 
 // A jar with classes and resources from main sourceSet, excluding internal
@@ -1106,6 +1096,10 @@ test {
   // TODO(weiminyu): Remove dependency on sqlIntegrationTest
 }.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
 
+// When we override tests, we also break the cleanTest command.
+cleanTest.dependsOn(cleanFragileTest, cleanOutcastTest, cleanStandardTest,
+                    cleanRegistryToolIntegrationTest, cleanSqlIntegrationTest)
+
 project.build.dependsOn devtool
 project.build.dependsOn buildToolImage
 project.build.dependsOn ':stage'

core/gradle/dependency-locks/annotationProcessor.lockfile

@@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33
 com.google.dagger:dagger-spi:2.33
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotation:2.3.4
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.errorprone:error_prone_check_api:2.3.4
 com.google.errorprone:error_prone_core:2.3.4
 com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
@@ -32,7 +32,7 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.checkerframework:dataflow:3.0.0
 org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20

core/gradle/dependency-locks/closureCompiler.lockfile

@@ -1,15 +1,4 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-args4j:args4j:2.0.26
-com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.7
-com.google.errorprone:error_prone_annotations:2.3.1
-com.google.guava:guava:25.1-jre
-com.google.j2objc:j2objc-annotations:1.1
-com.google.javascript:closure-compiler-externs:v20190301
-com.google.javascript:closure-compiler:v20190301
-com.google.jsinterop:jsinterop-annotations:1.0.0
-com.google.protobuf:protobuf-java:3.0.2
-org.checkerframework:checker-qual:2.0.0
-org.codehaus.mojo:animal-sniffer-annotations:1.14
+com.google.javascript:closure-compiler:v20210505

core/gradle/dependency-locks/compile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,105 +24,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -131,11 +143,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -147,32 +159,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -189,38 +205,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -253,15 +272,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,103 +24,114 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -129,11 +142,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -145,30 +158,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -184,37 +201,40 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -246,15 +266,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/default.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,106 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -136,11 +148,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -155,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -197,41 +213,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -267,16 +285,18 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,106 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -136,11 +148,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -155,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -197,40 +213,42 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -266,16 +284,18 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,105 +24,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -131,11 +143,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -147,32 +159,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -189,38 +205,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -253,15 +272,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,103 +24,114 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -129,11 +142,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -145,30 +158,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -184,38 +201,41 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -247,15 +267,17 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,105 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -135,11 +147,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -154,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -196,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -265,16 +283,18 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,105 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -135,11 +147,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -154,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -196,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -265,16 +283,18 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/runtime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,105 +28,115 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -135,11 +147,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -154,32 +166,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -196,41 +212,43 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 junit:junit:4.13.1
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -265,16 +283,18 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,106 +28,116 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -136,11 +148,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -155,32 +167,36 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -197,40 +213,42 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
+joda-time:joda-time:2.10.10
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy:1.10.17
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
-org.apache.commons:commons-lang3:3.5
+org.apache.commons:commons-lang3:3.8.1
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -266,16 +284,18 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/soy.lockfile

@@ -5,25 +5,25 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.7
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.3.4
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:5.0.1
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.protobuf:protobuf-java:3.13.0
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.checkerframework:checker-qual:3.5.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0

core/gradle/dependency-locks/testAnnotationProcessor.lockfile

@@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33
 com.google.dagger:dagger-spi:2.33
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotation:2.3.4
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.errorprone:error_prone_check_api:2.3.4
 com.google.errorprone:error_prone_core:2.3.4
 com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
@@ -30,7 +30,7 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.checkerframework:dataflow:3.0.0
 org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20

core/gradle/dependency-locks/testCompile.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,61 +24,63 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -84,45 +88,54 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-nio:0.123.4
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -134,11 +147,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -155,32 +168,36 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -197,30 +214,34 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+joda-time:joda-time:2.10.10
+junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -229,7 +250,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -240,7 +260,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -300,17 +320,19 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -5,11 +5,13 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -22,60 +24,62 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -83,44 +87,54 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-nio:0.123.4
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.6
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -132,11 +146,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -153,30 +167,34 @@ com.zaxxer:HikariCP:3.4.5
 commons-codec:commons-codec:1.15
 commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 it.unimi.dsi:fastutil:6.5.16
 javax.activation:activation:1.1
@@ -192,30 +210,34 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+joda-time:joda-time:2.10.10
+junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -224,7 +246,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.6.2
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -235,7 +256,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -294,17 +315,19 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testRuntime.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,61 +28,63 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -88,46 +92,55 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-nio:0.123.4
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,11 +152,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -163,33 +176,37 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -206,31 +223,35 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+joda-time:joda-time:2.10.10
+junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -239,8 +260,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,7 +270,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -313,18 +332,20 @@ org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -5,15 +5,17 @@ antlr:antlr:2.7.7
 aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
+com.101tec:zkclient:0.10
 com.beust:jcommander:1.60
 com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.4
+com.fasterxml.jackson.core:jackson-core:2.12.4
+com.fasterxml.jackson.core:jackson-databind:2.12.4
+com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.4
+com.fasterxml.jackson:jackson-bom:2.12.4
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -26,61 +28,63 @@ com.github.jnr:jnr-ffi:2.2.1
 com.github.jnr:jnr-posix:3.1.4
 com.github.jnr:jnr-unixsocket:0.38.5
 com.github.jnr:jnr-x86asm:1.0.2
-com.github.rholder:guava-retrying:2.0.0
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client-appengine:1.31.3
-com.google.api-client:google-api-client-jackson2:1.30.10
+com.google.api-client:google-api-client-jackson2:1.31.3
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.14.0
-com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.90.3
-com.google.api.grpc:grpc-google-cloud-pubsublite-v1:0.7.0
-com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:grpc-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:grpc-google-common-protos:1.18.1
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:1.5.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.105.5
-com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.105.5
-com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:1.14.0
-com.google.api.grpc:proto-google-cloud-bigtable-v2:1.16.1
-com.google.api.grpc:proto-google-cloud-datastore-v1:0.88.9
-com.google.api.grpc:proto-google-cloud-pubsub-v1:1.92.0
-com.google.api.grpc:proto-google-cloud-pubsublite-v1:0.7.0
+com.google.api-client:google-api-client:1.32.1
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-bigtable-v2:1.27.1
+com.google.api.grpc:grpc-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:grpc-google-cloud-pubsublite-v1:1.2.0
+com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:grpc-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:grpc-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:2.1.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.125.0
+com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.125.0
+com.google.api.grpc:proto-google-cloud-bigtable-admin-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-bigtable-v2:2.1.0
+com.google.api.grpc:proto-google-cloud-datastore-v1:0.91.3
+com.google.api.grpc:proto-google-cloud-firestore-v1:3.0.2
+com.google.api.grpc:proto-google-cloud-pubsub-v1:1.96.2
+com.google.api.grpc:proto-google-cloud-pubsublite-v1:1.2.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1:1.4.0
 com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
-com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
-com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-spanner-v1:6.12.1
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.5.0
+com.google.api.grpc:proto-google-iam-v1:1.1.0
+com.google.api:api-common:2.0.2
+com.google.api:gax-grpc:2.4.1
+com.google.api:gax-httpjson:0.88.0
+com.google.api:gax:2.4.1
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
-com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
-com.google.apis:google-api-services-clouddebugger:v2-rev20200501-1.30.10
+com.google.apis:google-api-services-bigquery:v2-rev20210813-1.32.1
+com.google.apis:google-api-services-clouddebugger:v2-rev20210813-1.32.1
 com.google.apis:google-api-services-cloudkms:v1-rev108-1.25.0
-com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200720-1.30.10
-com.google.apis:google-api-services-dataflow:v1b3-rev20200713-1.30.10
+com.google.apis:google-api-services-cloudresourcemanager:v1-rev20210815-1.32.1
+com.google.apis:google-api-services-dataflow:v1b3-rev20210818-1.32.1
 com.google.apis:google-api-services-dns:v2beta1-rev99-1.25.0
 com.google.apis:google-api-services-drive:v2-rev393-1.25.0
 com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0
-com.google.apis:google-api-services-healthcare:v1beta1-rev20200713-1.30.10
-com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.30.10
+com.google.apis:google-api-services-healthcare:v1-rev20210806-1.32.1
+com.google.apis:google-api-services-iamcredentials:v1-rev20201022-1.31.0
 com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
-com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
+com.google.apis:google-api-services-pubsub:v1-rev20210809-1.32.1
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -88,46 +92,55 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:1.1.0
+com.google.auth:google-auth-library-oauth2-http:1.1.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.2
 com.google.auto.value:auto-value:1.7.4
-com.google.cloud.bigdataoss:gcsio:2.1.6
-com.google.cloud.bigdataoss:util:2.1.6
-com.google.cloud.bigtable:bigtable-client-core:1.16.0
+com.google.cloud.bigdataoss:gcsio:2.2.2
+com.google.cloud.bigdataoss:util:2.2.2
+com.google.cloud.bigtable:bigtable-client-core:1.23.1
+com.google.cloud.bigtable:bigtable-metrics-api:1.23.1
 com.google.cloud.datastore:datastore-v1-proto-client:1.6.3
 com.google.cloud.sql:jdbc-socket-factory-core:1.2.1
 com.google.cloud.sql:postgres-socket-factory:1.2.1
-com.google.cloud:google-cloud-bigquery:1.122.2
-com.google.cloud:google-cloud-bigquerystorage:1.5.5
-com.google.cloud:google-cloud-bigtable:1.14.0
-com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
-com.google.cloud:google-cloud-pubsub:1.110.0
-com.google.cloud:google-cloud-pubsublite:0.7.0
+com.google.cloud:google-cloud-bigquery:2.1.2
+com.google.cloud:google-cloud-bigquerystorage:2.1.0
+com.google.cloud:google-cloud-bigtable:1.27.1
+com.google.cloud:google-cloud-core-grpc:2.1.0
+com.google.cloud:google-cloud-core-http:2.1.0
+com.google.cloud:google-cloud-core:2.1.0
+com.google.cloud:google-cloud-firestore:3.0.2
+com.google.cloud:google-cloud-nio:0.123.4
+com.google.cloud:google-cloud-pubsub:1.114.2
+com.google.cloud:google-cloud-pubsublite:1.2.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
-com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-spanner:6.12.1
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
+com.google.cloud:grpc-gcp:1.1.0
+com.google.cloud:proto-google-cloud-firestore-bundle-v1:3.0.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.8
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
-com.google.flogger:flogger-system-backend:0.5.1
-com.google.flogger:flogger:0.5.1
-com.google.flogger:google-extensions:0.5.1
+com.google.errorprone:error_prone_annotations:2.9.0
+com.google.escapevelocity:escapevelocity:0.9.1
+com.google.flatbuffers:flatbuffers-java:1.12.0
+com.google.flogger:flogger-system-backend:0.7.3
+com.google.flogger:flogger:0.7.3
+com.google.flogger:google-extensions:0.6
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.40.0
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.40.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,11 +152,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -163,33 +176,37 @@ dnsjava:dnsjava:3.3.1
 guru.nidi.com.kitfox:svgSalamander:1.1.3
 guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
-io.dropwizard.metrics:metrics-core:3.2.6
-io.github.classgraph:classgraph:4.8.102
+io.confluent:common-config:5.3.2
+io.confluent:common-utils:5.3.2
+io.confluent:kafka-avro-serializer:5.3.2
+io.confluent:kafka-schema-registry-client:5.3.2
+io.dropwizard.metrics:metrics-core:3.1.2
+io.github.classgraph:classgraph:4.8.104
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
-io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
-io.netty:netty-buffer:4.1.51.Final
-io.netty:netty-codec-http2:4.1.51.Final
-io.netty:netty-codec-http:4.1.51.Final
-io.netty:netty-codec-socks:4.1.51.Final
-io.netty:netty-codec:4.1.51.Final
-io.netty:netty-common:4.1.51.Final
-io.netty:netty-handler-proxy:4.1.51.Final
-io.netty:netty-handler:4.1.51.Final
-io.netty:netty-resolver:4.1.51.Final
+io.grpc:grpc-alts:1.40.1
+io.grpc:grpc-api:1.40.1
+io.grpc:grpc-auth:1.40.1
+io.grpc:grpc-context:1.40.1
+io.grpc:grpc-core:1.40.1
+io.grpc:grpc-grpclb:1.40.1
+io.grpc:grpc-netty-shaded:1.40.1
+io.grpc:grpc-netty:1.40.0
+io.grpc:grpc-protobuf-lite:1.40.1
+io.grpc:grpc-protobuf:1.40.1
+io.grpc:grpc-stub:1.40.1
+io.netty:netty-buffer:4.1.52.Final
+io.netty:netty-codec-http2:4.1.52.Final
+io.netty:netty-codec-http:4.1.52.Final
+io.netty:netty-codec-socks:4.1.52.Final
+io.netty:netty-codec:4.1.52.Final
+io.netty:netty-common:4.1.52.Final
+io.netty:netty-handler-proxy:4.1.52.Final
+io.netty:netty-handler:4.1.52.Final
+io.netty:netty-resolver:4.1.52.Final
 io.netty:netty-tcnative-boringssl-static:2.0.33.Final
-io.netty:netty-transport:4.1.51.Final
+io.netty:netty-transport:4.1.52.Final
 io.opencensus:opencensus-api:0.28.0
-io.opencensus:opencensus-contrib-grpc-util:0.24.0
+io.opencensus:opencensus-contrib-grpc-util:0.28.0
 io.opencensus:opencensus-contrib-http-util:0.28.0
 io.perfmark:perfmark-api:0.23.0
 it.unimi.dsi:fastutil:6.5.16
@@ -206,31 +223,35 @@ javax.transaction:transaction-api:1.1
 javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
-joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+joda-time:joda-time:2.10.10
+junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
+org.apache.arrow:arrow-format:5.0.0
+org.apache.arrow:arrow-memory-core:5.0.0
+org.apache.arrow:arrow-vector:5.0.0
 org.apache.avro:avro:1.8.2
-org.apache.beam:beam-model-fn-execution:2.28.0
-org.apache.beam:beam-model-job-management:2.28.0
-org.apache.beam:beam-model-pipeline:2.28.0
-org.apache.beam:beam-runners-core-construction-java:2.28.0
-org.apache.beam:beam-runners-core-java:2.28.0
-org.apache.beam:beam-runners-direct-java:2.28.0
-org.apache.beam:beam-runners-google-cloud-dataflow-java:2.28.0
-org.apache.beam:beam-runners-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-core:2.28.0
-org.apache.beam:beam-sdks-java-expansion-service:2.28.0
-org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.28.0
-org.apache.beam:beam-sdks-java-extensions-protobuf:2.28.0
-org.apache.beam:beam-sdks-java-fn-execution:2.28.0
-org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.28.0
-org.apache.beam:beam-vendor-bytebuddy-1_10_8:0.1
-org.apache.beam:beam-vendor-grpc-1_26_0:0.3
+org.apache.beam:beam-model-fn-execution:2.34.0
+org.apache.beam:beam-model-job-management:2.34.0
+org.apache.beam:beam-model-pipeline:2.34.0
+org.apache.beam:beam-runners-core-construction-java:2.34.0
+org.apache.beam:beam-runners-core-java:2.34.0
+org.apache.beam:beam-runners-direct-java:2.34.0
+org.apache.beam:beam-runners-google-cloud-dataflow-java:2.34.0
+org.apache.beam:beam-runners-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-core:2.34.0
+org.apache.beam:beam-sdks-java-expansion-service:2.34.0
+org.apache.beam:beam-sdks-java-extensions-arrow:2.34.0
+org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.34.0
+org.apache.beam:beam-sdks-java-extensions-protobuf:2.34.0
+org.apache.beam:beam-sdks-java-fn-execution:2.34.0
+org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.34.0
+org.apache.beam:beam-sdks-java-io-kafka:2.34.0
+org.apache.beam:beam-vendor-bytebuddy-1_11_0:0.1
+org.apache.beam:beam-vendor-grpc-1_36_0:0.2
 org.apache.beam:beam-vendor-guava-26_0-jre:0.1
-org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.28.0
 org.apache.commons:commons-compress:1.20
 org.apache.commons:commons-exec:1.3
 org.apache.commons:commons-lang3:3.8.1
@@ -239,8 +260,6 @@ org.apache.ftpserver:ftplet-api:1.0.6
 org.apache.ftpserver:ftpserver-core:1.0.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
-org.apache.logging.log4j:log4j-api:2.13.3
-org.apache.logging.log4j:log4j-core:2.13.3
 org.apache.mina:mina-core:2.0.4
 org.apache.sshd:sshd-core:2.0.0
 org.apache.sshd:sshd-scp:2.0.0
@@ -251,7 +270,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.9.1
+org.checkerframework:checker-qual:3.18.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -314,18 +333,20 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
-org.threeten:threetenbp:1.5.0
+org.springframework:spring-core:4.3.18.RELEASE
+org.springframework:spring-expression:4.3.18.RELEASE
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
-org.xerial.snappy:snappy-java:1.1.4
-org.yaml:snakeyaml:1.17
+org.xerial.snappy:snappy-java:1.1.8.4
+org.yaml:snakeyaml:1.27
 us.fatehi:schemacrawler-api:16.10.1
 us.fatehi:schemacrawler-diagram:16.10.1
 us.fatehi:schemacrawler-tools:16.10.1

core/karma.conf.js

@@ -1,75 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-process.env.CHROME_BIN = require('puppeteer').executablePath()
-
-module.exports = function(config) {
-  config.set({
-    basePath: '..',
-    browsers: ['ChromeHeadlessNoSandbox'],
-    customLaunchers: {
-      ChromeHeadlessNoSandbox: {
-        base: 'ChromeHeadless',
-        flags: ['--no-sandbox']
-      }
-    },
-    frameworks: ['jasmine', 'closure'],
-    singleRun: true,
-    autoWatch: false,
-    files: [
-      'node_modules/google-closure-library/closure/goog/base.js',
-      'core/src/test/javascript/**/*_test.js',
-      {
-        pattern: 'core/src/test/javascript/**/!(*_test).js',
-        included: false
-      },
-      {
-        pattern: 'core/src/main/javascript/**/*.js',
-        included: false
-      },
-      {
-        pattern: 'core/build/generated/sources/custom/java/main/**/*.soy.js',
-        included: false
-      },
-      {
-        pattern: 'node_modules/google-closure-library/closure/goog/deps.js',
-        included: false,
-        served: false
-      },
-      {
-        pattern: 'node_modules/google-closure-library/closure/goog/**/*.js',
-        included: false
-      },
-      {
-        pattern: 'core/build/resources/main/google/registry/ui/assets/images/*.png',
-        included: false
-      },
-      {
-        pattern: 'core/build/resources/main/google/registry/ui/assets/images/icons/svg/*.svg',
-        included: false
-      }
-    ],
-    preprocessors: {
-      'node_modules/google-closure-library/closure/goog/deps.js': ['closure', 'closure-deps'],
-      'node_modules/google-closure-library/closure/goog/base.js': ['closure'],
-      'node_modules/google-closure-library/closure/**/*.js': ['closure'],
-      'core/src/*/javascript/**/*.js': ['closure'],
-      'core/build/generated/sources/custom/java/main/**/*.soy.js': ['closure'],
-    },
-    proxies: {
-      "/assets/": "/base/core/build/resources/main/google/registry/ui/assets/"
-    }
-  });
-};
-

core/src/main/java/google/registry/backup/BackupModule.java

@@ -18,8 +18,10 @@ import static com.google.appengine.api.ThreadManager.currentRequestThreadFactory
 import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
+import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
+import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static java.util.concurrent.Executors.newFixedThreadPool;
@@ -32,6 +34,9 @@ import google.registry.cron.CommitLogFanoutAction;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
 import java.lang.annotation.Documented;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import javax.inject.Qualifier;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.DateTime;
@@ -75,6 +80,12 @@ public final class BackupModule {
     return extractRequiredDatetimeParameter(req, UPPER_CHECKPOINT_TIME_PARAM);
   }
 
+  @Provides
+  @Parameter(BUCKET_OVERRIDE_PARAM)
+  static Optional<String> provideBucketOverride(HttpServletRequest req) {
+    return extractOptionalParameter(req, BUCKET_OVERRIDE_PARAM);
+  }
+
   @Provides
   @Parameter(FROM_TIME_PARAM)
   static DateTime provideFromTime(HttpServletRequest req) {
@@ -92,4 +103,9 @@ public final class BackupModule {
   static ListeningExecutorService provideListeningExecutorService() {
     return listeningDecorator(newFixedThreadPool(NUM_THREADS, currentRequestThreadFactory()));
   }
+
+  @Provides
+  static ScheduledExecutorService provideScheduledExecutorService() {
+    return Executors.newSingleThreadScheduledExecutor();
+  }
 }

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -14,13 +14,14 @@
 
 package google.registry.backup;
 
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 
 import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.ImmutableList;
 import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,6 +29,7 @@ import java.io.OutputStream;
 import java.util.Iterator;
 
 /** Utilities for working with backups. */
+@DeleteAfterMigration
 public class BackupUtils {
 
   /** Keys for user metadata fields on commit log files in GCS. */
@@ -41,11 +43,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(ofy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnlyWithoutBackup().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**
@@ -56,19 +59,25 @@ public class BackupUtils {
    *
    * <p>The iterator reads from the stream on demand, and as such will fail if the stream is closed.
    */
-  public static Iterator<ImmutableObject> createDeserializingIterator(final InputStream input) {
+  public static Iterator<ImmutableObject> createDeserializingIterator(
+      final InputStream input, boolean withAppIdOverride) {
     return new AbstractIterator<ImmutableObject>() {
       @Override
       protected ImmutableObject computeNext() {
         EntityProto proto = new EntityProto();
-        if (proto.parseDelimitedFrom(input)) {  // False means end of stream; other errors throw.
-          return ofy().load().fromEntity(EntityTranslator.createFromPb(proto));
+        if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw.
+          if (withAppIdOverride) {
+            proto = EntityImports.fixEntity(proto);
+          }
+          return auditedOfy().load().fromEntity(EntityTranslator.createFromPb(proto));
         }
         return endOfData();
-      }};
+      }
+    };
   }
 
   public static ImmutableList<ImmutableObject> deserializeEntities(byte[] bytes) {
-    return ImmutableList.copyOf(createDeserializingIterator(new ByteArrayInputStream(bytes)));
+    return ImmutableList.copyOf(
+        createDeserializingIterator(new ByteArrayInputStream(bytes), false));
   }
 }

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -14,21 +14,22 @@
 
 package google.registry.backup;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -48,6 +49,7 @@ import org.joda.time.DateTime;
     method = Action.Method.GET,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public final class CommitLogCheckpointAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -56,7 +58,8 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Inject Clock clock;
   @Inject CommitLogCheckpointStrategy strategy;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject CommitLogCheckpointAction() {}
 
   @Override
@@ -64,7 +67,7 @@ public final class CommitLogCheckpointAction implements Runnable {
     final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
     logger.atInfo().log(
         "Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
-    tm()
+    ofyTm()
         .transact(
             () -> {
               DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
@@ -73,17 +76,21 @@ public final class CommitLogCheckpointAction implements Runnable {
                     "Newer checkpoint already written at time: %s", lastWrittenTime);
                 return;
               }
-              ofy()
-                  .saveWithoutBackup()
+              auditedOfy()
+                  .saveIgnoringReadOnlyWithoutBackup()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.
-              taskQueueUtils.enqueue(
-                  getQueue(QUEUE_NAME),
-                  withUrl(ExportCommitLogDiffAction.PATH)
-                      .param(LOWER_CHECKPOINT_TIME_PARAM, lastWrittenTime.toString())
-                      .param(
-                          UPPER_CHECKPOINT_TIME_PARAM, checkpoint.getCheckpointTime().toString()));
+              cloudTasksUtils.enqueue(
+                  QUEUE_NAME,
+                  CloudTasksUtils.createPostTask(
+                      ExportCommitLogDiffAction.PATH,
+                      Service.BACKEND.toString(),
+                      ImmutableMultimap.of(
+                          LOWER_CHECKPOINT_TIME_PARAM,
+                          lastWrittenTime.toString(),
+                          UPPER_CHECKPOINT_TIME_PARAM,
+                          checkpoint.getCheckpointTime().toString())));
             });
   }
 }

core/src/main/java/google/registry/backup/CommitLogCheckpointStrategy.java

@@ -23,6 +23,7 @@ import static google.registry.util.DateTimeUtils.earliestOf;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -36,14 +37,14 @@ import org.joda.time.DateTime;
 /**
  * Implementation of the procedure for determining point-in-time consistent commit log checkpoint.
  *
- * <p>This algorithm examines the recently written commit log data and uses a dual-read approach
- * to determine a point-in-time consistent set of checkpoint times for the commit log buckets.  By
+ * <p>This algorithm examines the recently written commit log data and uses a dual-read approach to
+ * determine a point-in-time consistent set of checkpoint times for the commit log buckets. By
  * "consistent" we mean, generally speaking, that if the Datastore were restored by replaying all
  * the commit logs up to the checkpoint times of the buckets, the result would be transactionally
  * correct; there must be no "holes" where restored state depends on non-restored state.
  *
  * <p>The consistency guarantee really has two parts, only one of which is provided by this
- * algorithm.  The procedure below guarantees only that if the resulting checkpoint includes any
+ * algorithm. The procedure below guarantees only that if the resulting checkpoint includes any
  * given commit log, it will also include all the commit logs that were both 1) actually written
  * before that commit log "in real life", and 2) have an earlier timestamp than that commit log.
  * (These criteria do not necessarily imply each other, due to the lack of a global shared clock.)
@@ -51,8 +52,8 @@ import org.joda.time.DateTime;
  * that depends on state from a previous transaction does indeed have a later timestamp.
  *
  * <h2>Procedure description</h2>
- * <pre>
- * {@code
+ *
+ * <pre>{@code
  * ComputeCheckpoint() -> returns a set consisting of a timestamp c(b_i) for every bucket b_i
  *
  * 1) read off the latest commit timestamp t(b_i) for every bucket b_i
@@ -63,35 +64,29 @@ import org.joda.time.DateTime;
  *   a) if S is empty, let T* = +∞ (or the "end of time")
  *   b) else, let T* = T - Δ, for T = min(S) and some small Δ > 0
  * 4) return the set given by: min(t(b_i), T*) for all b_i
- * }
- * </pre>
+ * }</pre>
  *
  * <h2>Correctness proof of algorithm</h2>
  *
- * <p>{@literal
- * As described above, the algorithm is correct as long as it can ensure the following: given a
- * commit log X written at time t(X) to bucket b_x, and another commit log Y that was written "in
- * real life" before X and for which t(Y) < t(X), then if X is included in the checkpoint, so is Y;
- * that is, t(X) <= c(b_x) implies t(Y) <= c(b_y).
- * }
+ * <p>{@literal As described above, the algorithm is correct as long as it can ensure the following:
+ * given a commit log X written at time t(X) to bucket b_x, and another commit log Y that was
+ * written "in real life" before X and for which t(Y) < t(X), then if X is included in the
+ * checkpoint, so is Y; that is, t(X) <= c(b_x) implies t(Y) <= c(b_y). }
  *
- * <p>{@literal
- * To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e. every commit
- * log included in the checkpoint must have been seen in the first pass.  Hence if X was included,
- * then X must have been written by the time we started the second pass.  But since Y was written
- * "in real life" prior to X, we must have seen Y by the second pass too.
- * }
+ * <p>{@literal To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e.
+ * every commit log included in the checkpoint must have been seen in the first pass. Hence if X was
+ * included, then X must have been written by the time we started the second pass. But since Y was
+ * written "in real life" prior to X, we must have seen Y by the second pass too. }
  *
- * <p>{@literal
- * Now assume towards a contradiction that X is indeed included but Y is not, i.e. that we have
- * t(X) <= c(b_x) but t(Y) > c(b_y).  If Y was seen in the first pass, i.e. t(Y) <= t(b_y), then by
- * our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y).  By the definition of
- * c(b_y) it must then equal T*, so we have T* < t(Y).  However, this is a contradiction since
- * t(Y) < t(X) and t(X) <= c(b_x) <= T*.  If instead Y was seen in the second pass but not the
- * first, t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by
- * definition, we again reach the contradiction T* < t(Y).
- * }
+ * <p>{@literal Now assume towards a contradiction that X is indeed included but Y is not, i.e. that
+ * we have t(X) <= c(b_x) but t(Y) > c(b_y). If Y was seen in the first pass, i.e. t(Y) <= t(b_y),
+ * then by our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y). By the definition
+ * of c(b_y) it must then equal T*, so we have T* < t(Y). However, this is a contradiction since
+ * t(Y) < t(X) and t(X) <= c(b_x) <= T*. If instead Y was seen in the second pass but not the first,
+ * t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by definition,
+ * we again reach the contradiction T* < t(Y). }
  */
+@DeleteAfterMigration
 class CommitLogCheckpointStrategy {
 
   @Inject Ofy ofy;

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -20,6 +20,7 @@ import static google.registry.backup.BackupUtils.createDeserializingIterator;
 
 import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
@@ -38,6 +39,7 @@ import java.util.Iterator;
  * <p>This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial
  * population of the Cloud SQL database.
  */
+@DeleteAfterMigration
 public final class CommitLogImports {
 
   private CommitLogImports() {}
@@ -55,11 +57,10 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+  static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
       InputStream inputStream) {
-    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
-        InputStream input = new BufferedInputStream(inputStream)) {
-      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+    try (InputStream input = new BufferedInputStream(inputStream)) {
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
       checkState(commitLogs.hasNext());
       checkState(commitLogs.next() instanceof CommitLogCheckpoint);
 
@@ -105,7 +106,7 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+  static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
     return loadEntitiesByTransaction(inputStream).stream()
         .flatMap(ImmutableList::stream)
         .collect(toImmutableList());

core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java

@@ -17,8 +17,8 @@ package google.registry.backup;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static java.lang.Boolean.FALSE;
 import static java.lang.Boolean.TRUE;
 
@@ -35,6 +35,7 @@ import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.CommitLogManifestInput;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
@@ -66,6 +67,9 @@ import org.joda.time.Duration;
     service = Action.Service.BACKEND,
     path = "/_dr/task/deleteOldCommitLogs",
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+// No longer needed in SQL. Subject to future removal.
+@Deprecated
+@DeleteAfterMigration
 public final class DeleteOldCommitLogsAction implements Runnable {
 
   private static final int NUM_MAP_SHARDS = 20;
@@ -75,15 +79,23 @@ public final class DeleteOldCommitLogsAction implements Runnable {
   @Inject MapreduceRunner mrRunner;
   @Inject Response response;
   @Inject Clock clock;
-  @Inject @Config("commitLogDatastoreRetention") Duration maxAge;
-  @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
-  @Inject DeleteOldCommitLogsAction() {}
+
+  @Inject
+  @Config("commitLogDatastoreRetention")
+  Duration maxAge;
+
+  @Inject
+  @Parameter(PARAM_DRY_RUN)
+  boolean isDryRun;
+
+  @Inject
+  DeleteOldCommitLogsAction() {}
 
   @Override
   public void run() {
     DateTime deletionThreshold = clock.nowUtc().minus(maxAge);
     logger.atInfo().log(
-        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s",
+        "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s.",
         deletionThreshold);
 
     mrRunner
@@ -138,12 +150,12 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       // If it isn't a Key<CommitLogManifest> then it should be an EppResource, which we need to
       // load to emit the revisions.
       //
-      Object object = ofy().load().key(key).now();
+      Object object = auditedOfy().load().key(key).now();
       checkNotNull(object, "Received a key to a missing object. key: %s", key);
       checkState(
           object instanceof EppResource,
           "Received a key to an object that isn't EppResource nor CommitLogManifest."
-          + " Key: %s object type: %s",
+              + " Key: %s object type: %s",
           key,
           object.getClass().getName());
 
@@ -198,7 +210,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       getContext().incrementCounter("EPP resources missing pre-threshold revision (SEE LOGS)");
       logger.atSevere().log(
           "EPP resource missing old enough revision: "
-              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s",
+              + "%s (created on %s) has %d revisions between %s and %s, while threshold is %s.",
           Key.create(eppResource),
           eppResource.getCreationTime(),
           eppResource.getRevisions().size(),
@@ -224,8 +236,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
    * OK to delete this manifestKey. If even one source returns "false" (meaning "it's not OK to
    * delete this manifest") then it won't be deleted.
    */
-  static class DeleteOldCommitLogsReducer
-      extends Reducer<Key<CommitLogManifest>, Boolean, Void> {
+  static class DeleteOldCommitLogsReducer extends Reducer<Key<CommitLogManifest>, Boolean, Void> {
 
     private static final long serialVersionUID = -4918760187627937268L;
 
@@ -241,12 +252,12 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       }
 
       public abstract Status status();
+
       public abstract int numDeleted();
 
       static DeletionResult create(Status status, int numDeleted) {
-        return
-            new AutoValue_DeleteOldCommitLogsAction_DeleteOldCommitLogsReducer_DeletionResult(
-                status, numDeleted);
+        return new AutoValue_DeleteOldCommitLogsAction_DeleteOldCommitLogsReducer_DeletionResult(
+            status, numDeleted);
       }
     }
 
@@ -257,8 +268,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
 
     @Override
     public void reduce(
-        final Key<CommitLogManifest> manifestKey,
-        ReducerInput<Boolean> canDeleteVerdicts) {
+        final Key<CommitLogManifest> manifestKey, ReducerInput<Boolean> canDeleteVerdicts) {
       ImmutableMultiset<Boolean> canDeleteMultiset = ImmutableMultiset.copyOf(canDeleteVerdicts);
       if (canDeleteMultiset.count(TRUE) > 1) {
         getContext().incrementCounter("commit log manifests incorrectly mapped multiple times");
@@ -267,47 +277,55 @@ public final class DeleteOldCommitLogsAction implements Runnable {
         getContext().incrementCounter("commit log manifests referenced multiple times");
       }
       if (canDeleteMultiset.contains(FALSE)) {
-        getContext().incrementCounter(
-            canDeleteMultiset.contains(TRUE)
-            ? "old commit log manifests still referenced"
-            : "new (or nonexistent) commit log manifests referenced");
-        getContext().incrementCounter(
-            "EPP resource revisions handled",
-            canDeleteMultiset.count(FALSE));
+        getContext()
+            .incrementCounter(
+                canDeleteMultiset.contains(TRUE)
+                    ? "old commit log manifests still referenced"
+                    : "new (or nonexistent) commit log manifests referenced");
+        getContext()
+            .incrementCounter("EPP resource revisions handled", canDeleteMultiset.count(FALSE));
         return;
       }
 
-      DeletionResult deletionResult = tm().transactNew(() -> {
-        CommitLogManifest manifest = ofy().load().key(manifestKey).now();
-        // It is possible that the same manifestKey was run twice, if a shard had to be restarted
-        // or some weird failure. If this happens, we want to exit immediately.
-        // Note that this can never happen in dryRun.
-        if (manifest == null) {
-          return DeletionResult.create(DeletionResult.Status.ALREADY_DELETED, 0);
-        }
-        // Doing a sanity check on the date. This is the only place we use the CommitLogManifest,
-        // so maybe removing this test will improve performance. However, unless it's proven that
-        // the performance boost is significant (and we've tested this enough to be sure it never
-        // happens)- the safty of "let's not delete stuff we need from prod" is more important.
-        if (manifest.getCommitTime().isAfter(deletionThreshold)) {
-          return DeletionResult.create(DeletionResult.Status.AFTER_THRESHOLD, 0);
-        }
-        Iterable<Key<CommitLogMutation>> commitLogMutationKeys = ofy().load()
-            .type(CommitLogMutation.class)
-            .ancestor(manifestKey)
-            .keys()
-            .iterable();
-        ImmutableList<Key<?>> keysToDelete = ImmutableList.<Key<?>>builder()
-            .addAll(commitLogMutationKeys)
-            .add(manifestKey)
-            .build();
-        // Normally in a dry run we would log the entities that would be deleted, but those can
-        // number in the millions so we skip the logging.
-        if (!isDryRun) {
-          ofy().deleteWithoutBackup().keys(keysToDelete);
-        }
-        return DeletionResult.create(DeletionResult.Status.SUCCESS, keysToDelete.size());
-      });
+      DeletionResult deletionResult =
+          ofyTm()
+              .transactNew(
+                  () -> {
+                    CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
+                    // It is possible that the same manifestKey was run twice, if a shard had to be
+                    // restarted or some weird failure. If this happens, we want to exit
+                    // immediately. Note that this can never happen in dryRun.
+                    if (manifest == null) {
+                      return DeletionResult.create(DeletionResult.Status.ALREADY_DELETED, 0);
+                    }
+                    // Doing a sanity check on the date. This is the only place we use the
+                    // CommitLogManifest, so maybe removing this test will improve performance.
+                    // However, unless it's proven that the performance boost is significant (and
+                    // we've tested this enough to be sure it never happens)- the safety of "let's
+                    // not delete stuff we need from prod" is more important.
+                    if (manifest.getCommitTime().isAfter(deletionThreshold)) {
+                      return DeletionResult.create(DeletionResult.Status.AFTER_THRESHOLD, 0);
+                    }
+                    Iterable<Key<CommitLogMutation>> commitLogMutationKeys =
+                        auditedOfy()
+                            .load()
+                            .type(CommitLogMutation.class)
+                            .ancestor(manifestKey)
+                            .keys()
+                            .iterable();
+                    ImmutableList<Key<?>> keysToDelete =
+                        ImmutableList.<Key<?>>builder()
+                            .addAll(commitLogMutationKeys)
+                            .add(manifestKey)
+                            .build();
+                    // Normally in a dry run we would log the entities that would be deleted, but
+                    // those can number in the millions so we skip the logging.
+                    if (!isDryRun) {
+                      auditedOfy().deleteWithoutBackup().keys(keysToDelete);
+                    }
+                    return DeletionResult.create(
+                        DeletionResult.Status.SUCCESS, keysToDelete.size());
+                  });
 
       switch (deletionResult.status()) {
         case SUCCESS:

core/src/main/java/google/registry/backup/EntityImports.java

@@ -0,0 +1,117 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.apphosting.api.ApiProxy;
+import com.google.storage.onestore.v3.OnestoreEntity;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import com.google.storage.onestore.v3.OnestoreEntity.Path;
+import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
+import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
+import google.registry.model.annotations.DeleteAfterMigration;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+/** Utilities for handling imported Datastore entities. */
+@DeleteAfterMigration
+public class EntityImports {
+
+  /**
+   * Transitively sets the {@code appId} of all keys in a foreign entity to that of the current
+   * system.
+   */
+  public static EntityProto fixEntity(EntityProto entityProto) {
+    String currentAappId = ApiProxy.getCurrentEnvironment().getAppId();
+    if (Objects.equals(currentAappId, entityProto.getKey().getApp())) {
+      return entityProto;
+    }
+    return fixEntity(entityProto, currentAappId);
+  }
+
+  private static EntityProto fixEntity(EntityProto entityProto, String appId) {
+    if (entityProto.hasKey()) {
+      fixKey(entityProto, appId);
+    }
+
+    for (OnestoreEntity.Property property : entityProto.mutablePropertys()) {
+      fixProperty(property, appId);
+    }
+
+    for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
+      fixProperty(property, appId);
+    }
+
+    // CommitLogMutation embeds an entity as bytes, which needs additional fixes.
+    if (isCommitLogMutation(entityProto)) {
+      fixMutationEntityProtoBytes(entityProto, appId);
+    }
+    return entityProto;
+  }
+
+  private static boolean isCommitLogMutation(EntityProto entityProto) {
+    if (!entityProto.hasKey()) {
+      return false;
+    }
+    Path path = entityProto.getKey().getPath();
+    if (path.elementSize() == 0) {
+      return false;
+    }
+    return Objects.equals(
+        path.getElement(path.elementSize() - 1).getType(StandardCharsets.UTF_8),
+        "CommitLogMutation");
+  }
+
+  private static void fixMutationEntityProtoBytes(EntityProto entityProto, String appId) {
+    for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
+      if (Objects.equals(property.getName(), "entityProtoBytes")) {
+        OnestoreEntity.PropertyValue value = property.getValue();
+        EntityProto fixedProto =
+            fixEntity(bytesToEntityProto(value.getStringValueAsBytes()), appId);
+        value.setStringValueAsBytes(fixedProto.toByteArray());
+        return;
+      }
+    }
+  }
+
+  private static void fixKey(EntityProto entityProto, String appId) {
+    entityProto.getMutableKey().setApp(appId);
+  }
+
+  private static void fixKey(ReferenceValue referenceValue, String appId) {
+    referenceValue.setApp(appId);
+  }
+
+  private static void fixProperty(OnestoreEntity.Property property, String appId) {
+    OnestoreEntity.PropertyValue value = property.getMutableValue();
+    if (value.hasReferenceValue()) {
+      fixKey(value.getMutableReferenceValue(), appId);
+      return;
+    }
+    if (property.getMeaningEnum().equals(Meaning.ENTITY_PROTO)) {
+      EntityProto embeddedProto = bytesToEntityProto(value.getStringValueAsBytes());
+      fixEntity(embeddedProto, appId);
+      value.setStringValueAsBytes(embeddedProto.toByteArray());
+    }
+  }
+
+  private static EntityProto bytesToEntityProto(byte[] bytes) {
+    EntityProto entityProto = new EntityProto();
+    boolean isParsed = entityProto.parseFrom(bytes);
+    if (!isParsed) {
+      throw new IllegalStateException("Failed to parse raw bytes as EntityProto.");
+    }
+    return entityProto;
+  }
+}

core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java

@@ -25,22 +25,22 @@ import static google.registry.backup.BackupUtils.GcsMetadataKeys.NUM_TRANSACTION
 import static google.registry.backup.BackupUtils.GcsMetadataKeys.UPPER_BOUND_CHECKPOINT;
 import static google.registry.backup.BackupUtils.serializeEntity;
 import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
-import static java.nio.channels.Channels.newOutputStream;
 import static java.util.Comparator.comparingLong;
 
-import com.google.appengine.tools.cloudstorage.GcsFileOptions;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobId;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -64,6 +64,7 @@ import org.joda.time.DateTime;
     method = Action.Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public final class ExportCommitLogDiffAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -74,7 +75,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
 
   public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
 
-  @Inject GcsService gcsService;
+  @Inject GcsUtils gcsUtils;
+
   @Inject @Config("commitLogGcsBucket") String gcsBucket;
   @Inject @Config("commitLogDiffExportBatchSize") int batchSize;
   @Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
@@ -89,23 +91,26 @@ public final class ExportCommitLogDiffAction implements Runnable {
     checkArgument(lowerCheckpointTime.isBefore(upperCheckpointTime));
     // Load the boundary checkpoints - lower is exclusive and may not exist (on the first export,
     // when lowerCheckpointTime is START_OF_TIME), whereas the upper is inclusive and must exist.
-    CommitLogCheckpoint lowerCheckpoint = lowerCheckpointTime.isAfter(START_OF_TIME)
-        ? verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
-        : null;
+    CommitLogCheckpoint lowerCheckpoint =
+        lowerCheckpointTime.isAfter(START_OF_TIME)
+            ? verifyNotNull(
+                auditedOfy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
+            : null;
     CommitLogCheckpoint upperCheckpoint =
-        verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());
+        verifyNotNull(
+            auditedOfy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());
 
     // Load the keys of all the manifests to include in this diff.
     List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
-    logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
+    logger.atInfo().log("Found %d manifests to export.", sortedKeys.size());
     // Open an output channel to GCS, wrapped in a stream for convenience.
-    try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
-        new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
-        new GcsFileOptions.Builder()
-            .addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
-            .addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
-            .addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
-            .build()))) {
+    try (OutputStream gcsStream =
+        gcsUtils.openOutputStream(
+            BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
+            ImmutableMap.of(
+                LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
+                UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
+                NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
       // Export the upper checkpoint itself.
       serializeEntity(upperCheckpoint, gcsStream);
       // If there are no manifests to export, stop early, now that we've written out the file with
@@ -117,26 +122,26 @@ public final class ExportCommitLogDiffAction implements Runnable {
       // asynchronously load the entities for the next one.
       List<List<Key<CommitLogManifest>>> keyChunks = partition(sortedKeys, batchSize);
       // Objectify's map return type is asynchronous. Calling .values() will block until it loads.
-      Map<?, CommitLogManifest> nextChunkToExport = ofy().load().keys(keyChunks.get(0));
+      Map<?, CommitLogManifest> nextChunkToExport = auditedOfy().load().keys(keyChunks.get(0));
       for (int i = 0; i < keyChunks.size(); i++) {
         // Force the async load to finish.
         Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
-        logger.atInfo().log("Loaded %d manifests", chunkValues.size());
+        logger.atInfo().log("Loaded %d manifests.", chunkValues.size());
         // Since there is no hard bound on how much data this might be, take care not to let the
         // Objectify session cache fill up and potentially run out of memory. This is the only safe
         // point to do this since at this point there is no async load in progress.
-        ofy().clearSessionCache();
+        auditedOfy().clearSessionCache();
         // Kick off the next async load, which can happen in parallel to the current GCS export.
         if (i + 1 < keyChunks.size()) {
-          nextChunkToExport = ofy().load().keys(keyChunks.get(i + 1));
+          nextChunkToExport = auditedOfy().load().keys(keyChunks.get(i + 1));
         }
         exportChunk(gcsStream, chunkValues);
-        logger.atInfo().log("Exported %d manifests", chunkValues.size());
+        logger.atInfo().log("Exported %d manifests.", chunkValues.size());
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    logger.atInfo().log("Exported %d manifests in total", sortedKeys.size());
+    logger.atInfo().log("Exported %d total manifests.", sortedKeys.size());
   }
 
   /**
@@ -192,7 +197,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
       return ImmutableSet.of();
     }
     Key<CommitLogBucket> bucketKey = getBucketKey(bucketNum);
-    return ofy().load()
+    return auditedOfy()
+        .load()
         .type(CommitLogManifest.class)
         .ancestor(bucketKey)
         .filterKey(">=", CommitLogManifest.createKey(bucketKey, lowerBound))
@@ -208,7 +214,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
         new ImmutableList.Builder<>();
     for (CommitLogManifest manifest : chunk) {
       entities.add(ImmutableList.of(manifest));
-      entities.add(ofy().load().type(CommitLogMutation.class).ancestor(manifest));
+      entities.add(auditedOfy().load().type(CommitLogMutation.class).ancestor(manifest));
     }
     for (ImmutableObject entity : concat(entities.build())) {
       serializeEntity(entity, gcsStream);

core/src/main/java/google/registry/backup/GcsDiffFileLister.java

@@ -15,79 +15,100 @@
 package google.registry.backup;
 
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
 import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;
 
-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
-import com.google.appengine.tools.cloudstorage.ListItem;
-import com.google.appengine.tools.cloudstorage.ListOptions;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.UncheckedExecutionException;
 import google.registry.backup.BackupModule.Backups;
-import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.annotations.DeleteAfterMigration;
 import java.io.IOException;
-import java.util.Iterator;
+import java.time.Duration;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.ScheduledExecutorService;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
+import javax.inject.Provider;
 import org.joda.time.DateTime;
 
 /** Utility class to list commit logs diff files stored on GCS. */
+@DeleteAfterMigration
 class GcsDiffFileLister {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  @Inject GcsService gcsService;
-  @Inject @Config("commitLogGcsBucket") String gcsBucket;
-  @Inject @Backups ListeningExecutorService executor;
-  @Inject GcsDiffFileLister() {}
+  /** Timeout for retrieving per-file information from GCS. */
+  private static final Duration FILE_INFO_TIMEOUT_DURATION = Duration.ofMinutes(1);
+
+  @Inject GcsUtils gcsUtils;
+
+  @Inject @Backups Provider<ListeningExecutorService> executorProvider;
+  @Inject ScheduledExecutorService scheduledExecutorService;
+
+  @Inject
+  GcsDiffFileLister() {}
 
   /**
    * Traverses the sequence of diff files backwards from checkpointTime and inserts the file
-   * metadata into "sequence".  Returns true if a complete sequence was discovered, false if one or
+   * metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
    * more files are missing.
+   *
+   * @throws UncheckedExecutionException wrapping a {@link java.util.concurrent.TimeoutException} if
+   *     the GCS call fails to finish within one minute, or wrapping any other exception if
+   *     something else goes wrong.
    */
   private boolean constructDiffSequence(
-      Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
+      String gcsBucket,
+      Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
       DateTime fromTime,
       DateTime lastTime,
-      TreeMap<DateTime, GcsFileMetadata> sequence) {
+      TreeMap<DateTime, BlobInfo> sequence) {
     DateTime checkpointTime = lastTime;
     while (isBeforeOrAt(fromTime, checkpointTime)) {
-      GcsFileMetadata metadata;
-      if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
-        metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
+      BlobInfo blobInfo;
+      if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
+        blobInfo =
+            Futures.getUnchecked(
+                Futures.withTimeout(
+                    upperBoundTimesToBlobInfo.get(checkpointTime),
+                    FILE_INFO_TIMEOUT_DURATION,
+                    scheduledExecutorService));
       } else {
         String filename = DIFF_FILE_PREFIX + checkpointTime;
         logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
-        metadata = getMetadata(filename);
+        blobInfo = getBlobInfo(gcsBucket, filename);
 
         // If we hit a gap, quit.
-        if (metadata == null) {
-          logger.atInfo().log(
+        if (blobInfo == null) {
+          logger.atWarning().log(
               "Gap discovered in sequence terminating at %s, missing file: %s",
               sequence.lastKey(), filename);
-          logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+          logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
           return false;
         }
       }
-      sequence.put(checkpointTime, metadata);
-      checkpointTime = getLowerBoundTime(metadata);
+      sequence.put(checkpointTime, blobInfo);
+      checkpointTime = getLowerBoundTime(blobInfo);
     }
-    logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
+    logger.atInfo().log("Found sequence from %s to %s.", checkpointTime, lastTime);
     return true;
   }
 
-  ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
+  ImmutableList<BlobInfo> listDiffFiles(
+      String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
     logger.atInfo().log("Requested restore from time: %s", fromTime);
     if (toTime != null) {
       logger.atInfo().log("  Until time: %s", toTime);
@@ -95,70 +116,82 @@ class GcsDiffFileLister {
     // List all of the diff files on GCS and build a map from each file's upper checkpoint time
     // (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
     // an upper checkpoint time > fromTime.
-    TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
-        = new TreeMap<>();
-    Iterator<ListItem> listItems;
+    TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
+    String commitLogDiffPrefix = getCommitLogDiffPrefix(fromTime, toTime);
+    ImmutableList<String> filenames;
     try {
-      // TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
-      listItems = gcsService.list(
-          gcsBucket,
-          new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
+      filenames =
+          gcsUtils.listFolderObjects(gcsBucket, commitLogDiffPrefix).stream()
+              .map(s -> commitLogDiffPrefix + s)
+              .collect(toImmutableList());
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
     DateTime lastUpperBoundTime = START_OF_TIME;
-    while (listItems.hasNext()) {
-      final String filename = listItems.next().getName();
-      DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
-      if (isInRange(upperBoundTime, fromTime, toTime)) {
-        upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
-        lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
-      }
-    }
-    if (upperBoundTimesToMetadata.isEmpty()) {
-      logger.atInfo().log("No files found");
-      return ImmutableList.of();
-    }
 
-    // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
-    // last file that we found) and finding its previous file until we either run out of files or
-    // get to one that precedes "fromTime".
-    //
-    // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
-    // metadata of a file is sufficient to identify the preceding file, so if we start from the
-    // last file and work backwards we can verify that we have no holes in our chain (although we
-    // may be missing files at the end).
-    TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
-    logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
-    boolean inconsistentFileSet = !constructDiffSequence(
-        upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
-
-    // Verify that all of the elements in the original set are represented in the sequence.  If we
-    // find anything that's not represented, construct a sequence for it.
-    boolean checkForMoreExtraDiffs = true;  // Always loop at least once.
-    while (checkForMoreExtraDiffs) {
-      checkForMoreExtraDiffs = false;
-      for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
-        if (!isInRange(key, fromTime, toTime)) {
-          break;
-        }
-        if (!sequence.containsKey(key)) {
-          constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
-          checkForMoreExtraDiffs = true;
-          inconsistentFileSet = true;
-          break;
+    TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
+    ListeningExecutorService executor = executorProvider.get();
+    try {
+      for (String filename : filenames) {
+        String strippedFilename = filename.replaceFirst(DIFF_FILE_PREFIX, "");
+        DateTime upperBoundTime = DateTime.parse(strippedFilename);
+        if (isInRange(upperBoundTime, fromTime, toTime)) {
+          upperBoundTimesToBlobInfo.put(
+              upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
+          lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
         }
       }
-    }
+      if (upperBoundTimesToBlobInfo.isEmpty()) {
+        logger.atInfo().log("No files found.");
+        return ImmutableList.of();
+      }
 
-    checkState(
-        !inconsistentFileSet,
-        "Unable to compute commit diff history, there are either gaps or forks in the history "
-        + "file set.  Check log for details.");
+      // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e.
+      // the last file that we found) and finding its previous file until we either run out of files
+      // or get to one that precedes "fromTime".
+      //
+      // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
+      // metadata of a file is sufficient to identify the preceding file, so if we start from the
+      // last file and work backwards we can verify that we have no holes in our chain (although we
+      // may be missing files at the end).
+      logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
+      boolean inconsistentFileSet =
+          !constructDiffSequence(
+              gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
+
+      // Verify that all of the elements in the original set are represented in the sequence.  If we
+      // find anything that's not represented, construct a sequence for it.
+      boolean checkForMoreExtraDiffs = true; // Always loop at least once.
+      while (checkForMoreExtraDiffs) {
+        checkForMoreExtraDiffs = false;
+        for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
+          if (!isInRange(key, fromTime, toTime)) {
+            break;
+          }
+          if (!sequence.containsKey(key)) {
+            // Recalculate the sequence for purely informational purposes.
+            logger.atWarning().log(
+                "Fork found in commit log history.  The following sequence "
+                    + "is disconnected from the sequence of the final commit:");
+            constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
+            checkForMoreExtraDiffs = true;
+            inconsistentFileSet = true;
+            break;
+          }
+        }
+      }
+
+      checkState(
+          !inconsistentFileSet,
+          "Unable to compute commit diff history, there are either gaps or forks in the history "
+              + "file set.  Check log for details.");
+    } finally {
+      executor.shutdown();
+    }
 
     logger.atInfo().log(
         "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
-    logger.atInfo().log("Found %d files to restore", sequence.size());
+    logger.atInfo().log("Found %d files to restore.", sequence.size());
     return ImmutableList.copyOf(sequence.values());
   }
 
@@ -171,15 +204,43 @@ class GcsDiffFileLister {
     return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
   }
 
-  private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
-    return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
+  private DateTime getLowerBoundTime(BlobInfo blobInfo) {
+    return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
   }
 
-  private GcsFileMetadata getMetadata(String filename) {
-    try {
-      return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
+  private BlobInfo getBlobInfo(String gcsBucket, String filename) {
+    return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
+  }
+
+  /**
+   * Returns a prefix guaranteed to cover all commit log diff files in the given range.
+   *
+   * <p>The listObjects call can be fairly slow if we search over many thousands or tens of
+   * thousands of files, so we restrict the search space. The commit logs have a file format of
+   * "commit_diff_until_2021-05-11T06:48:00.070Z" so we can often filter down as far as the hour.
+   *
+   * <p>Here, we get the longest prefix possible based on which fields (year, month, day, hour) the
+   * times in question have in common.
+   */
+  @VisibleForTesting
+  static String getCommitLogDiffPrefix(DateTime from, @Nullable DateTime to) {
+    StringBuilder result = new StringBuilder(DIFF_FILE_PREFIX);
+    if (to == null || from.getYear() != to.getYear()) {
+      return result.toString();
     }
+    result.append(from.getYear()).append('-');
+    if (from.getMonthOfYear() != to.getMonthOfYear()) {
+      return result.toString();
+    }
+    result.append(String.format("%02d-", from.getMonthOfYear()));
+    if (from.getDayOfMonth() != to.getDayOfMonth()) {
+      return result.toString();
+    }
+    result.append(String.format("%02dT", from.getDayOfMonth()));
+    if (from.getHourOfDay() != to.getHourOfDay()) {
+      return result.toString();
+    }
+    result.append(String.format("%02d:", from.getHourOfDay()));
+    return result.toString();
   }
 }

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -14,77 +14,107 @@
 
 package google.registry.backup;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
+import static google.registry.backup.RestoreCommitLogsAction.DRY_RUN_PARAM;
 import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.DateTimeUtils.isAtOrAfter;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardHours;
 
 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.Key;
-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobInfo;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.replay.DatastoreEntity;
+import google.registry.model.replay.DatastoreOnlyEntity;
+import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.ReplaySpecializer;
+import google.registry.model.replay.SqlReplayCheckpoint;
 import google.registry.model.server.Lock;
 import google.registry.model.translators.VKeyTranslatorFactory;
 import google.registry.persistence.VKey;
 import google.registry.request.Action;
+import google.registry.request.Action.Method;
+import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.DatastoreOnlyEntity;
-import google.registry.schema.replay.NonReplicatedEntity;
-import google.registry.schema.replay.ReplaySpecializer;
-import google.registry.schema.replay.SqlReplayCheckpoint;
+import google.registry.util.Clock;
 import google.registry.util.RequestStatusChecker;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.channels.Channels;
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.servlet.http.HttpServletResponse;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
+import org.joda.time.Seconds;
 
 /** Action that replays commit logs to Cloud SQL to keep it up to date. */
 @Action(
     service = Action.Service.BACKEND,
     path = ReplayCommitLogsToSqlAction.PATH,
-    method = Action.Method.POST,
+    method = Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class ReplayCommitLogsToSqlAction implements Runnable {
 
   static final String PATH = "/_dr/task/replayCommitLogsToSql";
 
-  private static final int BLOCK_SIZE =
-      1024 * 1024; // Buffer 1mb at a time, for no particular reason.
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final Duration LEASE_LENGTH = standardHours(1);
 
-  @Inject GcsService gcsService;
+  private static final Duration LEASE_LENGTH = standardHours(1);
+  // Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
+  // request timeouts
+  private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
+
+  @Inject GcsUtils gcsUtils;
   @Inject Response response;
   @Inject RequestStatusChecker requestStatusChecker;
   @Inject GcsDiffFileLister diffLister;
+  @Inject Clock clock;
+
+  @Inject
+  @Config("commitLogGcsBucket")
+  String gcsBucket;
+
+  /** If true, will exit after logging the commit log files that would otherwise be replayed. */
+  @Inject
+  @Parameter(DRY_RUN_PARAM)
+  boolean dryRun;
 
   @Inject
   ReplayCommitLogsToSqlAction() {}
 
   @Override
   public void run() {
-    if (!RegistryConfig.getCloudSqlReplayCommitLogs()) {
-      String message = "ReplayCommitLogsToSqlAction was called but disabled in the config.";
-      logger.atWarning().log(message);
+    DateTime startTime = clock.nowUtc();
+    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(startTime);
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      String message =
+          String.format(
+              "Skipping ReplayCommitLogsToSqlAction because we are in migration phase %s.", state);
+      logger.atInfo().log(message);
       // App Engine will retry on any non-2xx status code, which we don't want in this case.
       response.setStatus(SC_NO_CONTENT);
       response.setPayload(message);
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
@@ -96,44 +126,120 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       return;
     }
     try {
-      replayFiles();
-      response.setStatus(HttpServletResponse.SC_OK);
-      logger.atInfo().log("ReplayCommitLogsToSqlAction completed successfully.");
+      logger.atInfo().log("Beginning replay of commit logs.");
+      String resultMessage;
+      if (dryRun) {
+        resultMessage = executeDryRun();
+      } else {
+        resultMessage = replayFiles(startTime);
+      }
+      response.setStatus(SC_OK);
+      response.setPayload(resultMessage);
+      logger.atInfo().log(resultMessage);
+    } catch (Throwable t) {
+      String message = "Errored out replaying files.";
+      logger.atSevere().withCause(t).log(message);
+      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 
-  private void replayFiles() {
+  private String executeDryRun() {
     // Start at the first millisecond we haven't seen yet
-    DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
-    // If there's an inconsistent file set, this will throw IllegalStateException and the job
-    // will try later -- this is likely because an export hasn't finished yet.
-    ImmutableList<GcsFileMetadata> commitLogFiles =
-        diffLister.listDiffFiles(fromTime, /* current time */ null);
-    for (GcsFileMetadata metadata : commitLogFiles) {
-      // One transaction per GCS file
-      jpaTm().transact(() -> processFile(metadata));
-    }
-    logger.atInfo().log("Replayed %d commit log files to SQL successfully.", commitLogFiles.size());
+    DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    // Search through the end of the hour
+    DateTime searchEndTime =
+        searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
+    ImmutableList<String> fileBatch =
+        diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime).stream()
+            .map(BlobInfo::getName)
+            .collect(toImmutableList());
+    return String.format(
+        "Running in dry-run mode, the first set of commit log files processed would be from "
+            + "searching from %s to %s and would contain %d file(s). They are (limit 10): \n%s",
+        searchStartTime,
+        searchEndTime,
+        fileBatch.size(),
+        fileBatch.stream().limit(10).collect(toImmutableList()));
   }
 
-  private void processFile(GcsFileMetadata metadata) {
-    try (InputStream input =
-        Channels.newInputStream(
-            gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+  private String replayFiles(DateTime startTime) {
+    DateTime replayTimeoutTime = startTime.plus(REPLAY_TIMEOUT_DURATION);
+    DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    int filesProcessed = 0;
+    int transactionsProcessed = 0;
+    // Starting from one millisecond after the last file we processed, search for and import files
+    // one hour at a time until we catch up to the current time or we hit the replay timeout (in
+    // which case the next run will pick up from where we leave off).
+    //
+    // We use hour-long batches because GCS supports filename prefix-based searches.
+    while (true) {
+      if (isAtOrAfter(clock.nowUtc(), replayTimeoutTime)) {
+        return createResponseString(
+            "Reached max execution time", startTime, filesProcessed, transactionsProcessed);
+      }
+      if (isBeforeOrAt(clock.nowUtc(), searchStartTime)) {
+        return createResponseString(
+            "Caught up to current time", startTime, filesProcessed, transactionsProcessed);
+      }
+      // Search through the end of the hour
+      DateTime searchEndTime =
+          searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
+      ImmutableList<BlobInfo> fileBatch =
+          diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime);
+      if (fileBatch.isEmpty()) {
+        logger.atInfo().log(
+            "No remaining files found in hour %s, continuing search in the next hour.",
+            searchStartTime.toString("yyyy-MM-dd HH"));
+      }
+      for (BlobInfo file : fileBatch) {
+        transactionsProcessed += processFile(file);
+        filesProcessed++;
+        if (clock.nowUtc().isAfter(replayTimeoutTime)) {
+          return createResponseString(
+              "Reached max execution time", startTime, filesProcessed, transactionsProcessed);
+        }
+      }
+      searchStartTime = searchEndTime.plusMillis(1);
+    }
+  }
+
+  private String createResponseString(
+      String msg, DateTime startTime, int filesProcessed, int transactionsProcessed) {
+    double tps =
+        (double) transactionsProcessed
+            / (double) Seconds.secondsBetween(startTime, clock.nowUtc()).getSeconds();
+    return String.format(
+        "%s after replaying %d file(s) containing %d total transaction(s) (%.2f tx/s).",
+        msg, filesProcessed, transactionsProcessed, tps);
+  }
+
+  /**
+   * Replays the commit logs in the given commit log file and returns the number of transactions
+   * committed.
+   */
+  private int processFile(BlobInfo metadata) {
+    try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
       // Load and process the Datastore transactions one at a time
       ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
           CommitLogImports.loadEntitiesByTransaction(input);
-      allTransactions.forEach(this::replayTransaction);
+      try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+          UpdateAutoTimestamp.disableAutoUpdate()) {
+        allTransactions.forEach(
+            transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
+      }
       // if we succeeded, set the last-seen time
-      DateTime checkpoint =
-          DateTime.parse(
-              metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
-      SqlReplayCheckpoint.set(checkpoint);
-      logger.atInfo().log("Replayed %d transactions from commit log file.", allTransactions.size());
+      DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
+      jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
+      logger.atInfo().log(
+          "Replayed %d transactions from commit log file %s with size %d B.",
+          allTransactions.size(), metadata.getName(), metadata.getSize());
+      return allTransactions.size();
     } catch (IOException e) {
-      throw new RuntimeException(e);
+      throw new RuntimeException(
+          "Errored out while replaying commit log file " + metadata.getName(), e);
     }
   }
 
@@ -151,15 +257,26 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
   }
 
   private void handleEntityPut(Entity entity) {
-    Object ofyPojo = ofy().toPojo(entity);
-    if (ofyPojo instanceof DatastoreEntity) {
-      DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
-      datastoreEntity.toSqlEntity().ifPresent(jpaTm()::put);
-    } else {
-      // this should never happen, but we shouldn't fail on it
-      logger.atSevere().log(
-          "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
-          ofyPojo.getClass());
+    Object ofyPojo = auditedOfy().toPojo(entity);
+    try {
+      if (ofyPojo instanceof DatastoreEntity) {
+        DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
+        datastoreEntity
+            .toSqlEntity()
+            .ifPresent(
+                sqlEntity -> {
+                  sqlEntity.beforeSqlSaveOnReplay();
+                  jpaTm().putIgnoringReadOnlyWithoutBackup(sqlEntity);
+                });
+      } else {
+        // this should never happen, but we shouldn't fail on it
+        logger.atSevere().log(
+            "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
+            ofyPojo.getClass());
+      }
+    } catch (Throwable t) {
+      logger.atSevere().log("Error when replaying object %s.", ofyPojo);
+      throw t;
     }
   }
 
@@ -175,13 +292,18 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
           "Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
       return;
     }
-    Class<?> entityClass = entityVKey.getKind();
-    // Delete the key iff the class represents a JPA entity that is replicated
-    if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
-        && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
-        && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
-      ReplaySpecializer.beforeSqlDelete(entityVKey);
-      jpaTm().delete(entityVKey);
+    try {
+      Class<?> entityClass = entityVKey.getKind();
+      // Delete the key iff the class represents a JPA entity that is replicated
+      if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
+          && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
+          && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
+        ReplaySpecializer.beforeSqlDelete(entityVKey);
+        jpaTm().deleteIgnoringReadOnlyWithoutBackup(entityVKey);
+      }
+    } catch (Throwable t) {
+      logger.atSevere().log("Error when deleting key %s.", entityVKey);
+      throw t;
     }
   }
 

core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java

@@ -18,14 +18,14 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterators.peekingIterator;
 import static google.registry.backup.BackupUtils.createDeserializingIterator;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 
 import com.google.appengine.api.datastore.DatastoreService;
 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.EntityTranslator;
-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobInfo;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.PeekingIterator;
 import com.google.common.collect.Streams;
@@ -33,8 +33,11 @@ import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.Result;
 import com.googlecode.objectify.util.ResultNow;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
+import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
@@ -46,10 +49,10 @@ import google.registry.request.auth.Auth;
 import google.registry.util.Retrier;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.channels.Channels;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Stream;
 import javax.inject.Inject;
@@ -62,49 +65,62 @@ import org.joda.time.DateTime;
     method = Action.Method.POST,
     automaticallyPrintOk = true,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class RestoreCommitLogsAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  static final int BLOCK_SIZE = 1024 * 1024;  // Buffer 1mb at a time, for no particular reason.
-
   public static final String PATH = "/_dr/task/restoreCommitLogs";
   static final String DRY_RUN_PARAM = "dryRun";
   static final String FROM_TIME_PARAM = "fromTime";
   static final String TO_TIME_PARAM = "toTime";
+  static final String BUCKET_OVERRIDE_PARAM = "gcsBucket";
+
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
+
+  @Inject GcsUtils gcsUtils;
 
-  @Inject GcsService gcsService;
   @Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
   @Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
   @Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
+
+  @Inject
+  @Parameter(BUCKET_OVERRIDE_PARAM)
+  Optional<String> gcsBucketOverride;
+
   @Inject DatastoreService datastoreService;
   @Inject GcsDiffFileLister diffLister;
+
+  @Inject
+  @Config("commitLogGcsBucket")
+  String defaultGcsBucket;
+
   @Inject Retrier retrier;
   @Inject RestoreCommitLogsAction() {}
 
   @Override
   public void run() {
     checkArgument(
-        RegistryEnvironment.get() == RegistryEnvironment.ALPHA
-            || RegistryEnvironment.get() == RegistryEnvironment.CRASH
-            || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
-        "DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
+        !FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
+        "DO NOT RUN IN PRODUCTION OR SANDBOX.");
     if (dryRun) {
-      logger.atInfo().log("Running in dryRun mode");
+      logger.atInfo().log("Running in dry-run mode.");
     }
-    List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
+    String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
+    logger.atInfo().log("Restoring from %s.", gcsBucket);
+    List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
     if (diffFiles.isEmpty()) {
-      logger.atInfo().log("Nothing to restore");
+      logger.atInfo().log("Nothing to restore.");
       return;
     }
     Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
     CommitLogCheckpoint lastCheckpoint = null;
-    for (GcsFileMetadata metadata : diffFiles) {
-      logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
-      try (InputStream input = Channels.newInputStream(
-          gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+    for (BlobInfo metadata : diffFiles) {
+      logger.atInfo().log("Restoring: %s", metadata.getName());
+      try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
         PeekingIterator<ImmutableObject> commitLogs =
-            peekingIterator(createDeserializingIterator(input));
+            peekingIterator(createDeserializingIterator(input, true));
         lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
         saveOfy(ImmutableList.of(lastCheckpoint));  // Save the checkpoint itself.
         while (commitLogs.hasNext()) {
@@ -129,7 +145,7 @@ public class RestoreCommitLogsAction implements Runnable {
                                 .build()),
                 Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
             .collect(toImmutableList()));
-    logger.atInfo().log("Restore complete");
+    logger.atInfo().log("Restore complete.");
   }
 
   /**
@@ -146,10 +162,10 @@ public class RestoreCommitLogsAction implements Runnable {
   private CommitLogManifest restoreOneTransaction(PeekingIterator<ImmutableObject> commitLogs) {
     final CommitLogManifest manifest = (CommitLogManifest) commitLogs.next();
     Result<?> deleteResult = deleteAsync(manifest.getDeletions());
-    List<Entity> entitiesToSave = Lists.newArrayList(ofy().save().toEntity(manifest));
+    List<Entity> entitiesToSave = Lists.newArrayList(auditedOfy().save().toEntity(manifest));
     while (commitLogs.hasNext() && commitLogs.peek() instanceof CommitLogMutation) {
       CommitLogMutation mutation = (CommitLogMutation) commitLogs.next();
-      entitiesToSave.add(ofy().save().toEntity(mutation));
+      entitiesToSave.add(auditedOfy().save().toEntity(mutation));
       entitiesToSave.add(EntityTranslator.createFromPbBytes(mutation.getEntityProtoBytes()));
     }
     saveRaw(entitiesToSave);
@@ -176,7 +192,8 @@ public class RestoreCommitLogsAction implements Runnable {
       return;
     }
     retrier.callWithRetry(
-        () -> ofy().saveWithoutBackup().entities(objectsToSave).now(), RuntimeException.class);
+        () -> auditedOfy().saveWithoutBackup().entities(objectsToSave).now(),
+        RuntimeException.class);
   }
 
   private Result<?> deleteAsync(Set<Key<?>> keysToDelete) {
@@ -185,7 +202,7 @@ public class RestoreCommitLogsAction implements Runnable {
     }
     return dryRun || keysToDelete.isEmpty()
         ? new ResultNow<Void>(null)
-        : ofy().deleteWithoutBackup().keys(keysToDelete);
+        : auditedOfy().deleteWithoutBackup().keys(keysToDelete);
   }
 
 }

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -19,6 +19,7 @@ import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.appengine.api.datastore.Key;
 import com.google.auto.value.AutoValue;
 import com.google.auto.value.extension.memoized.Memoized;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.ofy.CommitLogManifest;
 import google.registry.model.ofy.CommitLogMutation;
 import java.io.Serializable;
@@ -47,7 +48,7 @@ import javax.annotation.Nullable;
  *
  * <ul>
  *   <li>Convert an Objectify entity to a Datastore {@link Entity}: {@code
- *       ofy().save().toEntity(..)}
+ *       auditedOfy().save().toEntity(..)}
  *   <li>Entity is serializable, but the more efficient approach is to convert an Entity to a
  *       ProtocolBuffer ({@link com.google.storage.onestore.v3.OnestoreEntity.EntityProto}) and then
  *       to raw bytes.
@@ -76,6 +77,7 @@ import javax.annotation.Nullable;
  * property type in this class.
  */
 @AutoValue
+@DeleteAfterMigration
 public abstract class VersionedEntity implements Serializable {
 
   private static final long serialVersionUID = 1L;
@@ -105,29 +107,29 @@ public abstract class VersionedEntity implements Serializable {
    * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
    * information.
    */
-  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+  static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
     long commitTimeMillis = manifest.getCommitTime().getMillis();
     return manifest.getDeletions().stream()
         .map(com.googlecode.objectify.Key::getRaw)
-        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+        .map(key -> newBuilder().commitTimeMills(commitTimeMillis).key(key).build());
   }
 
   /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
-  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+  static VersionedEntity fromMutation(CommitLogMutation mutation) {
     return from(
         com.googlecode.objectify.Key.create(mutation).getParent().getId(),
         mutation.getEntityProtoBytes());
   }
 
   public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
-    return builder()
+    return newBuilder()
         .entityProtoBytes(entityProtoBytes)
         .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
         .commitTimeMills(commitTimeMillis)
         .build();
   }
 
-  static Builder builder() {
+  private static Builder newBuilder() {
     return new AutoValue_VersionedEntity.Builder();
   }
 
@@ -142,7 +144,7 @@ public abstract class VersionedEntity implements Serializable {
 
     public abstract VersionedEntity build();
 
-    public Builder entityProtoBytes(byte[] bytes) {
+    Builder entityProtoBytes(byte[] bytes) {
       return entityProtoBytes(new ImmutableBytes(bytes));
     }
   }

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,14 +24,12 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
+import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
-import google.registry.schema.domain.RegistryLock;
 import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.Retrier;
 import javax.inject.Inject;
@@ -57,8 +55,6 @@ public final class AsyncTaskEnqueuer {
   public static final String QUEUE_ASYNC_DELETE = "async-delete-pull";
   public static final String QUEUE_ASYNC_HOST_RENAME = "async-host-rename-pull";
 
-  public static final String PATH_RESAVE_ENTITY = "/_dr/task/resaveEntity";
-
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final Duration MAX_ASYNC_ETA = Duration.standardDays(30);
 
@@ -86,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -98,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -112,11 +106,11 @@ public final class AsyncTaskEnqueuer {
     logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
     String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
     TaskOptions task =
-        TaskOptions.Builder.withUrl(PATH_RESAVE_ENTITY)
+        TaskOptions.Builder.withUrl(ResaveEntityAction.PATH)
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -128,18 +122,17 @@ public final class AsyncTaskEnqueuer {
   public void enqueueAsyncDelete(
       EppResource resourceToDelete,
       DateTime now,
-      String requestingClientId,
+      String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingClientId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
-            .param(PARAM_REQUESTING_CLIENT_ID, requestingClientId)
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().stringify())
+            .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))
             .param(PARAM_REQUESTED_TIME, now.toString());
@@ -155,7 +148,7 @@ public final class AsyncTaskEnqueuer {
     addTaskToQueueWithRetry(
         asyncDnsRefreshPullQueue,
         TaskOptions.Builder.withMethod(Method.PULL)
-            .param(PARAM_HOST_KEY, hostKey.getOfyKey().getString())
+            .param(PARAM_HOST_KEY, hostKey.stringify())
             .param(PARAM_REQUESTED_TIME, now.toString()));
   }
 

core/src/main/java/google/registry/batch/BatchModule.java

@@ -24,6 +24,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.request.RequestParameters.extractIntParameter;
 import static google.registry.request.RequestParameters.extractLongParameter;
 import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
+import static google.registry.request.RequestParameters.extractOptionalDatetimeParameter;
 import static google.registry.request.RequestParameters.extractOptionalIntParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
@@ -32,10 +33,8 @@ import static google.registry.request.RequestParameters.extractSetOfDatetimePara
 
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.model.ImmutableObject;
 import google.registry.request.Parameter;
 import java.util.Optional;
 import javax.inject.Named;
@@ -78,8 +77,8 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
-  static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
-    return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
+  static String provideResourceKey(HttpServletRequest req) {
+    return extractRequiredParameter(req, PARAM_RESOURCE_KEY);
   }
 
   @Provides
@@ -106,6 +105,13 @@ public class BatchModule {
     return extractIntParameter(req, RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM);
   }
 
+  @Provides
+  @Parameter(ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME)
+  static Optional<DateTime> provideCursorTime(HttpServletRequest req) {
+    return extractOptionalDatetimeParameter(
+        req, ExpandRecurringBillingEventsAction.PARAM_CURSOR_TIME);
+  }
+
   @Provides
   @Named(QUEUE_ASYNC_ACTIONS)
   static Queue provideAsyncActionsPushQueue() {

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -34,7 +34,7 @@ import static google.registry.model.ResourceTransferUtils.denyPendingTransfer;
 import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete;
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
 import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE;
 import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE_FAILURE;
 import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE;
@@ -73,6 +73,7 @@ import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.mapreduce.inputs.NullInput;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
@@ -109,10 +110,12 @@ import org.joda.time.Duration;
  * A mapreduce that processes batch asynchronous deletions of contact and host resources by mapping
  * over all domains and checking for any references to the contacts/hosts in pending deletion.
  */
+@Deprecated
 @Action(
     service = Action.Service.BACKEND,
     path = "/_dr/task/deleteContactsAndHosts",
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
 public class DeleteContactsAndHostsAction implements Runnable {
 
   static final String KIND_CONTACT = getKind(ContactResource.class);
@@ -310,7 +313,7 @@ public class DeleteContactsAndHostsAction implements Runnable {
     @Override
     public void reduce(final DeletionRequest deletionRequest, ReducerInput<Boolean> values) {
       final boolean hasNoActiveReferences = !Iterators.contains(values, true);
-      logger.atInfo().log("Processing async deletion request for %s", deletionRequest.key());
+      logger.atInfo().log("Processing async deletion request for %s.", deletionRequest.key());
       DeletionResult result =
           tm()
               .transactNew(
@@ -335,22 +338,22 @@ public class DeleteContactsAndHostsAction implements Runnable {
         DeletionRequest deletionRequest, boolean hasNoActiveReferences) {
       DateTime now = tm().getTransactionTime();
       EppResource resource =
-          ofy().load().key(deletionRequest.key()).now().cloneProjectedAtTime(now);
+          auditedOfy().load().key(deletionRequest.key()).now().cloneProjectedAtTime(now);
       // Double-check transactionally that the resource is still active and in PENDING_DELETE.
       if (!doesResourceStateAllowDeletion(resource, now)) {
         return DeletionResult.create(Type.ERRORED, "");
       }
       // Contacts and external hosts have a direct client id. For subordinate hosts it needs to be
       // read off of the superordinate domain.
-      String resourceClientId = resource.getPersistedCurrentSponsorClientId();
+      String resourceRegistrarId = resource.getPersistedCurrentSponsorRegistrarId();
       if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
-        resourceClientId =
+        resourceRegistrarId =
             tm().loadByKey(((HostResource) resource).getSuperordinateDomain())
                 .cloneProjectedAtTime(now)
-                .getCurrentSponsorClientId();
+                .getCurrentSponsorRegistrarId();
       }
       boolean requestedByCurrentOwner =
-          resourceClientId.equals(deletionRequest.requestingClientId());
+          resourceRegistrarId.equals(deletionRequest.requestingClientId());
 
       boolean deleteAllowed =
           hasNoActiveReferences && (requestedByCurrentOwner || deletionRequest.isSuperuser());
@@ -369,16 +372,15 @@ public class DeleteContactsAndHostsAction implements Runnable {
                       : "it was transferred prior to deletion");
 
       HistoryEntry historyEntry =
-          new HistoryEntry.Builder()
-              .setClientId(deletionRequest.requestingClientId())
+          HistoryEntry.createBuilderForResource(resource)
+              .setRegistrarId(deletionRequest.requestingClientId())
               .setModificationTime(now)
               .setType(getHistoryEntryType(resource, deleteAllowed))
-              .setParent(deletionRequest.key())
               .build();
 
       PollMessage.OneTime pollMessage =
           new PollMessage.OneTime.Builder()
-              .setClientId(deletionRequest.requestingClientId())
+              .setRegistrarId(deletionRequest.requestingClientId())
               .setMsg(pollMessageText)
               .setParent(historyEntry)
               .setEventTime(now)
@@ -408,7 +410,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
       } else {
         resourceToSave = resource.asBuilder().removeStatusValue(PENDING_DELETE).build();
       }
-      ofy().save().<ImmutableObject>entities(resourceToSave, historyEntry, pollMessage);
+      auditedOfy()
+          .save()
+          .<ImmutableObject>entities(resourceToSave, historyEntry.asHistoryEntry(), pollMessage);
       return DeletionResult.create(
           deleteAllowed ? Type.DELETED : Type.NOT_DELETED, pollMessageText);
     }
@@ -521,17 +525,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
-          checkNotNull(ofy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+          checkNotNull(
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(
@@ -602,12 +608,12 @@ public class DeleteContactsAndHostsAction implements Runnable {
   static boolean doesResourceStateAllowDeletion(EppResource resource, DateTime now) {
     Key<EppResource> key = Key.create(resource);
     if (isDeleted(resource, now)) {
-      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted", key);
+      logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted.", key);
       return false;
     }
     if (!resource.getStatusValues().contains(PENDING_DELETE)) {
       logger.atWarning().log(
-          "Cannot asynchronously delete %s because it is not in PENDING_DELETE", key);
+          "Cannot asynchronously delete %s because it is not in PENDING_DELETE.", key);
       return false;
     }
     return true;

core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java

@@ -17,8 +17,8 @@ package google.registry.batch;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.flows.FlowUtils.marshalWithLenientRetry;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -36,6 +36,7 @@ import google.registry.flows.StatelessRequestSessionMetadata;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.ProtocolDefinition;
 import google.registry.model.eppoutput.EppOutput;
+import google.registry.persistence.transaction.QueryComposer.Comparator;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
@@ -128,12 +129,15 @@ public class DeleteExpiredDomainsAction implements Runnable {
     logger.atInfo().log(
         "Deleting non-renewing domains with autorenew end times up through %s.", runTime);
 
-    // Note: This query is (and must be) non-transactional, and thus, is only eventually consistent.
+    // Note: in Datastore, this query is (and must be) non-transactional, and thus, is only
+    // eventually consistent.
     ImmutableList<DomainBase> domainsToDelete =
-        ofy().load().type(DomainBase.class).filter("autorenewEndTime <=", runTime).list().stream()
-            // Datastore can't do two inequalities in one query, so the second happens in-memory.
-            .filter(d -> d.getDeletionTime().isEqual(END_OF_TIME))
-            .collect(toImmutableList());
+        transactIfJpaTm(
+            () ->
+                tm().createQueryComposer(DomainBase.class)
+                    .where("autorenewEndTime", Comparator.LTE, runTime)
+                    .where("deletionTime", Comparator.EQ, END_OF_TIME)
+                    .list());
     if (domainsToDelete.isEmpty()) {
       logger.atInfo().log("Found 0 domains to delete.");
       response.setPayload("Found 0 domains to delete.");
@@ -163,7 +167,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
   /** Runs the actual domain delete flow and returns whether the deletion was successful. */
   private boolean runDomainDeleteFlow(DomainBase domain) {
-    logger.atInfo().log("Attempting to delete domain %s", domain.getDomainName());
+    logger.atInfo().log("Attempting to delete domain '%s'.", domain.getDomainName());
     // Create a new transaction that the flow's execution will be enlisted in that loads the domain
     // transactionally. This way we can ensure that nothing else has modified the domain in question
     // in the intervening period since the query above found it.
@@ -199,7 +203,7 @@ public class DeleteExpiredDomainsAction implements Runnable {
 
     if (eppOutput.isPresent()) {
       if (eppOutput.get().isSuccess()) {
-        logger.atInfo().log("Successfully deleted domain %s", domain.getDomainName());
+        logger.atInfo().log("Successfully deleted domain '%s'.", domain.getDomainName());
       } else {
         logger.atSevere().log(
             "Failed to delete domain %s; EPP response:\n\n%s",

core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java

@@ -15,12 +15,14 @@
 package google.registry.batch;
 
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryEnvironment.PRODUCTION;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.appengine.tools.mapreduce.Mapper;
 import com.google.common.collect.ImmutableList;
@@ -28,16 +30,24 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryEnvironment;
+import google.registry.flows.poll.PollFlowUtils;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.model.EppResource;
+import google.registry.model.EppResourceUtils;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.ForeignKeyIndex;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
 import java.util.List;
 import javax.inject.Inject;
 
@@ -46,8 +56,8 @@ import javax.inject.Inject;
  * the associated ForeignKey and EppResourceIndex entities.
  *
  * <p>This only deletes contacts and hosts, NOT domains. To delete domains, use {@link
- * DeleteLoadTestDataAction} and pass it the TLD(s) that the load test domains were created on. Note
- * that DeleteLoadTestDataAction is safe enough to run in production whereas this mapreduce is not,
+ * DeleteProberDataAction} and pass it the TLD(s) that the load test domains were created on. Note
+ * that DeleteProberDataAction is safe enough to run in production whereas this mapreduce is not,
  * but this one does not need to be runnable in production because load testing isn't run against
  * production.
  */
@@ -68,15 +78,22 @@ public class DeleteLoadTestDataAction implements Runnable {
    */
   private static final ImmutableSet<String> LOAD_TEST_REGISTRARS = ImmutableSet.of("proxy");
 
-  @Inject
-  @Parameter(PARAM_DRY_RUN)
-  boolean isDryRun;
-
-  @Inject MapreduceRunner mrRunner;
-  @Inject Response response;
+  private final boolean isDryRun;
+  private final MapreduceRunner mrRunner;
+  private final Response response;
+  private final Clock clock;
 
   @Inject
-  DeleteLoadTestDataAction() {}
+  DeleteLoadTestDataAction(
+      @Parameter(PARAM_DRY_RUN) boolean isDryRun,
+      MapreduceRunner mrRunner,
+      Response response,
+      Clock clock) {
+    this.isDryRun = isDryRun;
+    this.mrRunner = mrRunner;
+    this.response = response;
+    this.clock = clock;
+  }
 
   @Override
   public void run() {
@@ -87,14 +104,85 @@ public class DeleteLoadTestDataAction implements Runnable {
         !RegistryEnvironment.get().equals(PRODUCTION),
         "This mapreduce is not safe to run on PRODUCTION.");
 
-    mrRunner
-        .setJobName("Delete load test data")
-        .setModuleName("backend")
-        .runMapOnly(
-            new DeleteLoadTestDataMapper(isDryRun),
-            ImmutableList.of(
-                createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
-        .sendLinkToMapreduceConsole(response);
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Delete load test data")
+          .setModuleName("backend")
+          .runMapOnly(
+              new DeleteLoadTestDataMapper(isDryRun),
+              ImmutableList.of(
+                  createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      tm().transact(
+              () -> {
+                LOAD_TEST_REGISTRARS.forEach(this::deletePollMessages);
+                tm().loadAllOfStream(ContactResource.class).forEach(this::deleteContact);
+                tm().loadAllOfStream(HostResource.class).forEach(this::deleteHost);
+              });
+    }
+  }
+
+  private void deletePollMessages(String registrarId) {
+    ImmutableList<PollMessage> pollMessages =
+        PollFlowUtils.createPollMessageQuery(registrarId, END_OF_TIME).list();
+    if (isDryRun) {
+      logger.atInfo().log(
+          "Would delete %d poll messages for registrar %s.", pollMessages.size(), registrarId);
+    } else {
+      pollMessages.forEach(tm()::delete);
+    }
+  }
+
+  private void deleteContact(ContactResource contact) {
+    if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorRegistrarId())) {
+      return;
+    }
+    // We cannot remove contacts from domains in the general case, so we cannot delete contacts
+    // that are linked to domains (since it would break the foreign keys)
+    if (EppResourceUtils.isLinked(contact.createVKey(), clock.nowUtc())) {
+      logger.atWarning().log(
+          "Cannot delete contact with repo ID %s since it is referenced from a domain.",
+          contact.getRepoId());
+      return;
+    }
+    deleteResource(contact);
+  }
+
+  private void deleteHost(HostResource host) {
+    if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorRegistrarId())) {
+      return;
+    }
+    VKey<HostResource> hostVKey = host.createVKey();
+    // We can remove hosts from linked domains, so we should do so then delete the hosts
+    ImmutableSet<VKey<DomainBase>> linkedDomains =
+        EppResourceUtils.getLinkedDomainKeys(hostVKey, clock.nowUtc(), null);
+    tm().loadByKeys(linkedDomains)
+        .values()
+        .forEach(
+            domain -> {
+              ImmutableSet<VKey<HostResource>> remainingHosts =
+                  domain.getNsHosts().stream()
+                      .filter(vkey -> !vkey.equals(hostVKey))
+                      .collect(toImmutableSet());
+              tm().put(domain.asBuilder().setNameservers(remainingHosts).build());
+            });
+    deleteResource(host);
+  }
+
+  private void deleteResource(EppResource eppResource) {
+    // In SQL, the only objects parented on the resource are poll messages (deleted above) and
+    // history objects.
+    ImmutableList<HistoryEntry> historyObjects =
+        HistoryEntryDao.loadHistoryObjectsForResource(eppResource.createVKey());
+    if (isDryRun) {
+      logger.atInfo().log(
+          "Would delete repo ID %s along with %d history objects.",
+          eppResource.getRepoId(), historyObjects.size());
+    } else {
+      historyObjects.forEach(tm()::delete);
+      tm().delete(eppResource);
+    }
   }
 
   /** Provides the map method that runs for each existing contact and host entity. */
@@ -110,7 +198,7 @@ public class DeleteLoadTestDataAction implements Runnable {
 
     @Override
     public final void map(EppResource resource) {
-      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorClientId())) {
+      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorRegistrarId())) {
         deleteResource(resource);
         getContext()
             .incrementCounter(
@@ -125,12 +213,11 @@ public class DeleteLoadTestDataAction implements Runnable {
           Key.create(EppResourceIndex.create(Key.create(resource)));
       final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(resource);
       int numEntitiesDeleted =
-          tm()
-              .transact(
+          tm().transact(
                   () -> {
                     // This ancestor query selects all descendant entities.
                     List<Key<Object>> resourceAndDependentKeys =
-                        ofy().load().ancestor(resource).keys().list();
+                        auditedOfy().load().ancestor(resource).keys().list();
                     ImmutableSet<Key<?>> allKeys =
                         new ImmutableSet.Builder<Key<?>>()
                             .add(fki)
@@ -140,7 +227,7 @@ public class DeleteLoadTestDataAction implements Runnable {
                     if (isDryRun) {
                       logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
                     } else {
-                      ofy().deleteWithoutBackup().keys(allKeys);
+                      auditedOfy().deleteWithoutBackup().keys(allKeys);
                     }
                     return allKeys.size();
                   });

core/src/main/java/google/registry/batch/DeleteProberDataAction.java

@@ -20,9 +20,10 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryEnvironment.PRODUCTION;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.model.registry.Registries.getTldsOfType;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
+import static google.registry.model.tld.Registries.getTldsOfType;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
@@ -42,27 +43,31 @@ import google.registry.config.RegistryEnvironment;
 import google.registry.dns.DnsQueue;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
+import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.EppResourceUtils;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.ForeignKeyIndex;
-import google.registry.model.registry.Registry;
-import google.registry.model.registry.Registry.TldType;
-import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import google.registry.model.tld.Registry.TldType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.inject.Inject;
+import org.hibernate.CacheMode;
+import org.hibernate.ScrollMode;
+import org.hibernate.ScrollableResults;
+import org.hibernate.query.Query;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /**
- * Deletes all prober DomainBases and their subordinate history entries, poll messages, and
- * billing events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
- *
- * <p>See: https://www.youtube.com/watch?v=xuuv0syoHnM
+ * Deletes all prober DomainBases and their subordinate history entries, poll messages, and billing
+ * events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
  */
 @Action(
     service = Action.Service.BACKEND,
@@ -73,10 +78,51 @@ public class DeleteProberDataAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
+  /**
+   * The maximum amount of time we allow a prober domain to be in use.
+   *
+   * <p>In practice, the prober's connection will time out well before this duration. This includes
+   * a decent buffer.
+   */
+  private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
+
+  /**
+   * The minimum amount of time we want a domain to be "soft deleted".
+   *
+   * <p>The domain has to remain soft deleted for at least enough time for the DNS task to run and
+   * remove it from DNS itself. This is probably on the order of minutes.
+   */
+  private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
+
+  private static final DnsQueue dnsQueue = DnsQueue.create();
+
+  // Domains to delete must:
+  // 1. Be in one of the prober TLDs
+  // 2. Not be a nic domain
+  // 3. Have no subordinate hosts
+  // 4. Not still be used (within an hour of creation time)
+  // 5. Either be active (creationTime <= now < deletionTime) or have been deleted a while ago (this
+  //    prevents accidental double-map with the same key from immediately deleting active domains)
+  //
+  // Note: creationTime must be compared to a Java object (CreateAutoTimestamp) but deletionTime can
+  // be compared directly to the SQL timestamp (it's a DateTime)
+  private static final String DOMAIN_QUERY_STRING =
+      "FROM Domain d WHERE d.tld IN :tlds AND d.fullyQualifiedDomainName NOT LIKE 'nic.%' AND"
+          + " (d.subordinateHosts IS EMPTY OR d.subordinateHosts IS NULL) AND d.creationTime <"
+          + " :creationTimeCutoff AND ((d.creationTime <= :nowAutoTimestamp AND d.deletionTime >"
+          + " current_timestamp()) OR d.deletionTime < :nowMinusSoftDeleteDelay) ORDER BY d.repoId";
+
+  /** Number of domains to retrieve and delete per SQL transaction. */
+  private static final int BATCH_SIZE = 1000;
+
   @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
   /** List of TLDs to work on. If empty - will work on all TLDs that end with .test. */
   @Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds;
-  @Inject @Config("registryAdminClientId") String registryAdminClientId;
+
+  @Inject
+  @Config("registryAdminClientId")
+  String registryAdminRegistrarId;
+
   @Inject MapreduceRunner mrRunner;
   @Inject Response response;
   @Inject DeleteProberDataAction() {}
@@ -84,25 +130,14 @@ public class DeleteProberDataAction implements Runnable {
   @Override
   public void run() {
     checkState(
-        !Strings.isNullOrEmpty(registryAdminClientId),
+        !Strings.isNullOrEmpty(registryAdminRegistrarId),
         "Registry admin client ID must be configured for prober data deletion to work");
-    mrRunner
-        .setJobName("Delete prober data")
-        .setModuleName("backend")
-        .runMapOnly(
-            new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId),
-            ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
-        .sendLinkToMapreduceConsole(response);
-  }
-
-  private ImmutableSet<String> getProberRoidSuffixes() {
     checkArgument(
         !PRODUCTION.equals(RegistryEnvironment.get())
             || tlds.stream().allMatch(tld -> tld.endsWith(".test")),
         "On production, can only work on TLDs that end with .test");
     ImmutableSet<String> deletableTlds =
-        getTldsOfType(TldType.TEST)
-            .stream()
+        getTldsOfType(TldType.TEST).stream()
             .filter(tld -> tlds.isEmpty() ? tld.endsWith(".test") : tlds.contains(tld))
             .collect(toImmutableSet());
     checkArgument(
@@ -110,10 +145,161 @@ public class DeleteProberDataAction implements Runnable {
         "If tlds are given, they must all exist and be TEST tlds. Given: %s, not found: %s",
         tlds,
         Sets.difference(tlds, deletableTlds));
-    return deletableTlds
-        .stream()
-        .map(tld -> Registry.get(tld).getRoidSuffix())
-        .collect(toImmutableSet());
+    ImmutableSet<String> proberRoidSuffixes =
+        deletableTlds.stream()
+            .map(tld -> Registry.get(tld).getRoidSuffix())
+            .collect(toImmutableSet());
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Delete prober data")
+          .setModuleName("backend")
+          .runMapOnly(
+              new DeleteProberDataMapper(proberRoidSuffixes, isDryRun, registryAdminRegistrarId),
+              ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      runSqlJob(deletableTlds);
+    }
+  }
+
+  private void runSqlJob(ImmutableSet<String> deletableTlds) {
+    AtomicInteger softDeletedDomains = new AtomicInteger();
+    AtomicInteger hardDeletedDomains = new AtomicInteger();
+    jpaTm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have soft-deleted" : "Soft-deleted", softDeletedDomains.get());
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have hard-deleted" : "Hard-deleted", hardDeletedDomains.get());
+  }
+
+  private void processDomains(
+      ImmutableSet<String> deletableTlds,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    DateTime now = tm().getTransactionTime();
+    // Scroll through domains, soft-deleting as necessary (very few will be soft-deleted) and
+    // keeping track of which domains to hard-delete (there can be many, so we batch them up)
+    ScrollableResults scrollableResult =
+        jpaTm()
+            .query(DOMAIN_QUERY_STRING, DomainBase.class)
+            .setParameter("tlds", deletableTlds)
+            .setParameter(
+                "creationTimeCutoff", CreateAutoTimestamp.create(now.minus(DOMAIN_USED_DURATION)))
+            .setParameter("nowMinusSoftDeleteDelay", now.minus(SOFT_DELETE_DELAY))
+            .setParameter("nowAutoTimestamp", CreateAutoTimestamp.create(now))
+            .unwrap(Query.class)
+            .setCacheMode(CacheMode.IGNORE)
+            .scroll(ScrollMode.FORWARD_ONLY);
+    ImmutableList.Builder<String> domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+    ImmutableList.Builder<String> hostNamesToHardDelete = new ImmutableList.Builder<>();
+    for (int i = 1; scrollableResult.next(); i = (i + 1) % BATCH_SIZE) {
+      DomainBase domain = (DomainBase) scrollableResult.get(0);
+      processDomain(
+          domain,
+          domainRepoIdsToHardDelete,
+          hostNamesToHardDelete,
+          softDeletedDomains,
+          hardDeletedDomains);
+      // Batch the deletion and DB flush + session clearing so we don't OOM
+      if (i == 0) {
+        hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+        domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+        hostNamesToHardDelete = new ImmutableList.Builder<>();
+        jpaTm().getEntityManager().flush();
+        jpaTm().getEntityManager().clear();
+      }
+    }
+    // process the remainder
+    hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+  }
+
+  private void processDomain(
+      DomainBase domain,
+      ImmutableList.Builder<String> domainRepoIdsToHardDelete,
+      ImmutableList.Builder<String> hostNamesToHardDelete,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    // If the domain is still active, that means that the prober encountered a failure and did not
+    // successfully soft-delete the domain (thus leaving its DNS entry published). We soft-delete
+    // it now so that the DNS entry can be handled. The domain will then be hard-deleted the next
+    // time the job is run.
+    if (EppResourceUtils.isActive(domain, tm().getTransactionTime())) {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would soft-delete the active domain: %s (%s).",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue);
+      }
+      softDeletedDomains.incrementAndGet();
+    } else {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would hard-delete the non-active domain: %s (%s) and its dependents.",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        domainRepoIdsToHardDelete.add(domain.getRepoId());
+        hostNamesToHardDelete.addAll(domain.getSubordinateHosts());
+      }
+      hardDeletedDomains.incrementAndGet();
+    }
+  }
+
+  private void hardDeleteDomainsAndHosts(
+      ImmutableList<String> domainRepoIds, ImmutableList<String> hostNames) {
+    jpaTm()
+        .query("DELETE FROM Host WHERE fullyQualifiedHostName IN :hostNames")
+        .setParameter("hostNames", hostNames)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingRecurrence WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM DomainHistory WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM Domain WHERE repoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+  }
+
+  // Take a DNS queue + admin registrar id as input so that it can be called from the mapper as well
+  private static void softDeleteDomain(
+      DomainBase domain, String registryAdminRegistrarId, DnsQueue localDnsQueue) {
+    DomainBase deletedDomain =
+        domain.asBuilder().setDeletionTime(tm().getTransactionTime()).setStatusValues(null).build();
+    DomainHistory historyEntry =
+        new DomainHistory.Builder()
+            .setDomain(domain)
+            .setType(DOMAIN_DELETE)
+            .setModificationTime(tm().getTransactionTime())
+            .setBySuperuser(true)
+            .setReason("Deletion of prober data")
+            .setRegistrarId(registryAdminRegistrarId)
+            .build();
+    // Note that we don't bother handling grace periods, billing events, pending transfers, poll
+    // messages, or auto-renews because those will all be hard-deleted the next time the job runs
+    // anyway.
+    tm().putAllWithoutBackup(ImmutableList.of(deletedDomain, historyEntry));
+    // updating foreign keys is a no-op in SQL
+    updateForeignKeyIndexDeletionTime(deletedDomain);
+    localDnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
   }
 
   /** Provides the map method that runs for each existing DomainBase entity. */
@@ -122,32 +308,17 @@ public class DeleteProberDataAction implements Runnable {
     private static final DnsQueue dnsQueue = DnsQueue.create();
     private static final long serialVersionUID = -7724537393697576369L;
 
-    /**
-     * The maximum amount of time we allow a prober domain to be in use.
-     *
-     * In practice, the prober's connection will time out well before this duration. This includes a
-     * decent buffer.
-     *
-     */
-    private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
-
-    /**
-     * The minimum amount of time we want a domain to be "soft deleted".
-     *
-     * The domain has to remain soft deleted for at least enough time for the DNS task to run and
-     * remove it from DNS itself. This is probably on the order of minutes.
-     */
-    private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
-
     private final ImmutableSet<String> proberRoidSuffixes;
     private final Boolean isDryRun;
-    private final String registryAdminClientId;
+    private final String registryAdminRegistrarId;
 
     public DeleteProberDataMapper(
-        ImmutableSet<String> proberRoidSuffixes, Boolean isDryRun, String registryAdminClientId) {
+        ImmutableSet<String> proberRoidSuffixes,
+        Boolean isDryRun,
+        String registryAdminRegistrarId) {
       this.proberRoidSuffixes = proberRoidSuffixes;
       this.isDryRun = isDryRun;
-      this.registryAdminClientId = registryAdminClientId;
+      this.registryAdminRegistrarId = registryAdminRegistrarId;
     }
 
     @Override
@@ -160,13 +331,13 @@ public class DeleteProberDataAction implements Runnable {
           getContext().incrementCounter("skipped, non-prober data");
         }
       } catch (Throwable t) {
-        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s", key);
+        logger.atSevere().withCause(t).log("Error while deleting prober data for key %s.", key);
         getContext().incrementCounter(String.format("error, kind %s", key.getKind()));
       }
     }
 
     private void deleteDomain(final Key<DomainBase> domainKey) {
-      final DomainBase domain = ofy().load().key(domainKey).now();
+      final DomainBase domain = auditedOfy().load().key(domainKey).now();
 
       DateTime now = DateTime.now(UTC);
 
@@ -201,9 +372,9 @@ public class DeleteProberDataAction implements Runnable {
       if (EppResourceUtils.isActive(domain, now)) {
         if (isDryRun) {
           logger.atInfo().log(
-              "Would soft-delete the active domain: %s (%s)", domainName, domainKey);
+              "Would soft-delete the active domain: %s (%s).", domainName, domainKey);
         } else {
-          softDeleteDomain(domain);
+          tm().transact(() -> softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue));
         }
         getContext().incrementCounter("domains soft-deleted");
         return;
@@ -220,14 +391,12 @@ public class DeleteProberDataAction implements Runnable {
       final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(domain);
 
       int entitiesDeleted =
-          tm()
-              .transact(
+          tm().transact(
                   () -> {
                     // This ancestor query selects all descendant HistoryEntries, BillingEvents,
-                    // PollMessages,
-                    // and TLD-specific entities, as well as the domain itself.
+                    // PollMessages, and TLD-specific entities, as well as the domain itself.
                     List<Key<Object>> domainAndDependentKeys =
-                        ofy().load().ancestor(domainKey).keys().list();
+                        auditedOfy().load().ancestor(domainKey).keys().list();
                     ImmutableSet<Key<?>> allKeys =
                         new ImmutableSet.Builder<Key<?>>()
                             .add(fki)
@@ -237,41 +406,12 @@ public class DeleteProberDataAction implements Runnable {
                     if (isDryRun) {
                       logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
                     } else {
-                      ofy().deleteWithoutBackup().keys(allKeys);
+                      auditedOfy().deleteWithoutBackup().keys(allKeys);
                     }
                     return allKeys.size();
                   });
       getContext().incrementCounter("domains hard-deleted");
       getContext().incrementCounter("total entities hard-deleted", entitiesDeleted);
     }
-
-    private void softDeleteDomain(final DomainBase domain) {
-      tm().transactNew(
-              () -> {
-                DomainBase deletedDomain =
-                    domain
-                        .asBuilder()
-                        .setDeletionTime(tm().getTransactionTime())
-                        .setStatusValues(null)
-                        .build();
-                HistoryEntry historyEntry =
-                    new HistoryEntry.Builder()
-                        .setParent(domain)
-                        .setType(DOMAIN_DELETE)
-                        .setModificationTime(tm().getTransactionTime())
-                        .setBySuperuser(true)
-                        .setReason("Deletion of prober data")
-                        .setClientId(registryAdminClientId)
-                        .build();
-                // Note that we don't bother handling grace periods, billing events, pending
-                // transfers,
-                // poll messages, or auto-renews because these will all be hard-deleted the next
-                // time the
-                // mapreduce runs anyway.
-                ofy().save().entities(deletedDomain, historyEntry);
-                updateForeignKeyIndexDeletionTime(deletedDomain);
-                dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
-              });
-    }
   }
 }

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -21,9 +21,12 @@ import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createChildEntityInput;
 import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
 import static google.registry.model.domain.Period.Unit.YEARS;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
+import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
 import static google.registry.util.CollectionUtils.union;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -38,10 +41,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Range;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.NullInput;
-import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
@@ -49,11 +50,12 @@ import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.common.Cursor;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.Period;
-import google.registry.model.registry.Registry;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
-import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -91,30 +93,88 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
 
   @Override
   public void run() {
-    Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
     DateTime executeTime = clock.nowUtc();
-    DateTime persistedCursorTime = (cursor == null ? START_OF_TIME : cursor.getCursorTime());
+    DateTime persistedCursorTime =
+        transactIfJpaTm(
+            () ->
+                tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
+                    .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                    .getCursorTime());
     DateTime cursorTime = cursorTimeParam.orElse(persistedCursorTime);
     checkArgument(
-        cursorTime.isBefore(executeTime),
-        "Cursor time must be earlier than execution time.");
+        cursorTime.isBefore(executeTime), "Cursor time must be earlier than execution time.");
     logger.atInfo().log(
         "Running Recurring billing event expansion for billing time range [%s, %s).",
         cursorTime, executeTime);
-    mrRunner
-        .setJobName("Expand Recurring billing events into synthetic OneTime events.")
-        .setModuleName("backend")
-        .runMapreduce(
-            new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
-            new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
-            // Add an extra shard that maps over a null recurring event (see the mapper for why).
-            ImmutableList.of(
-                new NullInput<>(),
-                createChildEntityInput(
-                    ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
-        .sendLinkToMapreduceConsole(response);
-  }
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Expand Recurring billing events into synthetic OneTime events.")
+          .setModuleName("backend")
+          .runMapreduce(
+              new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
+              new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
+              // Add an extra shard that maps over a null recurring event (see the mapper for why).
+              ImmutableList.of(
+                  new NullInput<>(),
+                  createChildEntityInput(
+                      ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      int numBillingEventsSaved =
+          jpaTm()
+              .transact(
+                  () ->
+                      jpaTm()
+                          .query(
+                              "FROM BillingRecurrence "
+                                  + "WHERE eventTime <= :executeTime "
+                                  + "AND eventTime < recurrenceEndTime "
+                                  + "ORDER BY id ASC",
+                              Recurring.class)
+                          .setParameter("executeTime", executeTime)
+                          // Need to get a list from the transaction and then convert it to a stream
+                          // for further processing. If we get a stream directly, each elements gets
+                          // processed downstream eagerly but Hibernate returns a
+                          // ScrollableResultsIterator that cannot be advanced outside the
+                          // transaction, resulting in an exception.
+                          .getResultList())
+              .stream()
+              .map(
+                  recurring ->
+                      jpaTm()
+                          .transact(
+                              () ->
+                                  expandBillingEvent(recurring, executeTime, cursorTime, isDryRun)))
+              .reduce(0, Integer::sum);
 
+      if (!isDryRun) {
+        logger.atInfo().log("Saved OneTime billing events.", numBillingEventsSaved);
+      } else {
+        logger.atInfo().log("Generated OneTime billing events (dry run).", numBillingEventsSaved);
+      }
+      logger.atInfo().log(
+          "Recurring event expansion %s complete for billing event range [%s, %s).",
+          isDryRun ? "(dry run) " : "", cursorTime, executeTime);
+      tm().transact(
+              () -> {
+                // Check for the unlikely scenario where the cursor has been altered during the
+                // expansion.
+                DateTime currentCursorTime =
+                    tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
+                        .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                        .getCursorTime();
+                if (!currentCursorTime.equals(persistedCursorTime)) {
+                  throw new IllegalStateException(
+                      String.format(
+                          "Current cursor position %s does not match persisted cursor position %s.",
+                          currentCursorTime, persistedCursorTime));
+                }
+                if (!isDryRun) {
+                  tm().put(Cursor.createGlobal(RECURRING_BILLING, executeTime));
+                }
+              });
+    }
+  }
   /** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */
   public static class ExpandRecurringBillingEventsMapper
       extends Mapper<Recurring, DateTime, DateTime> {
@@ -153,98 +213,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       try {
         numBillingEventsSaved =
             tm().transactNew(
-                    () -> {
-                      ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
-                          new ImmutableSet.Builder<>();
-                      final Registry tld =
-                          Registry.get(getTldFromDomainName(recurring.getTargetId()));
-
-                      // Determine the complete set of times at which this recurring event should
-                      // occur (up to and including the runtime of the mapreduce).
-                      Iterable<DateTime> eventTimes =
-                          recurring
-                              .getRecurrenceTimeOfYear()
-                              .getInstancesInRange(
-                                  Range.closed(
-                                      recurring.getEventTime(),
-                                      earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
-
-                      // Convert these event times to billing times
-                      final ImmutableSet<DateTime> billingTimes =
-                          getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
-
-                      Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
-                      Iterable<OneTime> oneTimesForDomain =
-                          ofy().load().type(OneTime.class).ancestor(domainKey);
-
-                      // Determine the billing times that already have OneTime events persisted.
-                      ImmutableSet<DateTime> existingBillingTimes =
-                          getExistingBillingTimes(oneTimesForDomain, recurring);
-
-                      ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
-                          new ImmutableSet.Builder<>();
-                      // Create synthetic OneTime events for all billing times that do not yet have
-                      // an event persisted.
-                      for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
-                        // Construct a new HistoryEntry that parents over the OneTime
-                        HistoryEntry historyEntry =
-                            new HistoryEntry.Builder()
-                                .setBySuperuser(false)
-                                .setClientId(recurring.getClientId())
-                                .setModificationTime(tm().getTransactionTime())
-                                .setParent(domainKey)
-                                .setPeriod(Period.create(1, YEARS))
-                                .setReason(
-                                    "Domain autorenewal by ExpandRecurringBillingEventsAction")
-                                .setRequestedByRegistrar(false)
-                                .setType(DOMAIN_AUTORENEW)
-                                // Don't write a domain transaction record if the recurrence was
-                                // ended prior to the billing time (i.e. a domain was deleted
-                                // during the autorenew grace period).
-                                .setDomainTransactionRecords(
-                                    recurring.getRecurrenceEndTime().isBefore(billingTime)
-                                        ? ImmutableSet.of()
-                                        : ImmutableSet.of(
-                                            DomainTransactionRecord.create(
-                                                tld.getTldStr(),
-                                                // We report this when the autorenew grace period
-                                                // ends
-                                                billingTime,
-                                                TransactionReportField.netRenewsFieldFromYears(1),
-                                                1)))
-                                .build();
-                        historyEntriesBuilder.add(historyEntry);
-
-                        DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
-                        // Determine the cost for a one-year renewal.
-                        Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
-                        syntheticOneTimesBuilder.add(
-                            new OneTime.Builder()
-                                .setBillingTime(billingTime)
-                                .setClientId(recurring.getClientId())
-                                .setCost(renewCost)
-                                .setEventTime(eventTime)
-                                .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
-                                .setParent(historyEntry)
-                                .setPeriodYears(1)
-                                .setReason(recurring.getReason())
-                                .setSyntheticCreationTime(executeTime)
-                                .setCancellationMatchingBillingEvent(recurring.createVKey())
-                                .setTargetId(recurring.getTargetId())
-                                .build());
-                      }
-                      Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
-                      Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
-                      if (!isDryRun) {
-                        ImmutableSet<ImmutableObject> entitiesToSave =
-                            new ImmutableSet.Builder<ImmutableObject>()
-                                .addAll(historyEntries)
-                                .addAll(syntheticOneTimes)
-                                .build();
-                        ofy().save().entities(entitiesToSave).now();
-                      }
-                      return syntheticOneTimes.size();
-                    });
+                    () -> expandBillingEvent(recurring, executeTime, cursorTime, isDryRun));
       } catch (Throwable t) {
         getContext().incrementCounter("error: " + t.getClass().getSimpleName());
         getContext().incrementCounter(ERROR_COUNTER);
@@ -256,45 +225,12 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       if (!isDryRun) {
         getContext().incrementCounter("Saved OneTime billing events", numBillingEventsSaved);
       } else {
-        getContext().incrementCounter(
-            "Generated OneTime billing events (dry run)", numBillingEventsSaved);
+        getContext()
+            .incrementCounter("Generated OneTime billing events (dry run)", numBillingEventsSaved);
       }
     }
-
-    /**
-     * Filters a set of {@link DateTime}s down to event times that are in scope for a particular
-     * mapreduce run, given the cursor time and the mapreduce execution time.
-     */
-    private ImmutableSet<DateTime> getBillingTimesInScope(
-        Iterable<DateTime> eventTimes,
-        DateTime cursorTime,
-        DateTime executeTime,
-        final Registry tld) {
-      return Streams.stream(eventTimes)
-          .map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
-          .filter(Range.closedOpen(cursorTime, executeTime))
-          .collect(toImmutableSet());
-    }
-
-    /**
-     * Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted
-     * for a given recurring billing event.
-     */
-    private ImmutableSet<DateTime> getExistingBillingTimes(
-        Iterable<BillingEvent.OneTime> oneTimesForDomain,
-        final BillingEvent.Recurring recurringEvent) {
-      return Streams.stream(oneTimesForDomain)
-          .filter(
-              billingEvent ->
-                  recurringEvent
-                      .createVKey()
-                      .equals(billingEvent.getCancellationMatchingBillingEvent()))
-          .map(OneTime::getBillingTime)
-          .collect(toImmutableSet());
-    }
   }
 
-
   /**
    * "Reducer" to advance the cursor after all map jobs have been completed. The NullInput into the
    * mapper will cause the mapper to emit one timestamp pair (current cursor and execution time),
@@ -327,7 +263,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
           isDryRun ? "(dry run) " : "", cursorTime, executionTime);
       tm().transact(
               () -> {
-                Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
+                Cursor cursor =
+                    auditedOfy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
                 DateTime currentCursorTime =
                     (cursor == null ? START_OF_TIME : cursor.getCursorTime());
                 if (!currentCursorTime.equals(expectedPersistedCursorTime)) {
@@ -342,4 +279,135 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
               });
     }
   }
+
+  private static int expandBillingEvent(
+      Recurring recurring, DateTime executeTime, DateTime cursorTime, boolean isDryRun) {
+    ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder = new ImmutableSet.Builder<>();
+    final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
+
+    // Determine the complete set of times at which this recurring event should
+    // occur (up to and including the runtime of the mapreduce).
+    Iterable<DateTime> eventTimes =
+        recurring
+            .getRecurrenceTimeOfYear()
+            .getInstancesInRange(
+                Range.closed(
+                    recurring.getEventTime(),
+                    earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
+
+    // Convert these event times to billing times
+    final ImmutableSet<DateTime> billingTimes =
+        getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
+
+    VKey<DomainBase> domainKey =
+        VKey.create(
+            DomainBase.class, recurring.getDomainRepoId(), recurring.getParentKey().getParent());
+    Iterable<OneTime> oneTimesForDomain;
+    if (tm().isOfy()) {
+      oneTimesForDomain = auditedOfy().load().type(OneTime.class).ancestor(domainKey.getOfyKey());
+    } else {
+      oneTimesForDomain =
+          tm().createQueryComposer(OneTime.class)
+              .where("domainRepoId", EQ, recurring.getDomainRepoId())
+              .list();
+    }
+
+    // Determine the billing times that already have OneTime events persisted.
+    ImmutableSet<DateTime> existingBillingTimes =
+        getExistingBillingTimes(oneTimesForDomain, recurring);
+
+    ImmutableSet.Builder<DomainHistory> historyEntriesBuilder = new ImmutableSet.Builder<>();
+    // Create synthetic OneTime events for all billing times that do not yet have
+    // an event persisted.
+    for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
+      // Construct a new HistoryEntry that parents over the OneTime
+      DomainHistory historyEntry =
+          new DomainHistory.Builder()
+              .setBySuperuser(false)
+              .setRegistrarId(recurring.getRegistrarId())
+              .setModificationTime(tm().getTransactionTime())
+              .setDomain(tm().loadByKey(domainKey))
+              .setPeriod(Period.create(1, YEARS))
+              .setReason("Domain autorenewal by ExpandRecurringBillingEventsAction")
+              .setRequestedByRegistrar(false)
+              .setType(DOMAIN_AUTORENEW)
+              // Don't write a domain transaction record if the recurrence was
+              // ended prior to the billing time (i.e. a domain was deleted
+              // during the autorenew grace period).
+              .setDomainTransactionRecords(
+                  recurring.getRecurrenceEndTime().isBefore(billingTime)
+                      ? ImmutableSet.of()
+                      : ImmutableSet.of(
+                          DomainTransactionRecord.create(
+                              tld.getTldStr(),
+                              // We report this when the autorenew grace period
+                              // ends
+                              billingTime,
+                              TransactionReportField.netRenewsFieldFromYears(1),
+                              1)))
+              .build();
+      historyEntriesBuilder.add(historyEntry);
+
+      DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
+      // Determine the cost for a one-year renewal.
+      Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
+      syntheticOneTimesBuilder.add(
+          new OneTime.Builder()
+              .setBillingTime(billingTime)
+              .setRegistrarId(recurring.getRegistrarId())
+              .setCost(renewCost)
+              .setEventTime(eventTime)
+              .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
+              .setParent(historyEntry)
+              .setPeriodYears(1)
+              .setReason(recurring.getReason())
+              .setSyntheticCreationTime(executeTime)
+              .setCancellationMatchingBillingEvent(recurring.createVKey())
+              .setTargetId(recurring.getTargetId())
+              .build());
+    }
+    Set<DomainHistory> historyEntries = historyEntriesBuilder.build();
+    Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
+    if (!isDryRun) {
+      ImmutableSet<ImmutableObject> entitiesToSave =
+          new ImmutableSet.Builder<ImmutableObject>()
+              .addAll(historyEntries)
+              .addAll(syntheticOneTimes)
+              .build();
+      tm().putAll(entitiesToSave);
+    }
+    return syntheticOneTimes.size();
+  }
+
+  /**
+   * Filters a set of {@link DateTime}s down to event times that are in scope for a particular
+   * mapreduce run, given the cursor time and the mapreduce execution time.
+   */
+  protected static ImmutableSet<DateTime> getBillingTimesInScope(
+      Iterable<DateTime> eventTimes,
+      DateTime cursorTime,
+      DateTime executeTime,
+      final Registry tld) {
+    return Streams.stream(eventTimes)
+        .map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
+        .filter(Range.closedOpen(cursorTime, executeTime))
+        .collect(toImmutableSet());
+  }
+
+  /**
+   * Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted for a
+   * given recurring billing event.
+   */
+  private static ImmutableSet<DateTime> getExistingBillingTimes(
+      Iterable<BillingEvent.OneTime> oneTimesForDomain,
+      final BillingEvent.Recurring recurringEvent) {
+    return Streams.stream(oneTimesForDomain)
+        .filter(
+            billingEvent ->
+                recurringEvent
+                    .createVKey()
+                    .equals(billingEvent.getCancellationMatchingBillingEvent()))
+        .map(OneTime::getBillingTime)
+        .collect(toImmutableSet());
+  }
 }

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -23,9 +23,10 @@ import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
+import static google.registry.model.EppResourceUtils.getLinkedDomainKeys;
 import static google.registry.model.EppResourceUtils.isActive;
 import static google.registry.model.EppResourceUtils.isDeleted;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.latestOf;
 import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.SECONDS;
@@ -44,7 +45,6 @@ import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.batch.AsyncTaskMetrics.OperationResult;
 import google.registry.dns.DnsQueue;
 import google.registry.mapreduce.MapreduceRunner;
@@ -64,11 +64,13 @@ import google.registry.util.SystemClock;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.logging.Level;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Named;
+import org.apache.http.HttpStatus;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
@@ -86,6 +88,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
   @Inject Clock clock;
   @Inject MapreduceRunner mrRunner;
   @Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue;
+
+  @Inject DnsQueue dnsQueue;
   @Inject RequestStatusChecker requestStatusChecker;
   @Inject Response response;
   @Inject Retrier retrier;
@@ -123,7 +127,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     }
 
     ImmutableList.Builder<DnsRefreshRequest> requestsBuilder = new ImmutableList.Builder<>();
-    ImmutableList.Builder<Key<HostResource>> hostKeys = new ImmutableList.Builder<>();
+    ImmutableList.Builder<VKey<HostResource>> hostKeys = new ImmutableList.Builder<>();
     final List<DnsRefreshRequest> requestsToDelete = new ArrayList<>();
 
     for (TaskHandle task : tasks) {
@@ -153,7 +157,39 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     } else {
       logger.atInfo().log(
           "Processing asynchronous DNS refresh for renamed hosts: %s", hostKeys.build());
-      runMapreduce(refreshRequests, lock);
+      if (tm().isOfy()) {
+        runMapreduce(refreshRequests, lock);
+      } else {
+        try {
+          refreshRequests.stream()
+              .flatMap(
+                  request ->
+                      getLinkedDomainKeys(request.hostKey(), request.lastUpdateTime(), null)
+                          .stream())
+              .distinct()
+              .map(domainKey -> tm().transact(() -> tm().loadByKey(domainKey).getDomainName()))
+              .forEach(
+                  domainName -> {
+                    retrier.callWithRetry(
+                        () -> dnsQueue.addDomainRefreshTask(domainName),
+                        TransientFailureException.class);
+                    logger.atInfo().log("Enqueued DNS refresh for domain '%s'.", domainName);
+                  });
+          deleteTasksWithRetry(
+              refreshRequests,
+              getQueue(QUEUE_ASYNC_HOST_RENAME),
+              asyncTaskMetrics,
+              retrier,
+              OperationResult.SUCCESS);
+        } catch (Throwable t) {
+          String message = "Error refreshing DNS on host rename.";
+          logger.atSevere().withCause(t).log(message);
+          response.setPayload(message);
+          response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR);
+        } finally {
+          lock.get().release();
+        }
+      }
     }
   }
 
@@ -204,10 +240,10 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
         emit(true, true);
         return;
       }
-      Key<HostResource> referencingHostKey = null;
+      VKey<HostResource> referencingHostKey = null;
       for (DnsRefreshRequest request : refreshRequests) {
         if (isActive(domain, request.lastUpdateTime())
-            && domain.getNameservers().contains(VKey.from(request.hostKey()))) {
+            && domain.getNameservers().contains(request.hostKey())) {
           referencingHostKey = request.hostKey();
           break;
         }
@@ -293,7 +329,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
 
     private static final long serialVersionUID = 1772812852271288622L;
 
-    abstract Key<HostResource> hostKey();
+    abstract VKey<HostResource> hostKey();
+
     abstract DateTime lastUpdateTime();
     abstract DateTime requestedTime();
     abstract boolean isRefreshNeeded();
@@ -301,7 +338,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
 
     @AutoValue.Builder
     abstract static class Builder {
-      abstract Builder setHostKey(Key<HostResource> hostKey);
+      abstract Builder setHostKey(VKey<HostResource> hostKey);
+
       abstract Builder setLastUpdateTime(DateTime lastUpdateTime);
       abstract Builder setRequestedTime(DateTime requestedTime);
       abstract Builder setIsRefreshNeeded(boolean isRefreshNeeded);
@@ -314,10 +352,11 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
      */
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<HostResource> hostKey =
-          Key.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+      VKey<HostResource> hostKey =
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
-          checkNotNull(ofy().load().key(hostKey).now(), "Host to refresh doesn't exist");
+          tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
+              .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));
       boolean isHostDeleted =
           isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp()));
       if (isHostDeleted) {

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -16,7 +16,6 @@ package google.registry.batch;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
@@ -29,15 +28,16 @@ import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
-import google.registry.model.registry.RegistryLockDao;
+import google.registry.model.tld.RegistryLockDao;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.schema.domain.RegistryLock;
 import google.registry.tools.DomainLockUtils;
 import google.registry.util.DateTimeUtils;
 import google.registry.util.EmailMessage;
@@ -125,6 +125,7 @@ public class RelockDomainAction implements Runnable {
     response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
 
     // nb: DomainLockUtils relies on the JPA transaction being the outermost transaction
+    // if we have Datastore as the primary DB (if SQL is the primary DB, it's irrelevant)
     jpaTm().transact(() -> tm().transact(this::relockDomain));
   }
 
@@ -139,12 +140,8 @@ public class RelockDomainAction implements Runnable {
                       new IllegalArgumentException(
                           String.format("Unknown revision ID %d", oldUnlockRevisionId)));
       domain =
-          ofy()
-              .load()
-              .type(DomainBase.class)
-              .id(oldLock.getRepoId())
-              .now()
-              .cloneProjectedAtTime(jpaTm().getTransactionTime());
+          tm().loadByKey(VKey.create(DomainBase.class, oldLock.getRepoId()))
+              .cloneProjectedAtTime(tm().getTransactionTime());
     } catch (Throwable t) {
       handleTransientFailure(Optional.ofNullable(oldLock), t);
       return;
@@ -206,11 +203,11 @@ public class RelockDomainAction implements Runnable {
         "Domain %s has a pending transfer.",
         domainName);
     checkArgument(
-        domain.getCurrentSponsorClientId().equals(oldLock.getRegistrarId()),
+        domain.getCurrentSponsorRegistrarId().equals(oldLock.getRegistrarId()),
         "Domain %s has been transferred from registrar %s to registrar %s since the unlock.",
         domainName,
         oldLock.getRegistrarId(),
-        domain.getCurrentSponsorClientId());
+        domain.getCurrentSponsorRegistrarId());
   }
 
   private void handleNonRetryableFailure(RegistryLock oldLock, Throwable t) {
@@ -296,7 +293,7 @@ public class RelockDomainAction implements Runnable {
 
   private ImmutableSet<InternetAddress> getEmailRecipients(String registrarId) {
     Registrar registrar =
-        Registrar.loadByClientIdCached(registrarId)
+        Registrar.loadByRegistrarIdCached(registrarId)
             .orElseThrow(
                 () ->
                     new IllegalStateException(String.format("Unknown registrar %s", registrarId)));
@@ -316,7 +313,7 @@ public class RelockDomainAction implements Runnable {
         builder.add(new InternetAddress(registryLockEmailAddress));
       } catch (AddressException e) {
         // This shouldn't stop any other emails going out, so swallow it
-        logger.atWarning().log("Invalid email address %s", registryLockEmailAddress);
+        logger.atWarning().log("Invalid email address '%s'.", registryLockEmailAddress);
       }
     }
     return builder.build();

core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java

@@ -15,7 +15,7 @@
 package google.registry.batch;
 
 import static google.registry.mapreduce.MapreduceRunner.PARAM_FAST;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.appengine.tools.mapreduce.Mapper;
@@ -24,6 +24,7 @@ import com.googlecode.objectify.Key;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -54,6 +55,9 @@ import javax.inject.Inject;
     service = Action.Service.BACKEND,
     path = "/_dr/task/resaveAllEppResources",
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+// No longer needed in SQL. Subject to future removal.
+@Deprecated
+@DeleteAfterMigration
 public class ResaveAllEppResourcesAction implements Runnable {
 
   @Inject MapreduceRunner mrRunner;
@@ -104,13 +108,13 @@ public class ResaveAllEppResourcesAction implements Runnable {
       boolean resaved =
           tm().transact(
                   () -> {
-                    EppResource originalResource = ofy().load().key(resourceKey).now();
+                    EppResource originalResource = auditedOfy().load().key(resourceKey).now();
                     EppResource projectedResource =
                         originalResource.cloneProjectedAtTime(tm().getTransactionTime());
                     if (isFast && originalResource.equals(projectedResource)) {
                       return false;
                     } else {
-                      ofy().save().entity(projectedResource).now();
+                      auditedOfy().save().entity(projectedResource).now();
                       return true;
                     }
                   });

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -17,15 +17,14 @@ package google.registry.batch;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Action.Method;
 import google.registry.request.Parameter;
@@ -50,7 +49,7 @@ public class ResaveEntityAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private final Key<ImmutableObject> resourceKey;
+  private final String resourceKey;
   private final DateTime requestedTime;
   private final ImmutableSortedSet<DateTime> resaveTimes;
   private final AsyncTaskEnqueuer asyncTaskEnqueuer;
@@ -58,7 +57,7 @@ public class ResaveEntityAction implements Runnable {
 
   @Inject
   ResaveEntityAction(
-      @Parameter(PARAM_RESOURCE_KEY) Key<ImmutableObject> resourceKey,
+      @Parameter(PARAM_RESOURCE_KEY) String resourceKey,
       @Parameter(PARAM_REQUESTED_TIME) DateTime requestedTime,
       @Parameter(PARAM_RESAVE_TIMES) ImmutableSet<DateTime> resaveTimes,
       AsyncTaskEnqueuer asyncTaskEnqueuer,
@@ -74,16 +73,18 @@ public class ResaveEntityAction implements Runnable {
   public void run() {
     logger.atInfo().log(
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
-    tm().transact(() -> {
-      ImmutableObject entity = ofy().load().key(resourceKey).now();
-      ofy().save().entity(
-          (entity instanceof EppResource)
-              ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime()) : entity
-      );
-      if (!resaveTimes.isEmpty()) {
-        asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
-      }
-    });
+    tm().transact(
+            () -> {
+              ImmutableObject entity = tm().loadByKey(VKey.create(resourceKey));
+              tm().put(
+                      (entity instanceof EppResource)
+                          ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
+                          : entity);
+              if (!resaveTimes.isEmpty()) {
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.create(resourceKey), requestedTime, resaveTimes);
+              }
+            });
     response.setPayload("Entity re-saved.");
   }
 }

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -0,0 +1,353 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.joda.time.DateTimeZone.UTC;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.flows.certs.CertificateChecker;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registrar.RegistrarContact.Type;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.EmailMessage;
+import google.registry.util.SendEmailService;
+import java.util.Date;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.mail.internet.AddressException;
+import javax.mail.internet.InternetAddress;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/** An action that sends notification emails to registrars whose certificates are expiring soon. */
+@Action(
+    service = Action.Service.BACKEND,
+    path = SendExpiringCertificateNotificationEmailAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class SendExpiringCertificateNotificationEmailAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
+  /**
+   * Used as an offset when storing the last notification email sent date.
+   *
+   * <p>This is used to handle edges cases when the update happens in between the day switch. For
+   * instance,if the job starts at 2:00 am every day and it finishes at 2:03 of the same day, then
+   * next day at 2am, the date difference will be less than a day, which will lead to the date
+   * difference between two successive email sent date being the expected email interval days + 1;
+   */
+  protected static final Duration UPDATE_TIME_OFFSET = Duration.standardMinutes(10);
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd");
+
+  private final CertificateChecker certificateChecker;
+  private final String expirationWarningEmailBodyText;
+  private final SendEmailService sendEmailService;
+  private final String expirationWarningEmailSubjectText;
+  private final InternetAddress gSuiteOutgoingEmailAddress;
+  private final Response response;
+
+  @Inject
+  public SendExpiringCertificateNotificationEmailAction(
+      @Config("expirationWarningEmailBodyText") String expirationWarningEmailBodyText,
+      @Config("expirationWarningEmailSubjectText") String expirationWarningEmailSubjectText,
+      @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
+      SendEmailService sendEmailService,
+      CertificateChecker certificateChecker,
+      Response response) {
+    this.certificateChecker = certificateChecker;
+    this.expirationWarningEmailSubjectText = expirationWarningEmailSubjectText;
+    this.sendEmailService = sendEmailService;
+    this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress;
+    this.expirationWarningEmailBodyText = expirationWarningEmailBodyText;
+    this.response = response;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int numEmailsSent = sendNotificationEmails();
+      String message =
+          String.format(
+              "Done. Sent %d expiring certificate notification emails in total.", numEmailsSent);
+      logger.atInfo().log(message);
+      response.setStatus(SC_OK);
+      response.setPayload(message);
+    } catch (Exception e) {
+      logger.atWarning().withCause(e).log(
+          "Exception thrown when sending expiring certificate notification emails.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(String.format("Exception thrown with cause: %s", e));
+    }
+  }
+
+  /**
+   * Returns a list of registrars that should receive expiring notification emails.
+   *
+   * <p>There are two certificates that should be considered (the main certificate and failOver
+   * certificate). The registrars should receive notifications if one of the certificate checks
+   * returns true.
+   */
+  @VisibleForTesting
+  ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
+    logger.atInfo().log(
+        "Getting a list of registrars that should receive expiring notification emails.");
+    return Streams.stream(Registrar.loadAllCached())
+        .map(
+            registrar ->
+                RegistrarInfo.create(
+                    registrar,
+                    registrar.getClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringCertNotificationSentDate(),
+                            registrar.getClientCertificate().get()),
+                    registrar.getFailoverClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringFailoverCertNotificationSentDate(),
+                            registrar.getFailoverClientCertificate().get())))
+        .filter(
+            registrarInfo ->
+                registrarInfo.isCertExpiring() || registrarInfo.isFailOverCertExpiring())
+        .collect(toImmutableList());
+  }
+
+  /**
+   * Sends a notification email to the registrar regarding the expiring certificate and returns true
+   * if it's sent successfully.
+   */
+  @VisibleForTesting
+  boolean sendNotificationEmail(
+      Registrar registrar,
+      DateTime lastExpiringCertNotificationSentDate,
+      CertificateType certificateType,
+      Optional<String> certificate) {
+    if (!certificate.isPresent()
+        || !certificateChecker.shouldReceiveExpiringNotification(
+            lastExpiringCertNotificationSentDate, certificate.get())) {
+      return false;
+    }
+    try {
+      ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      ImmutableSet<InternetAddress> ccs = getEmailAddresses(registrar, Type.ADMIN);
+      Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
+      logger.atInfo().log(
+          " %s SSL certificate of registrar '%s' will expire on %s.",
+          certificateType.getDisplayName(),
+          registrar.getRegistrarName(),
+          expirationDate.toString());
+      if (recipients.isEmpty() && ccs.isEmpty()) {
+        logger.atWarning().log(
+            "Registrar %s contains no TECH nor ADMIN email addresses to receive notification"
+                + " email.",
+            registrar.getRegistrarName());
+        return false;
+      }
+      sendEmailService.sendEmail(
+          EmailMessage.newBuilder()
+              .setFrom(gSuiteOutgoingEmailAddress)
+              .setSubject(expirationWarningEmailSubjectText)
+              .setBody(
+                  getEmailBody(
+                      registrar.getRegistrarName(),
+                      certificateType,
+                      expirationDate,
+                      registrar.getRegistrarId()))
+              .setRecipients(recipients)
+              .setCcs(ccs)
+              .build());
+      /*
+       * A duration time offset is used here to ensure that date comparison between two
+       * successive dates is always greater than 1 day. This date is set as last updated date,
+       * for applicable certificate.
+       */
+      updateLastNotificationSentDate(
+          registrar,
+          DateTime.now(UTC).minusMinutes((int) UPDATE_TIME_OFFSET.getStandardMinutes()),
+          certificateType);
+      return true;
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to send expiring certificate notification email to registrar %s.",
+              registrar.getRegistrarName()));
+    }
+  }
+
+  /** Updates the last notification sent date in database. */
+  @VisibleForTesting
+  void updateLastNotificationSentDate(
+      Registrar registrar, DateTime now, CertificateType certificateType) {
+    try {
+      tm().transact(
+              () -> {
+                Registrar.Builder newRegistrar = tm().loadByEntity(registrar).asBuilder();
+                switch (certificateType) {
+                  case PRIMARY:
+                    newRegistrar.setLastExpiringCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  case FAILOVER:
+                    newRegistrar.setLastExpiringFailoverCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  default:
+                    throw new IllegalArgumentException(
+                        String.format(
+                            "Unsupported certificate type: %s being passed in when updating "
+                                + "the last notification sent date to registrar %s.",
+                            certificateType.toString(), registrar.getRegistrarName()));
+                }
+              });
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to update the last notification sent date to Registrar %s for the %s "
+                  + "certificate.",
+              registrar.getRegistrarName(), certificateType.getDisplayName()));
+    }
+  }
+
+  /** Sends notification emails to registrars with expiring certificates. */
+  @VisibleForTesting
+  int sendNotificationEmails() {
+    int numEmailsSent = 0;
+    for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
+      Registrar registrar = registrarInfo.registrar();
+      if (registrarInfo.isCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringCertNotificationSentDate(),
+              CertificateType.PRIMARY,
+              registrar.getClientCertificate())) {
+        numEmailsSent++;
+      }
+      if (registrarInfo.isFailOverCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringFailoverCertNotificationSentDate(),
+              CertificateType.FAILOVER,
+              registrar.getFailoverClientCertificate())) {
+        numEmailsSent++;
+      }
+    }
+    return numEmailsSent;
+  }
+
+  /**
+   * Returns a list of email addresses of the registrar that should receive a notification email.
+   */
+  @VisibleForTesting
+  ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
+    ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
+    ImmutableSet.Builder<InternetAddress> recipientEmails = new ImmutableSet.Builder<>();
+    for (RegistrarContact contact : contacts) {
+      try {
+        recipientEmails.add(new InternetAddress(contact.getEmailAddress()));
+      } catch (AddressException e) {
+        logger.atWarning().withCause(e).log(
+            "Registrar Contact email address %s of Registrar %s is invalid; skipping.",
+            contact.getEmailAddress(), registrar.getRegistrarName());
+      }
+    }
+    return recipientEmails.build();
+  }
+
+  /**
+   * Generates email content by taking registrar name, certificate type and expiration date as
+   * parameters.
+   */
+  @VisibleForTesting
+  @SuppressWarnings("lgtm[java/dereferenced-value-may-be-null]")
+  String getEmailBody(
+      String registrarName, CertificateType type, Date expirationDate, String registrarId) {
+    checkArgumentNotNull(expirationDate, "Expiration date cannot be null");
+    checkArgumentNotNull(type, "Certificate type cannot be null");
+    checkArgumentNotNull(registrarId, "Registrar Id cannot be null");
+    return String.format(
+        expirationWarningEmailBodyText,
+        registrarName,
+        type.getDisplayName(),
+        DATE_FORMATTER.print(new DateTime(expirationDate)),
+        registrarId);
+  }
+
+  /**
+   * Certificate types for X509Certificate.
+   *
+   * <p><b>Note:</b> These types are only used to indicate the type of expiring certificate in
+   * notification emails.
+   */
+  protected enum CertificateType {
+    PRIMARY("primary"),
+    FAILOVER("fail-over");
+
+    private final String displayName;
+
+    CertificateType(String displayName) {
+      this.displayName = displayName;
+    }
+
+    public String getDisplayName() {
+      return displayName;
+    }
+  }
+
+  @AutoValue
+  public abstract static class RegistrarInfo {
+
+    static RegistrarInfo create(
+        Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
+      return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(
+          registrar, isCertExpiring, isFailOverCertExpiring);
+    }
+
+    public abstract Registrar registrar();
+
+    public abstract boolean isCertExpiring();
+
+    public abstract boolean isFailOverCertExpiring();
+  }
+}

core/src/main/java/google/registry/batch/WipeOutCloudSqlAction.java

@@ -19,19 +19,21 @@ import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Retrier;
 import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.function.Supplier;
 import javax.inject.Inject;
-import org.flywaydb.core.api.FlywayException;
 
 /**
  * Wipes out all Cloud SQL data in a Nomulus GCP environment.
@@ -46,22 +48,18 @@ import org.flywaydb.core.api.FlywayException;
 public class WipeOutCloudSqlAction implements Runnable {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  // As a short-lived class, hardcode allowed projects here instead of using config files.
-  private static final ImmutableSet<String> ALLOWED_PROJECTS =
-      ImmutableSet.of("domain-registry-qa");
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
 
-  private final String projectId;
   private final Supplier<Connection> connectionSupplier;
   private final Response response;
   private final Retrier retrier;
 
   @Inject
   WipeOutCloudSqlAction(
-      @Config("projectId") String projectId,
       @SchemaManagerConnection Supplier<Connection> connectionSupplier,
       Response response,
       Retrier retrier) {
-    this.projectId = projectId;
     this.connectionSupplier = connectionSupplier;
     this.response = response;
     this.retrier = retrier;
@@ -71,28 +69,93 @@ public class WipeOutCloudSqlAction implements Runnable {
   public void run() {
     response.setContentType(PLAIN_TEXT_UTF_8);
 
-    if (!ALLOWED_PROJECTS.contains(projectId)) {
+    if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
       response.setStatus(SC_FORBIDDEN);
-      response.setPayload("Wipeout is not allowed in " + projectId);
+      response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
       return;
     }
 
     try {
       retrier.callWithRetry(
           () -> {
-            try (Connection conn = connectionSupplier.get();
-                Statement statement = conn.createStatement()) {
-              statement.execute("drop owned by schema_deployer;");
+            try (Connection conn = connectionSupplier.get()) {
+              dropAllTables(conn, listTables(conn));
+              dropAllSequences(conn, listSequences(conn));
             }
             return null;
           },
-          e -> !(e instanceof FlywayException));
+          e -> !(e instanceof SQLException));
       response.setStatus(SC_OK);
-      response.setPayload("Wiped out Cloud SQL in " + projectId);
+      response.setPayload("Wiped out Cloud SQL in " + RegistryEnvironment.get());
     } catch (RuntimeException e) {
       logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
+      response.setPayload("Failed to wipe out Cloud SQL in " + RegistryEnvironment.get());
+    }
+  }
+
+  /** Returns a list of all tables in the public schema of a Postgresql database. */
+  static ImmutableList<String> listTables(Connection connection) throws SQLException {
+    try (ResultSet resultSet =
+        connection.getMetaData().getTables(null, null, null, new String[] {"TABLE"})) {
+      ImmutableList.Builder<String> tables = new ImmutableList.Builder<>();
+      while (resultSet.next()) {
+        String schema = resultSet.getString("TABLE_SCHEM");
+        if (schema == null || !schema.equalsIgnoreCase("public")) {
+          continue;
+        }
+        String tableName = resultSet.getString("TABLE_NAME");
+        tables.add("public.\"" + tableName + "\"");
+      }
+      return tables.build();
+    }
+  }
+
+  static void dropAllTables(Connection conn, ImmutableList<String> tables) throws SQLException {
+    if (tables.isEmpty()) {
+      return;
+    }
+
+    try (Statement statement = conn.createStatement()) {
+      for (String table : tables) {
+        statement.addBatch(String.format("DROP TABLE IF EXISTS %s CASCADE;", table));
+      }
+      for (int code : statement.executeBatch()) {
+        if (code == Statement.EXECUTE_FAILED) {
+          throw new RuntimeException("Failed to drop some tables. Please check.");
+        }
+      }
+    }
+  }
+
+  /** Returns a list of all sequences in a Postgresql database. */
+  static ImmutableList<String> listSequences(Connection conn) throws SQLException {
+    try (Statement statement = conn.createStatement();
+        ResultSet resultSet =
+            statement.executeQuery("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';")) {
+      ImmutableList.Builder<String> sequences = new ImmutableList.Builder<>();
+      while (resultSet.next()) {
+        sequences.add('\"' + resultSet.getString(1) + '\"');
+      }
+      return sequences.build();
+    }
+  }
+
+  static void dropAllSequences(Connection conn, ImmutableList<String> sequences)
+      throws SQLException {
+    if (sequences.isEmpty()) {
+      return;
+    }
+
+    try (Statement statement = conn.createStatement()) {
+      for (String sequence : sequences) {
+        statement.addBatch(String.format("DROP SEQUENCE IF EXISTS %s CASCADE;", sequence));
+      }
+      for (int code : statement.executeBatch()) {
+        if (code == Statement.EXECUTE_FAILED) {
+          throw new RuntimeException("Failed to drop some sequences. Please check.");
+        }
+      }
     }
   }
 }

core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java

@@ -0,0 +1,138 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.contact.ContactHistory;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * An action that wipes out Personal Identifiable Information (PII) fields of {@link ContactHistory}
+ * entities.
+ *
+ * <p>ContactHistory entities should be retained in the database for only certain amount of time.
+ * This periodic wipe out action only applies to SQL.
+ */
+@Action(
+    service = Service.BACKEND,
+    path = WipeOutContactHistoryPiiAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeOutContactHistoryPiiAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/wipeOutContactHistoryPii";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private final Clock clock;
+  private final Response response;
+  private final int minMonthsBeforeWipeOut;
+  private final int wipeOutQueryBatchSize;
+
+  @Inject
+  public WipeOutContactHistoryPiiAction(
+      Clock clock,
+      @Config("minMonthsBeforeWipeOut") int minMonthsBeforeWipeOut,
+      @Config("wipeOutQueryBatchSize") int wipeOutQueryBatchSize,
+      Response response) {
+    this.clock = clock;
+    this.response = response;
+    this.minMonthsBeforeWipeOut = minMonthsBeforeWipeOut;
+    this.wipeOutQueryBatchSize = wipeOutQueryBatchSize;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int totalNumOfWipedEntities = 0;
+      DateTime wipeOutTime = clock.nowUtc().minusMonths(minMonthsBeforeWipeOut);
+      logger.atInfo().log(
+          "About to wipe out all PII of contact history entities prior to %s.", wipeOutTime);
+
+      int numOfWipedEntities = 0;
+      do {
+        numOfWipedEntities =
+            jpaTm()
+                .transact(
+                    () ->
+                        wipeOutContactHistoryData(
+                            getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime)));
+        totalNumOfWipedEntities += numOfWipedEntities;
+      } while (numOfWipedEntities > 0);
+      String msg =
+          String.format(
+              "Done. Wiped out PII of %d ContactHistory entities in total.",
+              totalNumOfWipedEntities);
+      logger.atInfo().log(msg);
+      response.setPayload(msg);
+      response.setStatus(SC_OK);
+
+    } catch (Exception e) {
+      logger.atSevere().withCause(e).log(
+          "Exception thrown during the process of wiping out contact history PII.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(
+          String.format(
+              "Exception thrown during the process of wiping out contact history PII with cause"
+                  + ": %s",
+              e));
+    }
+  }
+
+  /**
+   * Returns a stream of up to {@link #wipeOutQueryBatchSize} {@link ContactHistory} entities
+   * containing PII that are prior to @param wipeOutTime.
+   */
+  @VisibleForTesting
+  Stream<ContactHistory> getNextContactHistoryEntitiesWithPiiBatch(DateTime wipeOutTime) {
+    // email is one of the required fields in EPP, meaning it's initially not null.
+    // Therefore, checking if it's null is one way to avoid processing contact history entities
+    // that have been processed previously. Refer to RFC 5733 for more information.
+    return jpaTm()
+        .query(
+            "FROM ContactHistory WHERE modificationTime < :wipeOutTime " + "AND email IS NOT NULL",
+            ContactHistory.class)
+        .setParameter("wipeOutTime", wipeOutTime)
+        .setMaxResults(wipeOutQueryBatchSize)
+        .getResultStream();
+  }
+
+  /** Wipes out the PII of each of the {@link ContactHistory} entities in the stream. */
+  @VisibleForTesting
+  int wipeOutContactHistoryData(Stream<ContactHistory> contactHistoryEntities) {
+    AtomicInteger numOfEntities = new AtomicInteger(0);
+    contactHistoryEntities.forEach(
+        contactHistoryEntity -> {
+          jpaTm().update(contactHistoryEntity.asBuilder().wipeOutPii().build());
+          numOfEntities.incrementAndGet();
+        });
+    logger.atInfo().log(
+        "Wiped out all PII fields of %d ContactHistory entities.", numOfEntities.get());
+    return numOfEntities.get();
+  }
+}

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -0,0 +1,122 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static google.registry.beam.BeamUtils.createJobName;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import javax.inject.Inject;
+
+/**
+ * Wipes out all Cloud Datastore data in a Nomulus GCP environment.
+ *
+ * <p>This class is created for the QA environment, where migration testing with production data
+ * will happen. A regularly scheduled wipeout is a prerequisite to using production data there.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/wipeOutDatastore",
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
+public class WipeoutDatastoreAction implements Runnable {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
+
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
+
+  private final String projectId;
+  private final String jobRegion;
+  private final Response response;
+  private final Dataflow dataflow;
+  private final String stagingBucketUrl;
+  private final Clock clock;
+
+  @Inject
+  WipeoutDatastoreAction(
+      @Config("projectId") String projectId,
+      @Config("defaultJobRegion") String jobRegion,
+      @Config("beamStagingBucketUrl") String stagingBucketUrl,
+      Clock clock,
+      Response response,
+      Dataflow dataflow) {
+    this.projectId = projectId;
+    this.jobRegion = jobRegion;
+    this.stagingBucketUrl = stagingBucketUrl;
+    this.clock = clock;
+    this.response = response;
+    this.dataflow = dataflow;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+
+    if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
+      response.setStatus(SC_FORBIDDEN);
+      response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
+      return;
+    }
+
+    try {
+      LaunchFlexTemplateParameter parameters =
+          new LaunchFlexTemplateParameter()
+              .setJobName(createJobName("bulk-delete-datastore-", clock))
+              .setContainerSpecGcsPath(
+                  String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+              .setParameters(
+                  ImmutableMap.of(
+                      "kindsToDelete",
+                      "*",
+                      "registryEnvironment",
+                      RegistryEnvironment.get().name()));
+      LaunchFlexTemplateResponse launchResponse =
+          dataflow
+              .projects()
+              .locations()
+              .flexTemplates()
+              .launch(
+                  projectId,
+                  jobRegion,
+                  new LaunchFlexTemplateRequest().setLaunchParameter(parameters))
+              .execute();
+      response.setStatus(SC_OK);
+      response.setPayload("Launched " + launchResponse.getJob().getName());
+    } catch (Exception e) {
+      String msg = String.format("Failed to launch %s.", PIPELINE_NAME);
+      logger.atSevere().withCause(e).log(msg);
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(msg);
+    }
+  }
+}

core/src/main/java/google/registry/beam/BeamUtils.java

@@ -14,10 +14,14 @@
 
 package google.registry.beam;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.io.Resources;
+import google.registry.util.Clock;
 import google.registry.util.ResourceUtils;
+import java.util.regex.Pattern;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 
@@ -41,8 +45,7 @@ public class BeamUtils {
       ImmutableList<String> fieldNames, SchemaAndRecord schemaAndRecord) {
     GenericRecord record = schemaAndRecord.getRecord();
     ImmutableList<String> nullFields =
-        fieldNames
-            .stream()
+        fieldNames.stream()
             .filter(fieldName -> record.get(fieldName) == null)
             .collect(ImmutableList.toImmutableList());
     String missingFieldList = Joiner.on(", ").join(nullFields);
@@ -61,4 +64,19 @@ public class BeamUtils {
   public static String getQueryFromFile(Class<?> clazz, String filename) {
     return ResourceUtils.readResourceUtf8(Resources.getResource(clazz, "sql/" + filename));
   }
+
+  /** Creates a beam job name and validates that it conforms to the requirements. */
+  public static String createJobName(String prefix, Clock clock) {
+    // Flex template job name must be unique and consists of only characters [-a-z0-9], starting
+    // with a letter and ending with a letter or number. So we replace the "T" and "Z" in ISO 8601
+    // with lowercase letters.
+    String jobName =
+        String.format("%s-%s", prefix, clock.nowUtc().toString("yyyy-MM-dd't'HH-mm-ss'z'"));
+    checkArgument(
+        Pattern.compile("^[a-z][-a-z0-9]*[a-z0-9]*").matcher(jobName).matches(),
+        "The job name %s is illegal, it consists of only characters [-a-z0-9], "
+            + "starting with a letter and ending with a letter or number,",
+        jobName);
+    return jobName;
+  }
 }

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -0,0 +1,88 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import java.util.List;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+
+/**
+ * A database snapshot shareable by concurrent queries from multiple database clients. A snapshot is
+ * uniquely identified by its {@link #getSnapshotId snapshotId}, and must stay open until all
+ * concurrent queries to this snapshot have attached to it by calling {@link
+ * google.registry.persistence.transaction.JpaTransactionManager#setDatabaseSnapshot}. However, it
+ * can be closed before those queries complete.
+ *
+ * <p>This feature is <em>Postgresql-only</em>.
+ *
+ * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to avoid
+ * exhausting predicate locks at the SERIALIZABLE level.
+ */
+// TODO(b/193662898): vendor-independent support for richer transaction semantics.
+public class DatabaseSnapshot implements AutoCloseable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private String snapshotId;
+  private EntityManager entityManager;
+  private EntityTransaction transaction;
+
+  private DatabaseSnapshot() {}
+
+  public String getSnapshotId() {
+    checkState(entityManager != null, "Snapshot not opened yet.");
+    checkState(entityManager.isOpen(), "Snapshot already closed.");
+    return snapshotId;
+  }
+
+  private DatabaseSnapshot open() {
+    entityManager = jpaTm().getStandaloneEntityManager();
+    transaction = entityManager.getTransaction();
+    transaction.setRollbackOnly();
+    transaction.begin();
+
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+
+    List<?> snapshotIds =
+        entityManager.createNativeQuery("SELECT pg_export_snapshot();").getResultList();
+    checkState(snapshotIds.size() == 1, "Unexpected number of snapshots: %s", snapshotIds.size());
+    snapshotId = (String) snapshotIds.get(0);
+    return this;
+  }
+
+  @Override
+  public void close() {
+    if (transaction != null && transaction.isActive()) {
+      try {
+        transaction.rollback();
+      } catch (Exception e) {
+        logger.atWarning().withCause(e).log("Failed to close a Database Snapshot");
+      }
+    }
+    if (entityManager != null && entityManager.isOpen()) {
+      entityManager.close();
+    }
+  }
+
+  public static DatabaseSnapshot createSnapshot() {
+    return new DatabaseSnapshot().open();
+  }
+}

core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java

@@ -17,18 +17,23 @@ package google.registry.beam.common;
 import static com.google.common.base.Verify.verify;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
-import google.registry.backup.AppEngineEnvironment;
+import google.registry.model.contact.ContactResource;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import java.io.Serializable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
 
-/** Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines. */
+/**
+ * Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines.
+ *
+ * <p>This pipeline may also be used as an integration test for {@link RegistryJpaIO.Read} in a
+ * project with realistic data.
+ */
 public class JpaDemoPipeline implements Serializable {
 
   public static void main(String[] args) {
@@ -38,40 +43,31 @@ public class JpaDemoPipeline implements Serializable {
 
     Pipeline pipeline = Pipeline.create(options);
     pipeline
-        .apply("Start", Create.of((Void) null))
         .apply(
-            "Generate Elements",
-            ParDo.of(
-                new DoFn<Void, Void>() {
-                  @ProcessElement
-                  public void processElement(OutputReceiver<Void> output) {
-                    for (int i = 0; i < 500; i++) {
-                      output.output(null);
-                    }
-                  }
-                }))
+            "Read contacts",
+            RegistryJpaIO.read(
+                () -> CriteriaQueryBuilder.create(ContactResource.class).build(),
+                ContactResource::getRepoId))
         .apply(
-            "Make Query",
+            "Count Contacts",
             ParDo.of(
-                new DoFn<Void, Void>() {
-                  private Counter counter = Metrics.counter("Demo", "Read");
+                new DoFn<String, Void>() {
+                  private Counter counter = Metrics.counter("Contacts", "Read");
 
                   @ProcessElement
                   public void processElement() {
                     // AppEngineEnvironment is needed as long as JPA entity classes still depends
                     // on Objectify.
-                    try (AppEngineEnvironment allowOfyEntity = new AppEngineEnvironment()) {
-                      int result =
-                          (Integer)
-                              jpaTm()
-                                  .transact(
-                                      () ->
-                                          jpaTm()
-                                              .getEntityManager()
-                                              .createNativeQuery("select 1;")
-                                              .getSingleResult());
-                      verify(result == 1, "Expecting 1, got %s.", result);
-                    }
+                    int result =
+                        (Integer)
+                            jpaTm()
+                                .transact(
+                                    () ->
+                                        jpaTm()
+                                            .getEntityManager()
+                                            .createNativeQuery("select 1;")
+                                            .getSingleResult());
+                    verify(result == 1, "Expecting 1, got %s.", result);
                     counter.inc();
                   }
                 }));

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -20,22 +20,33 @@ import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
-import google.registry.backup.AppEngineEnvironment;
-import google.registry.model.ofy.ObjectifyService;
+import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
+import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
+import java.io.Serializable;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.Nullable;
+import javax.persistence.criteria.CriteriaQuery;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupIntoBatches;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.util.ShardedKey;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 
 /**
@@ -51,10 +62,190 @@ public final class RegistryJpaIO {
 
   private RegistryJpaIO() {}
 
+  public static <R> Read<R, R> read(CriteriaQuerySupplier<R> query) {
+    return read(query, x -> x);
+  }
+
+  public static <R, T> Read<R, T> read(
+      CriteriaQuerySupplier<R> query, SerializableFunction<R, T> resultMapper) {
+    return Read.<R, T>builder().criteriaQuery(query).resultMapper(resultMapper).build();
+  }
+
+  public static <R, T> Read<R, T> read(
+      String sql, boolean nativeQuery, SerializableFunction<R, T> resultMapper) {
+    return read(sql, null, nativeQuery, resultMapper);
+  }
+
+  /**
+   * Returns a {@link Read} connector based on the given {@code jpql} query string.
+   *
+   * <p>User should take care to prevent sql-injection attacks.
+   */
+  public static <R, T> Read<R, T> read(
+      String sql,
+      @Nullable Map<String, Object> parameter,
+      boolean nativeQuery,
+      SerializableFunction<R, T> resultMapper) {
+    Read.Builder<R, T> builder = Read.builder();
+    if (nativeQuery) {
+      builder.nativeQuery(sql, parameter);
+    } else {
+      builder.jpqlQuery(sql, parameter);
+    }
+    return builder.resultMapper(resultMapper).build();
+  }
+
+  public static <R, T> Read<R, T> read(
+      String jpql, Class<R> clazz, SerializableFunction<R, T> resultMapper) {
+    return read(jpql, null, clazz, resultMapper);
+  }
+
+  /**
+   * Returns a {@link Read} connector based on the given {@code jpql} typed query string.
+   *
+   * <p>User should take care to prevent sql-injection attacks.
+   */
+  public static <R, T> Read<R, T> read(
+      String jpql,
+      @Nullable Map<String, Object> parameter,
+      Class<R> clazz,
+      SerializableFunction<R, T> resultMapper) {
+    return Read.<R, T>builder()
+        .jpqlQuery(jpql, clazz, parameter)
+        .resultMapper(resultMapper)
+        .build();
+  }
+
   public static <T> Write<T> write() {
     return Write.<T>builder().build();
   }
 
+  /**
+   * A {@link PTransform transform} that transactionally executes a JPA {@link CriteriaQuery} and
+   * adds the results to the BEAM pipeline. Users have the option to transform the results before
+   * sending them to the next stages.
+   */
+  @AutoValue
+  public abstract static class Read<R, T> extends PTransform<PBegin, PCollection<T>> {
+
+    public static final String DEFAULT_NAME = "RegistryJpaIO.Read";
+
+    abstract String name();
+
+    abstract RegistryQuery<R> query();
+
+    abstract SerializableFunction<R, T> resultMapper();
+
+    abstract Coder<T> coder();
+
+    @Nullable
+    abstract String snapshotId();
+
+    abstract Builder<R, T> toBuilder();
+
+    @Override
+    @SuppressWarnings("deprecation") // Reshuffle still recommended by GCP.
+    public PCollection<T> expand(PBegin input) {
+      return input
+          .apply("Starting " + name(), Create.of((Void) null))
+          .apply(
+              "Run query for " + name(),
+              ParDo.of(new QueryRunner<>(query(), resultMapper(), snapshotId())))
+          .setCoder(coder())
+          .apply("Reshuffle", Reshuffle.viaRandomKey());
+    }
+
+    public Read<R, T> withName(String name) {
+      return toBuilder().name(name).build();
+    }
+
+    public Read<R, T> withResultMapper(SerializableFunction<R, T> mapper) {
+      return toBuilder().resultMapper(mapper).build();
+    }
+
+    public Read<R, T> withCoder(Coder<T> coder) {
+      return toBuilder().coder(coder).build();
+    }
+
+    /**
+     * Specifies the database snapshot to use for this query.
+     *
+     * <p>This feature is <em>Postgresql-only</em>. User is responsible for keeping the snapshot
+     * available until all JVM workers have started using it by calling {@link
+     * JpaTransactionManager#setDatabaseSnapshot}.
+     */
+    // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+    public Read<R, T> withSnapshot(@Nullable String snapshotId) {
+      return toBuilder().snapshotId(snapshotId).build();
+    }
+
+    static <R, T> Builder<R, T> builder() {
+      return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
+          .name(DEFAULT_NAME)
+          .coder(SerializableCoder.of(Serializable.class));
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder<R, T> {
+
+      abstract Builder<R, T> name(String name);
+
+      abstract Builder<R, T> query(RegistryQuery<R> query);
+
+      abstract Builder<R, T> resultMapper(SerializableFunction<R, T> mapper);
+
+      abstract Builder<R, T> coder(Coder coder);
+
+      abstract Builder<R, T> snapshotId(@Nullable String sharedSnapshotId);
+
+      abstract Read<R, T> build();
+
+      Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
+        return query(RegistryQuery.createQuery(criteriaQuery));
+      }
+
+      Builder<R, T> nativeQuery(String sql, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(sql, parameters, true));
+      }
+
+      Builder<R, T> jpqlQuery(String jpql, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(jpql, parameters, false));
+      }
+
+      Builder<R, T> jpqlQuery(String jpql, Class<R> clazz, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(jpql, parameters, clazz));
+      }
+    }
+
+    static class QueryRunner<R, T> extends DoFn<Void, T> {
+      private final RegistryQuery<R> query;
+      private final SerializableFunction<R, T> resultMapper;
+      // java.util.Optional is not serializable. Use of Guava Optional is discouraged.
+      @Nullable private final String snapshotId;
+
+      QueryRunner(
+          RegistryQuery<R> query,
+          SerializableFunction<R, T> resultMapper,
+          @Nullable String snapshotId) {
+        this.query = query;
+        this.resultMapper = resultMapper;
+        this.snapshotId = snapshotId;
+      }
+
+      @ProcessElement
+      public void processElement(OutputReceiver<T> outputReceiver) {
+        jpaTm()
+            .transactNoRetry(
+                () -> {
+                  if (snapshotId != null) {
+                    jpaTm().setDatabaseSnapshot(snapshotId);
+                  }
+                  query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
+                });
+      }
+    }
+  }
+
   /**
    * A {@link PTransform transform} that writes a PCollection of entities to the SQL database using
    * the {@link JpaTransactionManager}.
@@ -108,6 +299,12 @@ public final class RegistryJpaIO {
 
     public abstract SerializableFunction<T, Object> jpaConverter();
 
+    /**
+     * Signal to the writer that the {@link UpdateAutoTimestamp} property should be allowed to
+     * manipulate its value before persistence. The default value is {@code true}.
+     */
+    abstract boolean withUpdateAutoTimestamp();
+
     public Write<T> withName(String name) {
       return toBuilder().name(name).build();
     }
@@ -128,6 +325,10 @@ public final class RegistryJpaIO {
       return toBuilder().jpaConverter(jpaConverter).build();
     }
 
+    public Write<T> disableUpdateAutoTimestamp() {
+      return toBuilder().withUpdateAutoTimestamp(false).build();
+    }
+
     abstract Builder<T> toBuilder();
 
     @Override
@@ -144,7 +345,7 @@ public final class RegistryJpaIO {
               GroupIntoBatches.<Integer, T>ofSize(batchSize()).withShardedKey())
           .apply(
               "Write in batch for " + name(),
-              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter())));
+              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter(), withUpdateAutoTimestamp())));
     }
 
     static <T> Builder<T> builder() {
@@ -152,7 +353,8 @@ public final class RegistryJpaIO {
           .name(DEFAULT_NAME)
           .batchSize(DEFAULT_BATCH_SIZE)
           .shards(DEFAULT_SHARDS)
-          .jpaConverter(x -> x);
+          .jpaConverter(x -> x)
+          .withUpdateAutoTimestamp(true);
     }
 
     @AutoValue.Builder
@@ -166,6 +368,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<T> jpaConverter(SerializableFunction<T, Object> jpaConverter);
 
+      abstract Builder<T> withUpdateAutoTimestamp(boolean withUpdateAutoTimestamp);
+
       abstract Write<T> build();
     }
   }
@@ -174,36 +378,38 @@ public final class RegistryJpaIO {
   private static class SqlBatchWriter<T> extends DoFn<KV<ShardedKey<Integer>, Iterable<T>>, Void> {
     private final Counter counter;
     private final SerializableFunction<T, Object> jpaConverter;
+    private final boolean withAutoTimestamp;
 
-    SqlBatchWriter(String type, SerializableFunction<T, Object> jpaConverter) {
+    SqlBatchWriter(
+        String type, SerializableFunction<T, Object> jpaConverter, boolean withAutoTimestamp) {
       counter = Metrics.counter("SQL_WRITE", type);
       this.jpaConverter = jpaConverter;
-    }
-
-    @Setup
-    public void setup() {
-      // Below is needed as long as Objectify keys are still involved in the handling of SQL
-      // entities (e.g., in VKeys).
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ObjectifyService.initOfy();
-      }
+      this.withAutoTimestamp = withAutoTimestamp;
     }
 
     @ProcessElement
     public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ImmutableList<Object> ofyEntities =
-            Streams.stream(kv.getValue())
-                .map(this.jpaConverter::apply)
-                // TODO(b/177340730): post migration delete the line below.
-                .filter(Objects::nonNull)
-                .collect(ImmutableList.toImmutableList());
-        try {
-          jpaTm().transact(() -> jpaTm().putAll(ofyEntities));
-          counter.inc(ofyEntities.size());
-        } catch (RuntimeException e) {
-          processSingly(ofyEntities);
-        }
+      if (withAutoTimestamp) {
+        actuallyProcessElement(kv);
+        return;
+      }
+      try (DisableAutoUpdateResource disable = UpdateAutoTimestamp.disableAutoUpdate()) {
+        actuallyProcessElement(kv);
+      }
+    }
+
+    private void actuallyProcessElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
+      ImmutableList<Object> entities =
+          Streams.stream(kv.getValue())
+              .map(this.jpaConverter::apply)
+              // TODO(b/177340730): post migration delete the line below.
+              .filter(Objects::nonNull)
+              .collect(ImmutableList.toImmutableList());
+      try {
+        jpaTm().transact(() -> jpaTm().putAll(entities));
+        counter.inc(entities.size());
+      } catch (RuntimeException e) {
+        processSingly(entities);
       }
     }
 
@@ -211,19 +417,22 @@ public final class RegistryJpaIO {
      * Writes entities in a failed batch one by one to identify the first bad entity and throws a
      * {@link RuntimeException} on it.
      */
-    private void processSingly(ImmutableList<Object> ofyEntities) {
-      for (Object ofyEntity : ofyEntities) {
+    private void processSingly(ImmutableList<Object> entities) {
+      for (Object entity : entities) {
         try {
-          jpaTm().transact(() -> jpaTm().put(ofyEntity));
+          jpaTm().transact(() -> jpaTm().put(entity));
           counter.inc();
         } catch (RuntimeException e) {
-          throw new RuntimeException(toOfyKey(ofyEntity).toString(), e);
+          throw new RuntimeException(toEntityKeyString(entity), e);
         }
       }
     }
 
-    private com.googlecode.objectify.Key<?> toOfyKey(Object ofyEntity) {
-      return com.googlecode.objectify.Key.create(ofyEntity);
+    private String toEntityKeyString(Object entity) {
+      if (entity instanceof SqlEntity) {
+        return ((SqlEntity) entity).getPrimaryKeyString();
+      }
+      return "Non-SqlEntity: " + String.valueOf(entity);
     }
   }
 }

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -21,7 +21,9 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
+import google.registry.persistence.PersistenceModule.BeamReadOnlyReplicaJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.privileges.secretmanager.SecretManagerModule;
@@ -45,9 +47,26 @@ public interface RegistryPipelineComponent {
   @Config("projectId")
   String getProjectId();
 
+  /** Returns the regular {@link JpaTransactionManager} for general use. */
   @BeamJpaTm
   Lazy<JpaTransactionManager> getJpaTransactionManager();
 
+  /**
+   * Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
+   * ({@link google.registry.model.domain.DomainBase} and {@link
+   * google.registry.model.domain.DomainHistory}). Please refer to {@link
+   * google.registry.model.bulkquery.BulkQueryEntities} for more information.
+   */
+  @BeamBulkQueryJpaTm
+  Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
+
+  /**
+   * A {@link JpaTransactionManager} that uses the Postgres read-only replica if configured (uses
+   * the standard DB otherwise).
+   */
+  @BeamReadOnlyReplicaJpaTm
+  Lazy<JpaTransactionManager> getReadOnlyReplicaJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -34,7 +35,6 @@ import org.apache.beam.sdk.options.Description;
 public interface RegistryPipelineOptions extends GcpOptions {
 
   @Description("The Registry environment.")
-  @Nullable
   RegistryEnvironment getRegistryEnvironment();
 
   void setRegistryEnvironment(RegistryEnvironment environment);
@@ -45,6 +45,12 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setIsolationOverride(TransactionIsolationLevel isolationOverride);
 
+  @Description("The JPA Transaction Manager to use.")
+  @Default.Enum(value = "REGULAR")
+  JpaTransactionManagerType getJpaTransactionManagerType();
+
+  void setJpaTransactionManagerType(JpaTransactionManagerType jpaTransactionManagerType);
+
   @Description("The number of entities to write to the SQL database in one operation.")
   @Default.Integer(20)
   int getSqlWriteBatchSize();

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -20,6 +20,8 @@ import com.google.auto.service.AutoService;
 import com.google.common.flogger.FluentLogger;
 import dagger.Lazy;
 import google.registry.config.RegistryEnvironment;
+import google.registry.config.SystemPropertySetter;
+import google.registry.model.AppEngineEnvironment;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import org.apache.beam.sdk.harness.JvmInitializer;
@@ -35,18 +37,40 @@ import org.apache.beam.sdk.options.PipelineOptions;
 @AutoService(JvmInitializer.class)
 public class RegistryPipelineWorkerInitializer implements JvmInitializer {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  public static final String PROPERTY = "google.registry.beam";
 
   @Override
   public void beforeProcessing(PipelineOptions options) {
     RegistryPipelineOptions registryOptions = options.as(RegistryPipelineOptions.class);
     RegistryEnvironment environment = registryOptions.getRegistryEnvironment();
     if (environment == null || environment.equals(RegistryEnvironment.UNITTEST)) {
-      return;
+      throw new RuntimeException(
+          "A registry environment must be specified in the pipeline options.");
     }
-    logger.atInfo().log("Setting up RegistryEnvironment: %s", environment);
+    logger.atInfo().log("Setting up RegistryEnvironment %s.", environment);
     environment.setup();
-    Lazy<JpaTransactionManager> transactionManagerLazy =
-        toRegistryPipelineComponent(registryOptions).getJpaTransactionManager();
+    RegistryPipelineComponent registryPipelineComponent =
+        toRegistryPipelineComponent(registryOptions);
+    Lazy<JpaTransactionManager> transactionManagerLazy;
+    switch (registryOptions.getJpaTransactionManagerType()) {
+      case BULK_QUERY:
+        transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
+        break;
+      case READ_ONLY_REPLICA:
+        transactionManagerLazy =
+            registryPipelineComponent.getReadOnlyReplicaJpaTransactionManager();
+        break;
+      case REGULAR:
+      default:
+        transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
+    }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
+    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // loads all ofy entities.
+    new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
+        .setEnvironmentForAllThreads();
+    // Set the system property so that we can call IdService.allocateId() without access to
+    // datastore.
+    SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
   }
 }

core/src/main/java/google/registry/beam/common/RegistryQuery.java

@@ -0,0 +1,139 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import google.registry.persistence.transaction.JpaTransactionManager;
+import java.io.Serializable;
+import java.util.Map;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaQuery;
+
+/** Interface for query instances used by {@link RegistryJpaIO.Read}. */
+public interface RegistryQuery<T> extends Serializable {
+
+  /**
+   * Number of JPA entities to fetch in each batch during a query.
+   *
+   * <p>With Hibernate, for result streaming to work, a query's fetchSize property must be set to a
+   * non-zero value.
+   */
+  int QUERY_FETCH_SIZE = 1000;
+
+  Stream<T> stream();
+
+  interface CriteriaQuerySupplier<T> extends Supplier<CriteriaQuery<T>>, Serializable {}
+
+  /**
+   * Returns a {@link RegistryQuery} that creates a string query from constant text.
+   *
+   * @param nativeQuery whether the given string is to be interpreted as a native query or JPQL.
+   * @param parameters parameters to be substituted in the query.
+   * @param <T> Type of each row in the result set, {@link Object} in single-select queries, and
+   *     {@code Object[]} in multi-select queries.
+   */
+  static <T> RegistryQuery<T> createQuery(
+      String sql, @Nullable Map<String, Object> parameters, boolean nativeQuery) {
+    return () -> {
+      EntityManager entityManager = jpaTm().getEntityManager();
+      Query query =
+          nativeQuery ? entityManager.createNativeQuery(sql) : entityManager.createQuery(sql);
+      if (parameters != null) {
+        parameters.forEach(query::setParameter);
+      }
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      @SuppressWarnings("unchecked")
+      Stream<T> resultStream = query.getResultStream();
+      return nativeQuery ? resultStream : resultStream.map(e -> detach(entityManager, e));
+    };
+  }
+
+  /**
+   * Returns a {@link RegistryQuery} that creates a typed JPQL query from constant text.
+   *
+   * @param parameters parameters to be substituted in the query.
+   * @param <T> Type of each row in the result set.
+   */
+  static <T> RegistryQuery<T> createQuery(
+      String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
+    return () -> {
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(jpql, clazz);
+      if (parameters != null) {
+        parameters.forEach(query::setParameter);
+      }
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
+    };
+  }
+
+  /**
+   * Returns a {@link RegistryQuery} from a {@link CriteriaQuery} supplier.
+   *
+   * <p>A serializable supplier is needed in because {@link CriteriaQuery} itself must be created
+   * within a transaction, and we are not in a transaction yet when this function is called to set
+   * up the pipeline.
+   *
+   * @param <T> Type of each row in the result set.
+   */
+  static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
+    return () -> {
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(criteriaQuery.get());
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
+    };
+  }
+
+  /**
+   * Removes an object from the JPA session cache if applicable.
+   *
+   * @param object An object that represents a row in the result set. It may be a JPA entity, a
+   *     non-entity object, or an array that holds JPA entities and/or non-entities.
+   */
+  static <T> T detach(EntityManager entityManager, T object) {
+    if (object.getClass().isArray()) {
+      for (Object arrayElement : (Object[]) object) {
+        detachObject(entityManager, arrayElement);
+      }
+    } else {
+      detachObject(entityManager, object);
+    }
+    return object;
+  }
+
+  static void detachObject(EntityManager entityManager, Object object) {
+    Class<?> objectClass = object.getClass();
+    if (objectClass.isPrimitive() || objectClass == String.class) {
+      return;
+    }
+    try {
+      // TODO(b/193662898): choose detach() or clear() based on the type of transaction.
+      // For context, EntityManager.detach() does not remove all metadata about loaded entities.
+      // See b/193925312 or https://hibernate.atlassian.net/browse/HHH-14735 for details.
+      entityManager.clear();
+    } catch (IllegalArgumentException e) {
+      // Not an entity. Do nothing.
+    }
+  }
+}

core/src/main/java/google/registry/beam/comparedb/DatastoreSnapshots.java

@@ -0,0 +1,193 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.initsql.Transforms.createTagForKind;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.Transforms;
+import google.registry.model.EppResource;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.joda.time.DateTime;
+
+/** Utilities for loading Datastore snapshots. */
+@DeleteAfterMigration
+public final class DatastoreSnapshots {
+
+  private DatastoreSnapshots() {}
+
+  /**
+   * Datastore kinds eligible for validation. This set must be consistent with {@link
+   * SqlSnapshots#ALL_SQL_ENTITIES}.
+   */
+  @VisibleForTesting
+  static final ImmutableSet<Class<?>> ALL_DATASTORE_KINDS =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          HistoryEntry.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class);
+
+  /**
+   * Returns the Datastore snapshot right before {@code commitLogToTime} for the user specified
+   * {@code kinds}. The resulting snapshot has all changes that happened before {@code
+   * commitLogToTime}, and none at or after {@code commitLogToTime}.
+   *
+   * <p>If {@code HistoryEntry} is included in {@code kinds}, the result will contain {@code
+   * PCollections} for the child entities, {@code DomainHistory}, {@code ContactHistory}, and {@code
+   * HostHistory}.
+   */
+  static PCollectionTuple loadDatastoreSnapshotByKind(
+      Pipeline pipeline,
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<Class<?>> kinds,
+      Optional<DateTime> compareStartTime) {
+    PCollectionTuple snapshot =
+        pipeline.apply(
+            "Load Datastore snapshot.",
+            Transforms.loadDatastoreSnapshot(
+                exportDir,
+                commitLogDir,
+                commitLogFromTime,
+                commitLogToTime,
+                kinds.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet())));
+
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<?> kind : kinds) {
+      PCollection<VersionedEntity> perKindSnapshot =
+          snapshot.get(createTagForKind(Key.getKind(kind)));
+      if (SqlEntity.class.isAssignableFrom(kind)) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag((Class<? extends SqlEntity>) kind),
+                datastoreEntityToPojo(perKindSnapshot, kind.getSimpleName(), compareStartTime));
+        continue;
+      }
+      Verify.verify(kind == HistoryEntry.class, "Unexpected Non-SqlEntity class: %s", kind);
+      PCollectionTuple historyEntriesByType = splitHistoryEntry(perKindSnapshot, compareStartTime);
+      for (Map.Entry<TupleTag<?>, PCollection<?>> entry :
+          historyEntriesByType.getAll().entrySet()) {
+        perTypeSnapshots = perTypeSnapshots.and(entry.getKey().getId(), entry.getValue());
+      }
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Splits a {@link PCollection} of {@link HistoryEntry HistoryEntries} into three collections of
+   * its child entities by type.
+   */
+  static PCollectionTuple splitHistoryEntry(
+      PCollection<VersionedEntity> historyEntries, Optional<DateTime> compareStartTime) {
+    DateTime nullableStartTime = compareStartTime.orElse(null);
+    return historyEntries.apply(
+        "Split HistoryEntry by Resource Type",
+        ParDo.of(
+                new DoFn<VersionedEntity, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity historyEntry, MultiOutputReceiver out) {
+                    Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(historyEntry))
+                        .filter(e -> isEntityIncludedForComparison(e, nullableStartTime))
+                        .ifPresent(
+                            sqlEntity ->
+                                out.get(createSqlEntityTupleTag(sqlEntity.getClass()))
+                                    .output(sqlEntity));
+                  }
+                })
+            .withOutputTags(
+                createSqlEntityTupleTag(DomainHistory.class),
+                TupleTagList.of(createSqlEntityTupleTag(ContactHistory.class))
+                    .and(createSqlEntityTupleTag(HostHistory.class))));
+  }
+
+  /**
+   * Transforms a {@link PCollection} of {@link VersionedEntity VersionedEntities} to Ofy Java
+   * objects.
+   */
+  static PCollection<SqlEntity> datastoreEntityToPojo(
+      PCollection<VersionedEntity> entities, String desc, Optional<DateTime> compareStartTime) {
+    DateTime nullableStartTime = compareStartTime.orElse(null);
+    return entities.apply(
+        "Datastore Entity to Pojo " + desc,
+        ParDo.of(
+            new DoFn<VersionedEntity, SqlEntity>() {
+              @ProcessElement
+              public void processElement(
+                  @Element VersionedEntity entity, OutputReceiver<SqlEntity> out) {
+                Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(entity))
+                    .filter(e -> isEntityIncludedForComparison(e, nullableStartTime))
+                    .ifPresent(out::output);
+              }
+            }));
+  }
+
+  static boolean isEntityIncludedForComparison(
+      SqlEntity entity, @Nullable DateTime compareStartTime) {
+    if (compareStartTime == null) {
+      return true;
+    }
+    if (entity instanceof HistoryEntry) {
+      return compareStartTime.isBefore(((HistoryEntry) entity).getModificationTime());
+    }
+    if (entity instanceof EppResource) {
+      return compareStartTime.isBefore(((EppResource) entity).getUpdateTimestamp().getTimestamp());
+    }
+    return true;
+  }
+}

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -0,0 +1,147 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import dagger.Component;
+import google.registry.config.CloudTasksUtilsModule;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.util.Clock;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.joda.time.DateTime;
+import org.joda.time.Instant;
+import org.joda.time.Interval;
+
+/** Finds the necessary information for loading the most recent Datastore snapshot. */
+@DeleteAfterMigration
+public class LatestDatastoreSnapshotFinder {
+  private final String projectId;
+  private final GcsUtils gcsUtils;
+  private final Clock clock;
+
+  @Inject
+  LatestDatastoreSnapshotFinder(
+      @Config("projectId") String projectId, GcsUtils gcsUtils, Clock clock) {
+    this.projectId = projectId;
+    this.gcsUtils = gcsUtils;
+    this.clock = clock;
+  }
+
+  /**
+   * Finds information of the most recent Datastore snapshot, including the GCS folder of the
+   * exported data files and the start and stop times of the export. The folder of the CommitLogs is
+   * also included in the return.
+   */
+  public DatastoreSnapshotInfo getSnapshotInfo() {
+    String bucketName = RegistryConfig.getDatastoreBackupsBucket().substring("gs://".length());
+    /**
+     * Find the bucket-relative path to the overall metadata file of the last Datastore export.
+     * Since Datastore export is saved daily, we may need to look back to yesterday. If found, the
+     * return value is like
+     * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
+     */
+    Optional<String> metaFilePathOptional = findMostRecentExportMetadataFile(bucketName, 2);
+    if (!metaFilePathOptional.isPresent()) {
+      throw new NoSuchElementException("No exports found over the past 2 days.");
+    }
+    String metaFilePath = metaFilePathOptional.get();
+    String metaFileFolder = metaFilePath.substring(0, metaFilePath.indexOf('/'));
+    Instant exportStartTime = Instant.parse(metaFileFolder.replace('_', '.') + 'Z');
+    BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath));
+    Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+    return DatastoreSnapshotInfo.create(
+        String.format("gs://%s/%s", bucketName, metaFileFolder),
+        getCommitLogDir(),
+        new Interval(exportStartTime, exportEndTime));
+  }
+
+  public String getCommitLogDir() {
+    return "gs://" + projectId + "-commits";
+  }
+
+  /**
+   * Finds the bucket-relative path of the overall export metadata file, in the given bucket,
+   * searching back up to {@code lookBackDays} days, including today.
+   *
+   * <p>The overall export metadata file is the last file created during a Datastore export. All
+   * data has been exported by the creation time of this file. The name of this file, like that of
+   * all files in the same export, begins with the timestamp when the export starts.
+   *
+   * <p>An example return value: {@code
+   * 2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata}.
+   */
+  private Optional<String> findMostRecentExportMetadataFile(String bucketName, int lookBackDays) {
+    DateTime today = clock.nowUtc();
+    for (int day = 0; day < lookBackDays; day++) {
+      String dateString = today.minusDays(day).toString("yyyy-MM-dd");
+      try {
+        Optional<String> metaFilePath =
+            gcsUtils.listFolderObjects(bucketName, dateString).stream()
+                .filter(s -> s.endsWith("overall_export_metadata"))
+                .map(s -> dateString + s)
+                .sorted(Comparator.<String>naturalOrder().reversed())
+                .findFirst();
+        if (metaFilePath.isPresent()) {
+          return metaFilePath;
+        }
+      } catch (IOException ioe) {
+        throw new RuntimeException(ioe);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /** Holds information about a Datastore snapshot. */
+  @AutoValue
+  abstract static class DatastoreSnapshotInfo {
+    abstract String exportDir();
+
+    abstract String commitLogDir();
+
+    abstract Interval exportInterval();
+
+    static DatastoreSnapshotInfo create(
+        String exportDir, String commitLogDir, Interval exportOperationInterval) {
+      return new AutoValue_LatestDatastoreSnapshotFinder_DatastoreSnapshotInfo(
+          exportDir, commitLogDir, exportOperationInterval);
+    }
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
+  interface LatestDatastoreSnapshotFinderFinderComponent {
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotInfoFinder();
+  }
+}

core/src/main/java/google/registry/beam/comparedb/SqlSnapshots.java

@@ -0,0 +1,539 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.comparedb.ValidateSqlUtils.getMedianIdForHistoryTable;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Strings;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Streams;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryJpaIO.Read;
+import google.registry.model.EppResource;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.model.bulkquery.DomainBaseLite;
+import google.registry.model.bulkquery.DomainHistoryHost;
+import google.registry.model.bulkquery.DomainHistoryLite;
+import google.registry.model.bulkquery.DomainHost;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import google.registry.util.DateTimeUtils;
+import java.io.Serializable;
+import java.util.Optional;
+import javax.persistence.Entity;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.joda.time.DateTime;
+
+/**
+ * Utilities for loading SQL snapshots.
+ *
+ * <p>For {@link DomainBase} and {@link DomainHistory}, this class assumes the presence of the
+ * {@link google.registry.persistence.PersistenceModule.JpaTransactionManagerType#BULK_QUERY
+ * bulk-query-capable JpaTransactionManager}, and takes advantage of it for higher throughput.
+ *
+ * <p>For now this class is meant for use during the database migration period only. Therefore, it
+ * contains optimizations specifically for the production database at the current size, e.g.,
+ * parallel queries for select tables.
+ */
+@DeleteAfterMigration
+public final class SqlSnapshots {
+
+  private SqlSnapshots() {}
+
+  /**
+   * SQL entity types that are eligible for validation. This set must be consistent with {@link
+   * DatastoreSnapshots#ALL_DATASTORE_KINDS}.
+   */
+  static final ImmutableSet<Class<? extends SqlEntity>> ALL_SQL_ENTITIES =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class,
+          ContactHistory.class,
+          HostHistory.class,
+          DomainHistory.class);
+
+  /**
+   * Loads a SQL snapshot for the given {@code sqlEntityTypes}.
+   *
+   * <p>If {@code snapshotId} is present, all queries use the specified database snapshot,
+   * guaranteeing a consistent result.
+   */
+  public static PCollectionTuple loadCloudSqlSnapshotByType(
+      Pipeline pipeline,
+      ImmutableSet<Class<? extends SqlEntity>> sqlEntityTypes,
+      Optional<String> snapshotId,
+      Optional<DateTime> compareStartTime) {
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<? extends SqlEntity> clazz : sqlEntityTypes) {
+      if (clazz == DomainBase.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainBase.class),
+                loadAndAssembleDomainBase(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == DomainHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainHistory.class),
+                loadAndAssembleDomainHistory(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == ContactHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(ContactHistory.class),
+                loadContactHistory(pipeline, snapshotId, compareStartTime));
+        continue;
+      }
+      if (clazz == HostHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(HostHistory.class),
+                loadHostHistory(
+                    pipeline, snapshotId, compareStartTime.orElse(DateTimeUtils.START_OF_TIME)));
+        continue;
+      }
+      if (EppResource.class.isAssignableFrom(clazz) && compareStartTime.isPresent()) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(clazz),
+                pipeline.apply(
+                    "SQL Load " + clazz.getSimpleName(),
+                    buildEppResourceQueryWithTimeFilter(
+                            clazz, SqlEntity.class, snapshotId, compareStartTime.get())
+                        .withSnapshot(snapshotId.orElse(null))));
+        continue;
+      }
+      perTypeSnapshots =
+          perTypeSnapshots.and(
+              createSqlEntityTupleTag(clazz),
+              pipeline.apply(
+                  "SQL Load " + clazz.getSimpleName(),
+                  RegistryJpaIO.read(
+                          () -> CriteriaQueryBuilder.create(clazz).build(), SqlEntity.class::cast)
+                      .withSnapshot(snapshotId.orElse(null))));
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Bulk-loads parts of {@link DomainBase} and assembles them in the pipeline.
+   *
+   * @see BulkQueryEntities
+   */
+  public static PCollection<SqlEntity> loadAndAssembleDomainBase(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PCollection<KV<String, Serializable>> baseObjects =
+        readAllAndAssignKey(
+            pipeline,
+            DomainBaseLite.class,
+            DomainBaseLite::getRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriod.class,
+            GracePeriod::getDomainRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DelegationSignerData.class,
+            DelegationSignerData::getDomainRepoId,
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline, DomainHost.class, DomainHost::getDomainRepoId, snapshotId, compareStartTime);
+
+    DateTime nullableCompareStartTime = compareStartTime.orElse(null);
+    return PCollectionList.of(
+            ImmutableList.of(baseObjects, gracePeriods, delegationSigners, domainHosts))
+        .apply("SQL Merge DomainBase parts", Flatten.pCollections())
+        .apply("Group by Domain Parts by RepoId", GroupByKey.create())
+        .apply(
+            "Assemble DomainBase",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainBaseLite> baseObjects =
+                        partsByType.getAllOf(DomainBaseLite.class);
+                    if (nullableCompareStartTime != null) {
+                      Verify.verify(
+                          baseObjects.size() <= 1,
+                          "Found duplicate DomainBaseLite object per repoId: " + kv.getKey());
+                      if (baseObjects.isEmpty()) {
+                        return;
+                      }
+                    }
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainBaseLite object per repoId: " + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainBase(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(GracePeriod.class),
+                            partsByType.getAllOf(DelegationSignerData.class),
+                            partsByType.getAllOf(DomainHost.class).stream()
+                                .map(DomainHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet())));
+                  }
+                }));
+  }
+
+  /**
+   * Loads all {@link ContactHistory} entities from the database.
+   *
+   * <p>This method uses two queries to load data in parallel. This is a performance optimization
+   * specifically for the production database.
+   */
+  static PCollection<SqlEntity> loadContactHistory(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PartitionedQuery partitionedQuery =
+        buildPartitonedHistoryQuery(ContactHistory.class, compareStartTime);
+    PCollection<SqlEntity> part1 =
+        pipeline.apply(
+            "SQL Load ContactHistory first half",
+            RegistryJpaIO.read(
+                    partitionedQuery.firstHalfQuery(),
+                    partitionedQuery.parameters(),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    PCollection<SqlEntity> part2 =
+        pipeline.apply(
+            "SQL Load ContactHistory second half",
+            RegistryJpaIO.read(
+                    partitionedQuery.secondHalfQuery(),
+                    partitionedQuery.parameters(),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    return PCollectionList.of(part1)
+        .and(part2)
+        .apply("Combine ContactHistory parts", Flatten.pCollections());
+  }
+
+  /** Loads all {@link HostHistory} entities from the database. */
+  static PCollection<SqlEntity> loadHostHistory(
+      Pipeline pipeline, Optional<String> snapshotId, DateTime compareStartTime) {
+    return pipeline.apply(
+        "SQL Load HostHistory",
+        RegistryJpaIO.read(
+                "select c from HostHistory c where :compareStartTime  < modificationTime",
+                ImmutableMap.of("compareStartTime", compareStartTime),
+                false,
+                SqlEntity.class::cast)
+            .withSnapshot(snapshotId.orElse(null)));
+  }
+
+  /**
+   * Bulk-loads all parts of {@link DomainHistory} and assembles them in the pipeline.
+   *
+   * <p>This method uses two queries to load {@link DomainBaseLite} in parallel. This is a
+   * performance optimization specifically for the production database.
+   *
+   * @see BulkQueryEntities
+   */
+  static PCollection<SqlEntity> loadAndAssembleDomainHistory(
+      Pipeline pipeline, Optional<String> snapshotId, Optional<DateTime> compareStartTime) {
+    PartitionedQuery partitionedQuery =
+        buildPartitonedHistoryQuery(DomainHistoryLite.class, compareStartTime);
+    PCollection<KV<String, Serializable>> baseObjectsPart1 =
+        queryAndAssignKey(
+            pipeline,
+            "first half",
+            partitionedQuery.firstHalfQuery(),
+            partitionedQuery.parameters(),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> baseObjectsPart2 =
+        queryAndAssignKey(
+            pipeline,
+            "second half",
+            partitionedQuery.secondHalfQuery(),
+            partitionedQuery.parameters(),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriodHistory.class,
+            compose(GracePeriodHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DomainDsDataHistory.class,
+            compose(DomainDsDataHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline,
+            DomainHistoryHost.class,
+            compose(DomainHistoryHost::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+    PCollection<KV<String, Serializable>> transactionRecords =
+        readAllAndAssignKey(
+            pipeline,
+            DomainTransactionRecord.class,
+            compose(DomainTransactionRecord::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId,
+            compareStartTime);
+
+    DateTime nullableCompareStartTime = compareStartTime.orElse(null);
+    return PCollectionList.of(
+            ImmutableList.of(
+                baseObjectsPart1,
+                baseObjectsPart2,
+                gracePeriods,
+                delegationSigners,
+                domainHosts,
+                transactionRecords))
+        .apply("Merge DomainHistory parts", Flatten.pCollections())
+        .apply("Group by DomainHistory Parts by DomainHistoryId string", GroupByKey.create())
+        .apply(
+            "Assemble DomainHistory",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainHistoryLite> baseObjects =
+                        partsByType.getAllOf(DomainHistoryLite.class);
+                    if (nullableCompareStartTime != null) {
+                      Verify.verify(
+                          baseObjects.size() <= 1,
+                          "Found duplicate DomainHistoryLite object per domainHistoryId: "
+                              + kv.getKey());
+                      if (baseObjects.isEmpty()) {
+                        return;
+                      }
+                    }
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainHistoryLite object per domainHistoryId: "
+                            + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainHistory(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(DomainDsDataHistory.class),
+                            partsByType.getAllOf(DomainHistoryHost.class).stream()
+                                .map(DomainHistoryHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet()),
+                            partsByType.getAllOf(GracePeriodHistory.class),
+                            partsByType.getAllOf(DomainTransactionRecord.class)));
+                  }
+                }));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> readAllAndAssignKey(
+      Pipeline pipeline,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId,
+      Optional<DateTime> compareStartTime) {
+    Read<R, R> queryObject;
+    if (compareStartTime.isPresent() && EppResource.class.isAssignableFrom(type)) {
+      queryObject =
+          buildEppResourceQueryWithTimeFilter(type, type, snapshotId, compareStartTime.get());
+    } else {
+      queryObject =
+          RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(type).build())
+              .withSnapshot(snapshotId.orElse(null));
+    }
+    return pipeline
+        .apply("SQL Load " + type.getSimpleName(), queryObject)
+        .apply(
+            "Assign Key to " + type.getSimpleName(),
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> queryAndAssignKey(
+      Pipeline pipeline,
+      String diffrentiator,
+      String jplQuery,
+      ImmutableMap<String, Object> queryParameters,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName() + " " + diffrentiator,
+            RegistryJpaIO.read(jplQuery, queryParameters, false, type::cast)
+                .withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName() + " " + diffrentiator,
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  // TODO(b/205988530): don't use beam serializablefunction, make one that extends Java's Function.
+  private static <R, I, T> SerializableFunction<R, T> compose(
+      SerializableFunction<R, I> f1, SerializableFunction<I, T> f2) {
+    return r -> f2.apply(f1.apply(r));
+  }
+
+  static <R, T> Read<R, T> buildEppResourceQueryWithTimeFilter(
+      Class<R> entityType,
+      Class<T> castOutputAsType,
+      Optional<String> snapshotId,
+      DateTime compareStartTime) {
+    String tableName = getJpaEntityName(entityType);
+    String jpql =
+        String.format("select c from %s c where :compareStartTime < updateTimestamp", tableName);
+    return RegistryJpaIO.read(
+            jpql,
+            ImmutableMap.of("compareStartTime", UpdateAutoTimestamp.create(compareStartTime)),
+            false,
+            (R x) -> castOutputAsType.cast(x))
+        .withSnapshot(snapshotId.orElse(null));
+  }
+
+  static PartitionedQuery buildPartitonedHistoryQuery(
+      Class<?> entityType, Optional<DateTime> compareStartTime) {
+    String tableName = getJpaEntityName(entityType);
+    Verify.verify(
+        !Strings.isNullOrEmpty(tableName), "Invalid entity type %s", entityType.getSimpleName());
+    long medianId =
+        getMedianIdForHistoryTable(tableName)
+            .orElseThrow(() -> new IllegalStateException("Not a valid database: no " + tableName));
+    String firstHalfQuery = String.format("select c from %s c where id <= :historyId", tableName);
+    String secondHalfQuery = String.format("select c from %s c where id > :historyId", tableName);
+    if (compareStartTime.isPresent()) {
+      String timeFilter = "  and :compareStartTime  < modificationTime";
+      firstHalfQuery += timeFilter;
+      secondHalfQuery += timeFilter;
+      return PartitionedQuery.createPartitionedQuery(
+          firstHalfQuery,
+          secondHalfQuery,
+          ImmutableMap.of("historyId", medianId, "compareStartTime", compareStartTime.get()));
+    } else {
+      return PartitionedQuery.createPartitionedQuery(
+          firstHalfQuery, secondHalfQuery, ImmutableMap.of("historyId", medianId));
+    }
+  }
+
+  private static String getJpaEntityName(Class entityType) {
+    Entity entityAnnotation = (Entity) entityType.getAnnotation(Entity.class);
+    checkState(
+        entityAnnotation != null, "Unexpected non-entity type %s", entityType.getSimpleName());
+    return Strings.isNullOrEmpty(entityAnnotation.name())
+        ? entityType.getSimpleName()
+        : entityAnnotation.name();
+  }
+
+  /** Contains two queries that partition the target table in two. */
+  @AutoValue
+  abstract static class PartitionedQuery {
+    abstract String firstHalfQuery();
+
+    abstract String secondHalfQuery();
+
+    abstract ImmutableMap<String, Object> parameters();
+
+    public static PartitionedQuery createPartitionedQuery(
+        String firstHalfQuery, String secondHalfQuery, ImmutableMap<String, Object> parameters) {
+      return new AutoValue_SqlSnapshots_PartitionedQuery(
+          firstHalfQuery, secondHalfQuery, parameters);
+    }
+  }
+
+  /** Container that receives mixed-typed data and groups them by {@link Class}. */
+  static class TypedClassifier {
+    private final ImmutableSetMultimap<Class<?>, Object> classifiedEntities;
+
+    TypedClassifier(Iterable<Serializable> inputs) {
+      this.classifiedEntities =
+          Streams.stream(inputs)
+              .collect(ImmutableSetMultimap.toImmutableSetMultimap(Object::getClass, x -> x));
+    }
+
+    <T> ImmutableSet<T> getAllOf(Class<T> clazz) {
+      return classifiedEntities.get(clazz).stream()
+          .map(clazz::cast)
+          .collect(ImmutableSet.toImmutableSet());
+    }
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -0,0 +1,244 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.model.server.Lock;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import google.registry.util.RequestStatusChecker;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
+ * (secondary storage).
+ */
+@DeleteAfterMigration
+public class ValidateSqlPipeline {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final Duration COMMITLOG_START_TIME_MARGIN = Duration.standardMinutes(10);
+
+  /**
+   * Name of the lock used by the commitlog replay process.
+   *
+   * <p>See {@link google.registry.backup.ReplayCommitLogsToSqlAction} for more information.
+   */
+  private static final String COMMITLOG_REPLAY_LOCK_NAME = "ReplayCommitLogsToSqlAction";
+
+  private static final Duration REPLAY_LOCK_LEASE_LENGTH = Duration.standardHours(1);
+  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_TIMEOUT =
+      java.time.Duration.ofMinutes(6);
+  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_DELAY =
+      java.time.Duration.ofSeconds(30);
+
+  private final ValidateSqlPipelineOptions options;
+  private final DatastoreSnapshotInfo mostRecentExport;
+
+  public ValidateSqlPipeline(
+      ValidateSqlPipelineOptions options, DatastoreSnapshotInfo mostRecentExport) {
+    this.options = options;
+    this.mostRecentExport = mostRecentExport;
+  }
+
+  void run() {
+    run(Pipeline.create(options));
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    // TODO(weiminyu): ensure migration stage is DATASTORE_PRIMARY or DATASTORE_PRIMARY_READ_ONLY
+    Optional<Lock> lock = acquireCommitLogReplayLock();
+    if (lock.isPresent()) {
+      logger.atInfo().log("Acquired CommitLog Replay lock.");
+    } else {
+      throw new RuntimeException("Failed to acquire CommitLog Replay lock.");
+    }
+
+    try {
+      DateTime latestCommitLogTime =
+          TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+      Preconditions.checkState(
+          latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
+          "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
+      try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+        // Eagerly release the commitlog replay lock so that replay can resume.
+        lock.ifPresent(Lock::releaseSql);
+        lock = Optional.empty();
+
+        setupPipeline(pipeline, Optional.of(databaseSnapshot.getSnapshotId()), latestCommitLogTime);
+        State state = pipeline.run().waitUntilFinish();
+        if (!State.DONE.equals(state)) {
+          throw new IllegalStateException("Unexpected pipeline state: " + state);
+        }
+      }
+    } finally {
+      lock.ifPresent(Lock::releaseSql);
+    }
+  }
+
+  void setupPipeline(
+      Pipeline pipeline, Optional<String> sqlSnapshotId, DateTime latestCommitLogTime) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    Optional<DateTime> compareStartTime =
+        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse);
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minus(COMMITLOG_START_TIME_MARGIN),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS,
+            compareStartTime);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId, compareStartTime);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      PCollectionList.of(datastoreSnapshot.get(tag))
+          .and(cloudSqlSnapshot.get(tag))
+          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
+          .apply(
+              "Assign primary key to merged " + clazz.getSimpleName(),
+              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
+          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  private static Optional<Lock> acquireCommitLogReplayLock() {
+    Stopwatch stopwatch = Stopwatch.createStarted();
+    while (stopwatch.elapsed().minus(REPLAY_LOCK_ACQUIRE_TIMEOUT).isNegative()) {
+      Optional<Lock> lock = tryAcquireCommitLogReplayLock();
+      if (lock.isPresent()) {
+        return lock;
+      }
+      logger.atInfo().log("Failed to acquired CommitLog Replay lock. Will retry...");
+      try {
+        Thread.sleep(REPLAY_LOCK_ACQUIRE_DELAY.toMillis());
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        throw new RuntimeException("Interrupted.");
+      }
+    }
+    return Optional.empty();
+  }
+
+  private static Optional<Lock> tryAcquireCommitLogReplayLock() {
+    return Lock.acquireSql(
+        COMMITLOG_REPLAY_LOCK_NAME,
+        null,
+        REPLAY_LOCK_LEASE_LENGTH,
+        getLockingRequestStatusChecker(),
+        false);
+  }
+
+  /**
+   * Returns a fake implementation of {@link RequestStatusChecker} that is required for lock
+   * acquisition. The default implementation is AppEngine-specific and is unusable on GCE.
+   */
+  private static RequestStatusChecker getLockingRequestStatusChecker() {
+    return new RequestStatusChecker() {
+      @Override
+      public String getLogId() {
+        return "ValidateSqlPipeline";
+      }
+
+      @Override
+      public boolean isRunning(String requestLogId) {
+        return true;
+      }
+    };
+  }
+
+  public static void main(String[] args) {
+    ValidateSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    DatastoreSnapshotInfo mostRecentExport =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder()
+            .getSnapshotInfo();
+
+    new ValidateSqlPipeline(options, mostRecentExport).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -0,0 +1,34 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.model.annotations.DeleteAfterMigration;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.options.Description;
+
+/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
+@DeleteAfterMigration
+public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {
+
+  @Description(
+      "For history entries and EPP resources, only those modified strictly after this time are "
+          + "included in comparison. Value is in ISO8601 format. "
+          + "Other entity types are not affected.")
+  @Nullable
+  String getComparisonStartTimestamp();
+
+  void setComparisonStartTimestamp(String comparisonStartTimestamp);
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -0,0 +1,387 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.initsql.Transforms;
+import google.registry.config.RegistryEnvironment;
+import google.registry.model.BackupGroupRoot;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.contact.ContactBase;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.eppcommon.AuthInfo;
+import google.registry.model.host.HostHistory;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import java.lang.reflect.Field;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Helpers for use by {@link ValidateSqlPipeline}. */
+@DeleteAfterMigration
+final class ValidateSqlUtils {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ValidateSqlUtils() {}
+
+  private static final ImmutableSet<String> PROBER_CELLS = ImmutableSet.of("IQ", "LG", "TL");
+  private static final ImmutableSet<String> PROBER_TYPES =
+      ImmutableSet.of("ANYT", "ANYTES", "CANARY");
+
+  /**
+   * Query template for finding the median value of the {@code history_revision_id} column in one of
+   * the History tables.
+   *
+   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * history tables. Although the {@code repo_id} column is the leading column in the primary keys
+   * of these tables, in practice and with production data, division by {@code history_revision_id}
+   * works slightly faster for unknown reasons.
+   */
+  private static final String MEDIAN_ID_QUERY_TEMPLATE =
+      "SELECT history_revision_id FROM (                                                        "
+          + "  SELECT"
+          + "    ROW_NUMBER() OVER (ORDER BY history_revision_id ASC) AS rownumber,"
+          + "    history_revision_id"
+          + "  FROM \"%TABLE%\""
+          + ") AS foo\n"
+          + "WHERE rownumber in (select count(*) / 2 + 1 from \"%TABLE%\")";
+
+  static Optional<Long> getMedianIdForHistoryTable(String tableName) {
+    Preconditions.checkArgument(
+        tableName.endsWith("History"), "Table must be one of the History tables.");
+    String sqlText = MEDIAN_ID_QUERY_TEMPLATE.replace("%TABLE%", tableName);
+    List results =
+        jpaTm()
+            .transact(() -> jpaTm().getEntityManager().createNativeQuery(sqlText).getResultList());
+    verify(results.size() < 2, "MidPoint query should have at most one result.");
+    if (results.isEmpty()) {
+      return Optional.empty();
+    }
+    return Optional.of(((BigInteger) results.get(0)).longValue());
+  }
+
+  static TupleTag<SqlEntity> createSqlEntityTupleTag(Class<? extends SqlEntity> actualType) {
+    return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
+  }
+
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+    private final HashMap<String, Counter> totalCounters = new HashMap<>();
+    private final HashMap<String, Counter> missingCounters = new HashMap<>();
+    private final HashMap<String, Counter> unequalCounters = new HashMap<>();
+    private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
+
+    private volatile boolean logPrinted = false;
+
+    private String getCounterKey(Class<?> clazz) {
+      return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
+    }
+
+    private synchronized void ensureCounterExists(String counterKey) {
+      if (totalCounters.containsKey(counterKey)) {
+        return;
+      }
+      totalCounters.put(counterKey, Metrics.counter("CompareDB", "Total Compared: " + counterKey));
+      missingCounters.put(
+          counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
+      unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
+      badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
+    }
+
+    /**
+     * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
+     * This will be removed when we start exporting such entities to GCS.
+     */
+    void logDiff(String key, Object entry0, Object entry1) {
+      if (logPrinted) {
+        return;
+      }
+      logPrinted = true;
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      StringBuilder sb = new StringBuilder();
+      fields0.forEach(
+          (field, value) -> {
+            if (fields1.containsKey(field)) {
+              if (!Objects.equals(value, fields1.get(field))) {
+                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
+              }
+            } else {
+              sb.append(field + "Not found in entity 2\n");
+            }
+          });
+      fields1.forEach(
+          (field, value) -> {
+            if (!fields0.containsKey(field)) {
+              sb.append(field + "Not found in entity 1\n");
+            }
+          });
+      logger.atWarning().log(key + "  " + sb.toString());
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+      ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
+
+      verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
+      verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
+
+      String counterKey = getCounterKey(entities.get(0).getClass());
+      ensureCounterExists(counterKey);
+      totalCounters.get(counterKey).inc();
+
+      if (entities.size() == 1) {
+        if (isSpecialCaseProberEntity(entities.get(0))) {
+          return;
+        }
+        missingCounters.get(counterKey).inc();
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
+        }
+        return;
+      }
+      SqlEntity entity0;
+      SqlEntity entity1;
+
+      try {
+        entity0 = normalizeEntity(entities.get(0));
+        entity1 = normalizeEntity(entities.get(1));
+      } catch (Exception e) {
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          badEntityCounters.get(counterKey).inc();
+        }
+        return;
+      }
+
+      if (!Objects.equals(entity0, entity1)) {
+        unequalCounters.get(counterKey).inc();
+        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+      }
+    }
+  }
+
+  static SqlEntity normalizeEntity(SqlEntity sqlEntity) {
+    if (sqlEntity instanceof EppResource) {
+      return normalizeEppResource(sqlEntity);
+    }
+    if (sqlEntity instanceof HistoryEntry) {
+      return (SqlEntity) normalizeHistoryEntry((HistoryEntry) sqlEntity);
+    }
+    return sqlEntity;
+  }
+
+  /**
+   * Normalizes an {@link EppResource} instance for comparison.
+   *
+   * <p>This method may modify the input object using reflection instead of making a copy with
+   * {@code eppResource.asBuilder().build()}, because when {@code eppResource} is a {@link
+   * google.registry.model.domain.DomainBase}, the {@code build} method accesses the Database, which
+   * we want to avoid.
+   */
+  static SqlEntity normalizeEppResource(SqlEntity eppResource) {
+    try {
+      if (isSpecialCaseProberEntity(eppResource)) {
+        // Clearing some timestamps. See isSpecialCaseProberEntity() for reasons.
+        Field lastUpdateTime = BackupGroupRoot.class.getDeclaredField("updateTimestamp");
+        lastUpdateTime.setAccessible(true);
+        lastUpdateTime.set(eppResource, null);
+        Field deletionTime = EppResource.class.getDeclaredField("deletionTime");
+        deletionTime.setAccessible(true);
+        deletionTime.set(eppResource, null);
+      }
+      Field authField =
+          eppResource instanceof DomainContent
+              ? DomainContent.class.getDeclaredField("authInfo")
+              : eppResource instanceof ContactBase
+                  ? ContactBase.class.getDeclaredField("authInfo")
+                  : null;
+      if (authField != null) {
+        authField.setAccessible(true);
+        AuthInfo authInfo = (AuthInfo) authField.get(eppResource);
+        // When AuthInfo is missing, the authInfo field is null if the object is loaded from
+        // Datastore, or a PasswordAuth with null properties if loaded from SQL. In the second case
+        // we set the authInfo field to null.
+        if (authInfo != null
+            && authInfo.getPw() != null
+            && authInfo.getPw().getRepoId() == null
+            && authInfo.getPw().getValue() == null) {
+          authField.set(eppResource, null);
+        }
+      }
+
+      Field field = EppResource.class.getDeclaredField("revisions");
+      field.setAccessible(true);
+      field.set(eppResource, null);
+      return eppResource;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Normalizes a {@link HistoryEntry} for comparison.
+   *
+   * <p>This method modifies the input using reflection because relevant builder methods performs
+   * unwanted checks and changes.
+   */
+  static HistoryEntry normalizeHistoryEntry(HistoryEntry historyEntry) {
+    // History objects from Datastore do not have details of their EppResource objects
+    // (domainContent, contactBase, hostBase).
+    try {
+      if (historyEntry instanceof DomainHistory) {
+        Field domainContent = DomainHistory.class.getDeclaredField("domainContent");
+        domainContent.setAccessible(true);
+        domainContent.set(historyEntry, null);
+        // Convert empty domainTransactionRecords to null for comparison.
+        Field domainTransactionRecords =
+            HistoryEntry.class.getDeclaredField("domainTransactionRecords");
+        domainTransactionRecords.setAccessible(true);
+        Set<?> domainTransactionRecordsValue = (Set<?>) domainTransactionRecords.get(historyEntry);
+        if (domainTransactionRecordsValue != null && domainTransactionRecordsValue.isEmpty()) {
+          domainTransactionRecords.set(historyEntry, null);
+        }
+        // DomainHistory in Datastore does not have the following properties either:
+        Field nsHosts = DomainHistory.class.getDeclaredField("nsHosts");
+        nsHosts.setAccessible(true);
+        nsHosts.set(historyEntry, null);
+        Field dsDataHistories = DomainHistory.class.getDeclaredField("dsDataHistories");
+        dsDataHistories.setAccessible(true);
+        dsDataHistories.set(historyEntry, null);
+        Field gracePeriodHistories = DomainHistory.class.getDeclaredField("gracePeriodHistories");
+        gracePeriodHistories.setAccessible(true);
+        gracePeriodHistories.set(historyEntry, null);
+      } else if (historyEntry instanceof ContactHistory) {
+        Field contactBase = ContactHistory.class.getDeclaredField("contactBase");
+        contactBase.setAccessible(true);
+        contactBase.set(historyEntry, null);
+      } else if (historyEntry instanceof HostHistory) {
+        Field hostBase = HostHistory.class.getDeclaredField("hostBase");
+        hostBase.setAccessible(true);
+        hostBase.set(historyEntry, null);
+      }
+      return historyEntry;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Returns {@code true} if {@code entity} is created by the prober and needs special treatment.
+   *
+   * <p>{@link EppResource} entities created by the prober are deleted by a cron job that bypasses
+   * the CommitLog mechanism. As a result, their deletions are not propagated to SQL, creating two
+   * types of mismatches: an entity exists in both databases but differs in lastUpdateTime and
+   * deletionTime; an entity only exists in the SQL database.
+   *
+   * <p>In production, there are few placeholder {@link Registrar registrars} that do not exist in
+   * Datastore. They were manually created to in SQL to solve a one-time problem (see b/187946868
+   * for details). They can be ignored in the database comparison.
+   */
+  static boolean isSpecialCaseProberEntity(Object entity) {
+    if (entity instanceof EppResource) {
+      EppResource host = (EppResource) entity;
+      if (host.getPersistedCurrentSponsorRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (entity instanceof HistoryEntry) {
+      HistoryEntry historyEntry = (HistoryEntry) entity;
+      if (historyEntry.getRegistrarId().startsWith("prober-")) {
+        // Not all prober entities have "prober-" as registrar prefix.
+        return true;
+      }
+      if (Objects.equals(historyEntry.getReason(), "Deletion of prober data")) {
+        // Soft-delete event in Datastore that is not propagated to SQL.
+        return true;
+      }
+    }
+    if (entity instanceof DomainHistory) {
+      DomainHistory domainHistory = (DomainHistory) entity;
+      if (domainHistory.getDomainContent().isPresent()
+          && domainHistory.getDomainContent().get().getDomainName().startsWith("prober-")) {
+        // Asynchronously replicated event in SQL.
+        return true;
+      }
+      if (domainHistory.getDomainRepoId() != null) {
+        // Some synthetic events only have domainRepoId.
+        String repoId = domainHistory.getDomainRepoId();
+        if (Transforms.IGNORED_DOMAINS.contains(repoId)) {
+          return true;
+        }
+        String suffix = repoId.substring(repoId.indexOf('-') + 1);
+        String cell = suffix.substring(0, 2);
+        suffix = suffix.substring(2);
+        if (PROBER_CELLS.contains(cell) && PROBER_TYPES.contains(suffix)) {
+          return true;
+        }
+      }
+    }
+    if (entity instanceof ContactHistory) {
+      if (Transforms.IGNORED_CONTACTS.contains(((ContactHistory) entity).getContactRepoId())) {
+        return true;
+      }
+    }
+    if (entity instanceof HostHistory) {
+      if (Transforms.IGNORED_HOSTS.contains(((HostHistory) entity).getHostRepoId())) {
+        return true;
+      }
+    }
+    if (entity instanceof BillingEvent) {
+      BillingEvent event = (BillingEvent) entity;
+      if (event.getRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (entity instanceof PollMessage) {
+      if (((PollMessage) entity).getRegistrarId().startsWith("prober-")) {
+        return true;
+      }
+    }
+    if (RegistryEnvironment.get().equals(RegistryEnvironment.PRODUCTION)
+        && entity instanceof Registrar) {
+      Registrar registrar = (Registrar) entity;
+      if (registrar.getRegistrarId().startsWith("prober-wj-")) {
+        return true;
+      }
+    }
+    return false;
+  }
+}