Update the initial value for lastExpiringCertNotificationSentDate to START_OF_TIME (#1321 )

* Update the initial value for lastExpiringCertNotificationSentDate to START_OF_TIME
Rename client ID to registrar ID in most places (#1317 )
2026-03-26 20:35:18 +00:00 · 2021-09-16 13:06:47 -04:00 · 2021-09-16 12:57:43 -04:00 · 2021-09-16 10:44:53 -04:00 · 2021-09-14 14:54:21 -04:00 · 2021-09-14 14:07:26 -04:00
1022 changed files with 31717 additions and 26880 deletions
--- a/.gcloudignore
+++ b/.gcloudignore
@@ -1,4 +1,5 @@
 python/
+node_modules/
 **/build/
 **/out/
 .*/
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 ######################################################################
 # Java Ignores

+gjf.out
 *.class

 # Mobile Tools for Java (J2ME)
@@ -111,3 +112,7 @@ core/**/registrar_bin*.js
 core/**/registrar_dbg*.js
 core/**/registrar_bin*.css
 core/**/registrar_dbg*.css
+
+# Appengine generated files
+core/WEB-INF/appengine-generated/*.bin
+core/WEB-INF/appengine-generated/*.xml
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -0,0 +1,4 @@
+To report a security issue, please use http://g.co/vulnz. We use
+http://g.co/vulnz for our intake, and do coordination and disclosure here on
+GitHub (including using GitHub Security Advisory). The Google Security Team will
+respond within 5 working days of your report on g.co/vulnz.
--- a/build.gradle
+++ b/build.gradle
@@ -318,7 +318,7 @@ subprojects {
  // expose to users.
  if (project.name != 'docs') {
    javadocSource << project.sourceSets.main.allJava
-    javadocClasspath << project.sourceSets.main.compileClasspath
+    javadocClasspath << project.sourceSets.main.runtimeClasspath
    javadocClasspath << "${buildDir}/generated/sources/annotationProcessor/java/main"
    javadocDependentTasks << project.tasks.compileJava
  }
@@ -457,6 +457,8 @@ task javaIncrementalFormatApply {
 task javadoc(type: Javadoc) {
  source javadocSource
  classpath = files(javadocClasspath)
+  // Exclude the misbehaving generated-by-Soy Java files
+  exclude "**/*SoyInfo.java"
  destinationDir = file("${buildDir}/docs/javadoc")
  options.encoding = "UTF-8"
  // In a lot of places we don't write @return so suppress warnings about that.
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -72,6 +72,7 @@ dependencies {
  compile deps['com.google.auth:google-auth-library-credentials']
  compile deps['com.google.auth:google-auth-library-oauth2-http']
  compile deps['com.google.auto.value:auto-value-annotations']
+  compile deps['com.google.common.html.types:types']
  compile deps['com.google.cloud:google-cloud-core']
  compile deps['com.google.cloud:google-cloud-storage']
  compile deps['com.google.guava:guava']
--- a/buildSrc/gradle/dependency-locks/compile.lockfile
+++ b/buildSrc/gradle/dependency-locks/compile.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/compileClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/compileClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/deploy_jar.lockfile
+++ b/buildSrc/gradle/dependency-locks/deploy_jar.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/runtime.lockfile
+++ b/buildSrc/gradle/dependency-locks/runtime.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testCompile.lockfile
+++ b/buildSrc/gradle/dependency-locks/testCompile.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testRuntime.lockfile
+++ b/buildSrc/gradle/dependency-locks/testRuntime.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java
+++ b/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java
@@ -23,6 +23,7 @@ import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.html.types.TrustedResourceUrls;
 import com.google.template.soy.SoyFileSet;
 import com.google.template.soy.tofu.SoyTofu;
 import google.registry.gradle.plugin.ProjectData.TaskData;
@@ -118,7 +119,7 @@ final class CoverPageGenerator {

    builder.put("projectState", state.toString());
    builder.put("title", title);
-    builder.put("cssFiles", ImmutableSet.of("css/style.css"));
+    builder.put("cssFiles", ImmutableSet.of(TrustedResourceUrls.fromConstant("css/style.css")));
    builder.put("invocation", getInvocation());
    builder.put("tasksByState", getTasksByStateSoyData());
    return builder.build();
--- a/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy
+++ b/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy
@@ -16,7 +16,7 @@

 {template .coverPage}
  {@param title: string}
-  {@param cssFiles: list<string>}
+  {@param cssFiles: list<trusted_resource_uri>}
  {@param projectState: string}
  {@param invocation: string}
  {@param tasksByState: map<string, list<[uniqueName: string, description: string, log: string, reports: map<string, string>]>>}
--- a/common/gradle/dependency-locks/compile.lockfile
+++ b/common/gradle/dependency-locks/compile.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/compileClasspath.lockfile
+++ b/common/gradle/dependency-locks/compileClasspath.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/default.lockfile
+++ b/common/gradle/dependency-locks/default.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/deploy_jar.lockfile
+++ b/common/gradle/dependency-locks/deploy_jar.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/runtime.lockfile
+++ b/common/gradle/dependency-locks/runtime.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/runtimeClasspath.lockfile
+++ b/common/gradle/dependency-locks/runtimeClasspath.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/testCompile.lockfile
+++ b/common/gradle/dependency-locks/testCompile.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testCompileClasspath.lockfile
+++ b/common/gradle/dependency-locks/testCompileClasspath.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testRuntime.lockfile
+++ b/common/gradle/dependency-locks/testRuntime.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testRuntimeClasspath.lockfile
+++ b/common/gradle/dependency-locks/testRuntimeClasspath.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingCompile.lockfile
+++ b/common/gradle/dependency-locks/testingCompile.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingCompileClasspath.lockfile
+++ b/common/gradle/dependency-locks/testingCompileClasspath.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingRuntime.lockfile
+++ b/common/gradle/dependency-locks/testingRuntime.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile
+++ b/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/src/main/java/google/registry/util/DateTimeUtils.java
+++ b/common/src/main/java/google/registry/util/DateTimeUtils.java
@@ -100,6 +100,15 @@ public class DateTimeUtils {
    return ZonedDateTime.ofInstant(instant, ZoneId.of(dateTime.getZone().getID()).normalized());
  }

+  /**
+   * Converts a Joda {@link DateTime} object to an equivalent java.time {@link ZonedDateTime}
+   * object.
+   */
+  public static ZonedDateTime toZonedDateTime(DateTime dateTime, ZoneId zoneId) {
+    java.time.Instant instant = java.time.Instant.ofEpochMilli(dateTime.getMillis());
+    return ZonedDateTime.ofInstant(instant, zoneId);
+  }
+
  /**
   * Converts a java.time {@link ZonedDateTime} object to an equivalent Joda {@link DateTime}
   * object.
--- a/common/src/testing/java/google/registry/testing/FakeClock.java
+++ b/common/src/testing/java/google/registry/testing/FakeClock.java
@@ -22,6 +22,7 @@ import google.registry.util.Clock;
 import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.concurrent.ThreadSafe;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 import org.joda.time.ReadableDuration;
 import org.joda.time.ReadableInstant;

@@ -81,4 +82,14 @@ public final class FakeClock implements Clock {
  public void setTo(ReadableInstant time) {
    currentTimeMillis.set(time.getMillis());
  }
+
+  /** Invokes {@link #setAutoIncrementStep} with one millisecond-step. */
+  public FakeClock setAutoIncrementByOneMilli() {
+    return setAutoIncrementStep(Duration.millis(1));
+  }
+
+  /** Disables the auto-increment mode. */
+  public FakeClock disableAutoIncrement() {
+    return setAutoIncrementStep(Duration.ZERO);
+  }
 }
--- a/config/nom_build.py
+++ b/config/nom_build.py
@@ -256,6 +256,7 @@ GRADLE_FLAGS = [
               'Specify a task to be excluded from execution.',
               True),
 ]
+
 def generate_gradle_properties() -> str:
    """Returns the expected contents of gradle.properties."""
    out = io.StringIO()
@@ -270,7 +271,7 @@ def generate_gradle_properties() -> str:
 def get_root() -> str:
    """Returns the root of the nomulus build tree."""
    cur_dir = os.getcwd()
-    if not os.path.exists(os.path.join(cur_dir, '.git')) or \
+    if not os.path.exists(os.path.join(cur_dir, 'buildSrc')) or \
       not os.path.exists(os.path.join(cur_dir, 'core')) or \
       not os.path.exists(os.path.join(cur_dir, 'gradle.properties')):
        raise Exception('You must run this script from the root directory')
--- a/config/presubmits.py
+++ b/config/presubmits.py
@@ -79,9 +79,9 @@ PRESUBMITS = {
        r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
        ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
            ".git", "/build/", "/generated/", "/generated_tests/",
-            "node_modules/", "JUnitBackports.java", "registrar_bin.",
-            "registrar_dbg.", "google-java-format-diff.py",
-            "nomulus.golden.sql", "soyutils_usegoog.js"
+            "node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
+            "registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
+            "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
        }, REQUIRED):
        "File did not include the license header.",

@@ -202,6 +202,8 @@ PRESUBMITS = {
        "java",
        # ActivityReportingQueryBuilder deals with Dremel queries
        {"src/test", "ActivityReportingQueryBuilder.java",
+         # This class contains helper method to make queries in Beam.
+         "RegistryJpaIO.java",
         # TODO(b/179158393): Remove everything below, which should be done
         # using Criteria
         "ForeignKeyIndex.java",
@@ -213,6 +215,8 @@ PRESUBMITS = {
         "RdapDomainSearchAction.java",
         "RdapNameserverSearchAction.java",
         "RdapSearchActionBase.java",
+         "ReadOnlyCheckingEntityManager.java",
+         "RegistryQuery",
         },
    ):
        "The first String parameter to EntityManager.create(Native)Query "
--- a/core/WEB-INF/appengine-generated/local_db.bin
+++ b/core/WEB-INF/appengine-generated/local_db.bin
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -44,7 +44,6 @@ def outcastTestPatterns = [
    "google/registry/flows/domain/DomainCreateFlowTest.*",
    "google/registry/flows/domain/DomainUpdateFlowTest.*",
    "google/registry/tools/CreateDomainCommandTest.*",
-    "google/registry/tools/server/CreatePremiumListActionTest.*",
 ]

 // Tests that fail when running Gradle in a docker container, e. g. when
@@ -70,15 +69,14 @@ def dockerIncompatibleTestPatterns = [
 // Nomulus classes, e.g., threads and objects retained by frameworks.
 // TODO(weiminyu): identify cause and fix offending tests.
 def fragileTestPatterns = [
-    // Problem seems to lie with AppEngine TaskQueue for test.
-    "google/registry/cron/TldFanoutActionTest.*",
    // Test Datastore inexplicably aborts transaction.
    "google/registry/model/tmch/ClaimsListShardTest.*",
-    // Creates large object (64MBytes), occasionally throws OOM error.
-    "google/registry/model/server/KmsSecretRevisionTest.*",
    // Changes cache timeouts and for some reason appears to have contention
    // with other tests.
    "google/registry/whois/WhoisCommandFactoryTest.*",
+    // Currently changes a global configuration parameter that for some reason
+    // results in timestamp inversions for other tests.  TODO(mmuller): fix.
+    "google/registry/flows/host/HostInfoFlowTest.*",
 ] + dockerIncompatibleTestPatterns

 sourceSets {
@@ -184,6 +182,7 @@ dependencies {
  compile deps['com.google.monitoring-client:metrics']
  compile deps['com.google.monitoring-client:stackdriver']
  compile deps['com.google.api-client:google-api-client-java6']
+  compile deps['com.google.api.grpc:proto-google-cloud-tasks-v2']
  compile deps['com.google.apis:google-api-services-admin-directory']
  compile deps['com.google.apis:google-api-services-appengine']
  compile deps['com.google.apis:google-api-services-bigquery']
@@ -194,6 +193,7 @@ dependencies {
  compile deps['com.google.apis:google-api-services-groupssettings']
  compile deps['com.google.apis:google-api-services-monitoring']
  compile deps['com.google.apis:google-api-services-sheets']
+  compile deps['com.google.apis:google-api-services-storage']
  testCompile deps['com.google.appengine:appengine-api-stubs']
  compile deps['com.google.appengine.tools:appengine-gcs-client']
  compile deps['com.google.appengine.tools:appengine-mapreduce']
@@ -215,9 +215,13 @@ dependencies {
  compile deps['com.google.flogger:flogger']
  runtime deps['com.google.flogger:flogger-system-backend']
  compile deps['com.google.guava:guava']
+  compile deps['com.google.protobuf:protobuf-java']
  gradleLint.ignore('unused-dependency') {
    compile deps['com.google.gwt:gwt-user']
  }
+  compile deps['com.google.cloud:google-cloud-core']
+  compile deps['com.google.cloud:google-cloud-storage']
+  compile deps['com.google.cloud:google-cloud-tasks']
  compile deps['com.google.http-client:google-http-client']
  compile deps['com.google.http-client:google-http-client-appengine']
  compile deps['com.google.http-client:google-http-client-jackson2']
@@ -312,11 +316,13 @@ dependencies {
  annotationProcessor project(':processor')
  testAnnotationProcessor project(':processor')

+  testCompile deps['com.google.cloud:google-cloud-nio']
  testCompile deps['com.google.appengine:appengine-testing']
  testCompile deps['com.google.guava:guava-testlib']
  testCompile deps['com.google.monitoring-client:contrib']
  testCompile deps['com.google.truth:truth']
  testCompile deps['com.google.truth.extensions:truth-java8-extension']
+  testCompile deps['org.checkerframework:checker-qual']
  testCompile deps['org.hamcrest:hamcrest']
  testCompile deps['org.hamcrest:hamcrest-core']
  testCompile deps['org.hamcrest:hamcrest-library']
@@ -330,7 +336,6 @@ dependencies {
  testCompile deps['org.junit.platform:junit-platform-suite-api']
  testCompile deps['org.mockito:mockito-core']
  testCompile deps['org.mockito:mockito-junit-jupiter']
-  testCompile 'org.checkerframework:checker-qual:3.9.1'
  runtime deps['org.postgresql:postgresql']

  // Indirect dependency found by undeclared-dependency check. Such
@@ -433,12 +438,9 @@ task soyToJava {
  // Relative paths of soy directories.
  def spec11SoyDir = "google/registry/reporting/spec11/soy"
  def toolsSoyDir = "google/registry/tools/soy"
-  def uiSoyDir = "google/registry/ui/soy"
  def registrarSoyDir = "google/registry/ui/soy/registrar"

-  def soyRelativeDirs = [
-          spec11SoyDir, toolsSoyDir, uiSoyDir, registrarSoyDir,
-  ]
+  def soyRelativeDirs = [spec11SoyDir, toolsSoyDir, registrarSoyDir]
  soyRelativeDirs.each {
    inputs.dir "${resourcesSourceDir}/${it}"
    outputs.dir "${generatedDir}/${it}"
@@ -452,7 +454,8 @@ task soyToJava {
              "--outputDirectory", "${outputDirectory}",
              "--javaClassNameSource", "filename",
              "--allowExternalCalls", "true",
-              "--srcs", "${soyFiles.join(',')}"
+              "--srcs", "${soyFiles.join(',')}",
+              "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
    }
  }

@@ -469,14 +472,6 @@ task soyToJava {
                    dir: "${resourcesSourceDir}/${registrarSoyDir}",
                    include: ['**/*.soy']))

-    soyToJava('google.registry.ui.soy',
-            "${generatedDir}/${uiSoyDir}",
-            files {
-              file("${resourcesSourceDir}/${uiSoyDir}").listFiles()
-            }.filter {
-              it.name.endsWith(".soy")
-            })
-
    soyToJava('google.registry.reporting.spec11.soy',
            "${generatedDir}/${spec11SoyDir}",
            fileTree(
@@ -485,42 +480,24 @@ task soyToJava {
  }
 }

-task soyToJS {
-  def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy"
-  def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy"
+task soyToJS(type: JavaExec) {
+  def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy/registrar"
+  def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy/registrar"
  inputs.dir rootSoyDirectory
  outputs.dir outputSoyDirectory

-  ext.soyToJS = { outputDirectory, soyFiles , deps->
-    javaexec {
-      main = "com.google.template.soy.SoyToJsSrcCompiler"
-      classpath configurations.soy
+  def inputSoyFiles = files {
+    file("${rootSoyDirectory}").listFiles()
+  }.filter {
+    it.name.endsWith(".soy")
+  }

-      args "--outputPathFormat", "${outputDirectory}/{INPUT_FILE_NAME}.js",
+  classpath configurations.soy
+  main = "com.google.template.soy.SoyToJsSrcCompiler"
+  args "--outputPathFormat", "${outputSoyDirectory}/{INPUT_FILE_NAME}.js",
          "--allowExternalCalls", "false",
-          "--srcs", "${soyFiles.join(',')}",
-          "--shouldProvideRequireSoyNamespaces", "true",
+          "--srcs", "${inputSoyFiles.join(',')}",
          "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
-      if (deps != "") {
-        args "--deps", "${deps.join(',')}"
-      }
-    }
-  }
-
-  doLast {
-    def rootSoyFiles =
-      fileTree(
-        dir: "${rootSoyDirectory}",
-        include: ['*.soy'])
-
-    soyToJS("${outputSoyDirectory}", rootSoyFiles, "")
-    soyToJS("${outputSoyDirectory}/registrar",
-      files {
-        file("${rootSoyDirectory}/registrar").listFiles()
-      }.filter {
-        it.name.endsWith(".soy")
-      }, rootSoyFiles)
-  }
 }

 task stylesheetsToJavascript {
@@ -603,8 +580,8 @@ task compileProdJS(type: JavaExec) {
  closureArgs << "--generate_exports"

  // manually include all the required js files
-  closureArgs << "--js=${nodeModulesDir}/google-closure-library/**.js"
-  closureArgs << "--js=${jsDir}/soyutils_usegoog.js"
+  closureArgs << "--js=${nodeModulesDir}/google-closure-library/**/*.js"
+  closureArgs << "--js=${jsDir}/*.js"
  closureArgs << "--js=${cssSourceDir}/registrar_bin.css.js"
  closureArgs << "--js=${jsSourceDir}/**.js"
  closureArgs << "--js=${externsDir}/json.js"
@@ -631,15 +608,6 @@ compileProdJS.dependsOn processResources
 compileProdJS.dependsOn processTestResources
 compileProdJS.dependsOn soyToJS

-task karmaTest(type: Exec) {
-  dependsOn ':npmInstall'
-  workingDir rootProject.projectDir
-  executable 'node_modules/karma/bin/karma'
-  args('start', "${project.projectDir}/karma.conf.js")
-}
-
-test.dependsOn karmaTest
-
 // Make testing artifacts available to be depended up on by other projects.
 // TODO: factor out google.registry.testing to be a separate project.
 task testJar(type: Jar) {
@@ -814,6 +782,10 @@ if (environment in ['alpha', 'crash']) {
          mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
          metaData: 'google/registry/beam/invoicing_pipeline_metadata.json'
      ],
+      [
+          mainClass: 'google.registry.beam.rde.RdePipeline',
+          metaData: 'google/registry/beam/rde_pipeline_metadata.json'
+      ],
  ]
  project.tasks.create("stage_beam_pipelines") {
    doLast {
--- a/core/gradle/dependency-locks/annotationProcessor.lockfile
+++ b/core/gradle/dependency-locks/annotationProcessor.lockfile
@@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33
 com.google.dagger:dagger-spi:2.33
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotation:2.3.4
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.errorprone:error_prone_check_api:2.3.4
 com.google.errorprone:error_prone_core:2.3.4
 com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
@@ -32,7 +32,7 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.checkerframework:dataflow:3.0.0
 org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
--- a/core/gradle/dependency-locks/closureCompiler.lockfile
+++ b/core/gradle/dependency-locks/closureCompiler.lockfile
@@ -1,15 +1,4 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-args4j:args4j:2.0.26
-com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.7
-com.google.errorprone:error_prone_annotations:2.3.1
-com.google.guava:guava:25.1-jre
-com.google.j2objc:j2objc-annotations:1.1
-com.google.javascript:closure-compiler-externs:v20190301
-com.google.javascript:closure-compiler:v20190301
-com.google.jsinterop:jsinterop-annotations:1.0.0
-com.google.protobuf:protobuf-java:3.0.2
-org.checkerframework:checker-qual:2.0.0
-org.codehaus.mojo:animal-sniffer-annotations:1.14
+com.google.javascript:closure-compiler:v20210505
--- a/core/gradle/dependency-locks/compile.lockfile
+++ b/core/gradle/dependency-locks/compile.lockfile
@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -76,17 +79,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -97,30 +100,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -132,10 +138,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -149,17 +155,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -220,7 +226,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -257,7 +263,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/compileClasspath.lockfile
+++ b/core/gradle/dependency-locks/compileClasspath.lockfile
@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -75,17 +78,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -96,29 +99,32 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -130,10 +136,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -147,17 +153,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -214,7 +220,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -250,7 +256,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/default.lockfile
+++ b/core/gradle/dependency-locks/default.lockfile
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -102,30 +105,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,10 +143,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -157,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -231,7 +237,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -271,7 +277,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/deploy_jar.lockfile
+++ b/core/gradle/dependency-locks/deploy_jar.lockfile
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -102,30 +105,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,10 +143,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -157,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -230,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -270,7 +276,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/nonprodCompile.lockfile
+++ b/core/gradle/dependency-locks/nonprodCompile.lockfile
@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -76,17 +79,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -97,30 +100,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -132,10 +138,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -149,17 +155,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -220,7 +226,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -257,7 +263,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile
+++ b/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile
@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -75,17 +78,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -96,29 +99,32 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -130,10 +136,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -147,17 +153,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -215,7 +221,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
@@ -251,7 +257,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/nonprodRuntime.lockfile
+++ b/core/gradle/dependency-locks/nonprodRuntime.lockfile
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -101,30 +104,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -136,10 +142,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -156,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -230,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -269,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile
+++ b/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -101,30 +104,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -136,10 +142,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -156,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -230,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -269,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/runtime.lockfile
+++ b/core/gradle/dependency-locks/runtime.lockfile
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -101,30 +104,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -136,10 +142,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -156,17 +162,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -230,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -269,7 +275,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/runtimeClasspath.lockfile
+++ b/core/gradle/dependency-locks/runtimeClasspath.lockfile
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.79.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,17 +83,17 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
 com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -102,30 +105,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
+com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client-jackson2:1.39.0
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -137,10 +143,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -157,17 +163,17 @@ guru.nidi:graphviz-java-all-j2v8:0.17.0
 guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.65
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -230,7 +236,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
@@ -270,7 +276,7 @@ org.testcontainers:database-commons:1.15.2
 org.testcontainers:jdbc:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/soy.lockfile
+++ b/core/gradle/dependency-locks/soy.lockfile
@@ -5,25 +5,25 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.7
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.3.4
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:5.0.1
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.protobuf:protobuf-java:3.13.0
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.checkerframework:checker-qual:3.5.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
--- a/core/gradle/dependency-locks/testAnnotationProcessor.lockfile
+++ b/core/gradle/dependency-locks/testAnnotationProcessor.lockfile
@@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33
 com.google.dagger:dagger-spi:2.33
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotation:2.3.4
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.errorprone:error_prone_check_api:2.3.4
 com.google.errorprone:error_prone_core:2.3.4
 com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
@@ -30,7 +30,7 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.checkerframework:dataflow:3.0.0
 org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
--- a/core/gradle/dependency-locks/testCompile.lockfile
+++ b/core/gradle/dependency-locks/testCompile.lockfile
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -54,12 +54,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -76,7 +79,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -84,10 +87,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -98,31 +101,35 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -134,11 +141,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -157,17 +164,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -198,7 +205,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
@@ -306,7 +313,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/testCompileClasspath.lockfile
+++ b/core/gradle/dependency-locks/testCompileClasspath.lockfile
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -53,12 +53,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -75,7 +78,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -83,10 +86,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -97,30 +100,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -132,11 +139,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -155,17 +162,17 @@ commons-logging:commons-logging:1.2
 dnsjava:dnsjava:3.3.1
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -193,7 +200,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
@@ -300,7 +307,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/testRuntime.lockfile
+++ b/core/gradle/dependency-locks/testRuntime.lockfile
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,7 +83,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -88,10 +91,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -103,31 +106,35 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,11 +146,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -166,17 +173,17 @@ guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -207,7 +214,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -319,7 +326,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/testRuntimeClasspath.lockfile
+++ b/core/gradle/dependency-locks/testRuntimeClasspath.lockfile
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -58,12 +58,15 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
-com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
+com.google.api.grpc:proto-google-cloud-tasks-v2beta3:0.89.2
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
+com.google.api:gax-grpc:1.66.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,7 +83,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -88,10 +91,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -103,31 +106,35 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
+com.google.cloud:google-cloud-tasks:1.33.2
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,11 +146,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -166,17 +173,17 @@ guru.nidi:graphviz-java:0.17.0
 io.dropwizard.metrics:metrics-core:3.2.6
 io.github.classgraph:classgraph:4.8.102
 io.github.java-diff-utils:java-diff-utils:4.9
-io.grpc:grpc-alts:1.36.0
-io.grpc:grpc-api:1.36.0
-io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
-io.grpc:grpc-core:1.36.0
-io.grpc:grpc-grpclb:1.36.0
-io.grpc:grpc-netty-shaded:1.36.0
+io.grpc:grpc-alts:1.39.0
+io.grpc:grpc-api:1.39.0
+io.grpc:grpc-auth:1.39.0
+io.grpc:grpc-context:1.39.0
+io.grpc:grpc-core:1.39.0
+io.grpc:grpc-grpclb:1.39.0
+io.grpc:grpc-netty-shaded:1.39.0
 io.grpc:grpc-netty:1.32.2
-io.grpc:grpc-protobuf-lite:1.36.0
-io.grpc:grpc-protobuf:1.36.0
-io.grpc:grpc-stub:1.36.0
+io.grpc:grpc-protobuf-lite:1.39.0
+io.grpc:grpc-protobuf:1.39.0
+io.grpc:grpc-stub:1.39.0
 io.netty:netty-buffer:4.1.51.Final
 io.netty:netty-codec-http2:4.1.51.Final
 io.netty:netty-codec-http:4.1.51.Final
@@ -207,7 +214,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -320,7 +327,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/karma.conf.js
+++ b/core/karma.conf.js
@@ -1,75 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-process.env.CHROME_BIN = require('puppeteer').executablePath()
-
-module.exports = function(config) {
-  config.set({
-    basePath: '..',
-    browsers: ['ChromeHeadlessNoSandbox'],
-    customLaunchers: {
-      ChromeHeadlessNoSandbox: {
-        base: 'ChromeHeadless',
-        flags: ['--no-sandbox']
-      }
-    },
-    frameworks: ['jasmine', 'closure'],
-    singleRun: true,
-    autoWatch: false,
-    files: [
-      'node_modules/google-closure-library/closure/goog/base.js',
-      'core/src/test/javascript/**/*_test.js',
-      {
-        pattern: 'core/src/test/javascript/**/!(*_test).js',
-        included: false
-      },
-      {
-        pattern: 'core/src/main/javascript/**/*.js',
-        included: false
-      },
-      {
-        pattern: 'core/build/generated/sources/custom/java/main/**/*.soy.js',
-        included: false
-      },
-      {
-        pattern: 'node_modules/google-closure-library/closure/goog/deps.js',
-        included: false,
-        served: false
-      },
-      {
-        pattern: 'node_modules/google-closure-library/closure/goog/**/*.js',
-        included: false
-      },
-      {
-        pattern: 'core/build/resources/main/google/registry/ui/assets/images/*.png',
-        included: false
-      },
-      {
-        pattern: 'core/build/resources/main/google/registry/ui/assets/images/icons/svg/*.svg',
-        included: false
-      }
-    ],
-    preprocessors: {
-      'node_modules/google-closure-library/closure/goog/deps.js': ['closure', 'closure-deps'],
-      'node_modules/google-closure-library/closure/goog/base.js': ['closure'],
-      'node_modules/google-closure-library/closure/**/*.js': ['closure'],
-      'core/src/*/javascript/**/*.js': ['closure'],
-      'core/build/generated/sources/custom/java/main/**/*.soy.js': ['closure'],
-    },
-    proxies: {
-      "/assets/": "/base/core/build/resources/main/google/registry/ui/assets/"
-    }
-  });
-};
-
--- a/core/src/main/java/google/registry/backup/BackupModule.java
+++ b/core/src/main/java/google/registry/backup/BackupModule.java
@@ -18,8 +18,10 @@ import static com.google.appengine.api.ThreadManager.currentRequestThreadFactory
 import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
+import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
+import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static java.util.concurrent.Executors.newFixedThreadPool;
@@ -32,6 +34,9 @@ import google.registry.cron.CommitLogFanoutAction;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
 import java.lang.annotation.Documented;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import javax.inject.Qualifier;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.DateTime;
@@ -75,6 +80,12 @@ public final class BackupModule {
    return extractRequiredDatetimeParameter(req, UPPER_CHECKPOINT_TIME_PARAM);
  }

+  @Provides
+  @Parameter(BUCKET_OVERRIDE_PARAM)
+  static Optional<String> provideBucketOverride(HttpServletRequest req) {
+    return extractOptionalParameter(req, BUCKET_OVERRIDE_PARAM);
+  }
+
  @Provides
  @Parameter(FROM_TIME_PARAM)
  static DateTime provideFromTime(HttpServletRequest req) {
@@ -92,4 +103,9 @@ public final class BackupModule {
  static ListeningExecutorService provideListeningExecutorService() {
    return listeningDecorator(newFixedThreadPool(NUM_THREADS, currentRequestThreadFactory()));
  }
+
+  @Provides
+  static ScheduledExecutorService provideScheduledExecutorService() {
+    return Executors.newSingleThreadScheduledExecutor();
+  }
 }
--- a/core/src/main/java/google/registry/backup/BackupUtils.java
+++ b/core/src/main/java/google/registry/backup/BackupUtils.java
@@ -14,7 +14,7 @@

 package google.registry.backup;

-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;

 import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.common.collect.AbstractIterator;
@@ -45,7 +45,7 @@ public class BackupUtils {
   * {@link OutputStream} in delimited protocol buffer format.
   */
  static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(ofy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
  }

  /**
@@ -56,19 +56,25 @@ public class BackupUtils {
   *
   * <p>The iterator reads from the stream on demand, and as such will fail if the stream is closed.
   */
-  public static Iterator<ImmutableObject> createDeserializingIterator(final InputStream input) {
+  public static Iterator<ImmutableObject> createDeserializingIterator(
+      final InputStream input, boolean withAppIdOverride) {
    return new AbstractIterator<ImmutableObject>() {
      @Override
      protected ImmutableObject computeNext() {
        EntityProto proto = new EntityProto();
-        if (proto.parseDelimitedFrom(input)) {  // False means end of stream; other errors throw.
-          return ofy().load().fromEntity(EntityTranslator.createFromPb(proto));
+        if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw.
+          if (withAppIdOverride) {
+            proto = EntityImports.fixEntity(proto);
+          }
+          return auditedOfy().load().fromEntity(EntityTranslator.createFromPb(proto));
        }
        return endOfData();
-      }};
+      }
+    };
  }

  public static ImmutableList<ImmutableObject> deserializeEntities(byte[] bytes) {
-    return ImmutableList.copyOf(createDeserializingIterator(new ByteArrayInputStream(bytes)));
+    return ImmutableList.copyOf(
+        createDeserializingIterator(new ByteArrayInputStream(bytes), false));
  }
 }
--- a/core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java
+++ b/core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java
@@ -18,7 +18,7 @@ import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;

@@ -64,8 +64,7 @@ public final class CommitLogCheckpointAction implements Runnable {
    final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
    logger.atInfo().log(
        "Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
-    tm()
-        .transact(
+    tm().transact(
            () -> {
              DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
              if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
@@ -73,7 +72,7 @@ public final class CommitLogCheckpointAction implements Runnable {
                    "Newer checkpoint already written at time: %s", lastWrittenTime);
                return;
              }
-              ofy()
+              auditedOfy()
                  .saveWithoutBackup()
                  .entities(
                      checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
--- a/core/src/main/java/google/registry/backup/CommitLogImports.java
+++ b/core/src/main/java/google/registry/backup/CommitLogImports.java
@@ -59,7 +59,7 @@ public final class CommitLogImports {
      InputStream inputStream) {
    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
        InputStream input = new BufferedInputStream(inputStream)) {
-      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
      checkState(commitLogs.hasNext());
      checkState(commitLogs.next() instanceof CommitLogCheckpoint);

--- a/core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java
+++ b/core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java
@@ -17,7 +17,7 @@ package google.registry.backup;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static java.lang.Boolean.FALSE;
 import static java.lang.Boolean.TRUE;
@@ -66,6 +66,8 @@ import org.joda.time.Duration;
    service = Action.Service.BACKEND,
    path = "/_dr/task/deleteOldCommitLogs",
    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+// No longer needed in SQL. Subject to future removal.
+@Deprecated
 public final class DeleteOldCommitLogsAction implements Runnable {

  private static final int NUM_MAP_SHARDS = 20;
@@ -75,9 +77,17 @@ public final class DeleteOldCommitLogsAction implements Runnable {
  @Inject MapreduceRunner mrRunner;
  @Inject Response response;
  @Inject Clock clock;
-  @Inject @Config("commitLogDatastoreRetention") Duration maxAge;
-  @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
-  @Inject DeleteOldCommitLogsAction() {}
+
+  @Inject
+  @Config("commitLogDatastoreRetention")
+  Duration maxAge;
+
+  @Inject
+  @Parameter(PARAM_DRY_RUN)
+  boolean isDryRun;
+
+  @Inject
+  DeleteOldCommitLogsAction() {}

  @Override
  public void run() {
@@ -138,12 +148,12 @@ public final class DeleteOldCommitLogsAction implements Runnable {
      // If it isn't a Key<CommitLogManifest> then it should be an EppResource, which we need to
      // load to emit the revisions.
      //
-      Object object = ofy().load().key(key).now();
+      Object object = auditedOfy().load().key(key).now();
      checkNotNull(object, "Received a key to a missing object. key: %s", key);
      checkState(
          object instanceof EppResource,
          "Received a key to an object that isn't EppResource nor CommitLogManifest."
-          + " Key: %s object type: %s",
+              + " Key: %s object type: %s",
          key,
          object.getClass().getName());

@@ -224,8 +234,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
   * OK to delete this manifestKey. If even one source returns "false" (meaning "it's not OK to
   * delete this manifest") then it won't be deleted.
   */
-  static class DeleteOldCommitLogsReducer
-      extends Reducer<Key<CommitLogManifest>, Boolean, Void> {
+  static class DeleteOldCommitLogsReducer extends Reducer<Key<CommitLogManifest>, Boolean, Void> {

    private static final long serialVersionUID = -4918760187627937268L;

@@ -241,12 +250,12 @@ public final class DeleteOldCommitLogsAction implements Runnable {
      }

      public abstract Status status();
+
      public abstract int numDeleted();

      static DeletionResult create(Status status, int numDeleted) {
-        return
-            new AutoValue_DeleteOldCommitLogsAction_DeleteOldCommitLogsReducer_DeletionResult(
-                status, numDeleted);
+        return new AutoValue_DeleteOldCommitLogsAction_DeleteOldCommitLogsReducer_DeletionResult(
+            status, numDeleted);
      }
    }

@@ -257,8 +266,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {

    @Override
    public void reduce(
-        final Key<CommitLogManifest> manifestKey,
-        ReducerInput<Boolean> canDeleteVerdicts) {
+        final Key<CommitLogManifest> manifestKey, ReducerInput<Boolean> canDeleteVerdicts) {
      ImmutableMultiset<Boolean> canDeleteMultiset = ImmutableMultiset.copyOf(canDeleteVerdicts);
      if (canDeleteMultiset.count(TRUE) > 1) {
        getContext().incrementCounter("commit log manifests incorrectly mapped multiple times");
@@ -267,47 +275,54 @@ public final class DeleteOldCommitLogsAction implements Runnable {
        getContext().incrementCounter("commit log manifests referenced multiple times");
      }
      if (canDeleteMultiset.contains(FALSE)) {
-        getContext().incrementCounter(
-            canDeleteMultiset.contains(TRUE)
-            ? "old commit log manifests still referenced"
-            : "new (or nonexistent) commit log manifests referenced");
-        getContext().incrementCounter(
-            "EPP resource revisions handled",
-            canDeleteMultiset.count(FALSE));
+        getContext()
+            .incrementCounter(
+                canDeleteMultiset.contains(TRUE)
+                    ? "old commit log manifests still referenced"
+                    : "new (or nonexistent) commit log manifests referenced");
+        getContext()
+            .incrementCounter("EPP resource revisions handled", canDeleteMultiset.count(FALSE));
        return;
      }

-      DeletionResult deletionResult = tm().transactNew(() -> {
-        CommitLogManifest manifest = ofy().load().key(manifestKey).now();
-        // It is possible that the same manifestKey was run twice, if a shard had to be restarted
-        // or some weird failure. If this happens, we want to exit immediately.
-        // Note that this can never happen in dryRun.
-        if (manifest == null) {
-          return DeletionResult.create(DeletionResult.Status.ALREADY_DELETED, 0);
-        }
-        // Doing a sanity check on the date. This is the only place we use the CommitLogManifest,
-        // so maybe removing this test will improve performance. However, unless it's proven that
-        // the performance boost is significant (and we've tested this enough to be sure it never
-        // happens)- the safty of "let's not delete stuff we need from prod" is more important.
-        if (manifest.getCommitTime().isAfter(deletionThreshold)) {
-          return DeletionResult.create(DeletionResult.Status.AFTER_THRESHOLD, 0);
-        }
-        Iterable<Key<CommitLogMutation>> commitLogMutationKeys = ofy().load()
-            .type(CommitLogMutation.class)
-            .ancestor(manifestKey)
-            .keys()
-            .iterable();
-        ImmutableList<Key<?>> keysToDelete = ImmutableList.<Key<?>>builder()
-            .addAll(commitLogMutationKeys)
-            .add(manifestKey)
-            .build();
-        // Normally in a dry run we would log the entities that would be deleted, but those can
-        // number in the millions so we skip the logging.
-        if (!isDryRun) {
-          ofy().deleteWithoutBackup().keys(keysToDelete);
-        }
-        return DeletionResult.create(DeletionResult.Status.SUCCESS, keysToDelete.size());
-      });
+      DeletionResult deletionResult =
+          tm().transactNew(
+                  () -> {
+                    CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
+                    // It is possible that the same manifestKey was run twice, if a shard had to be
+                    // restarted or some weird failure. If this happens, we want to exit
+                    // immediately. Note that this can never happen in dryRun.
+                    if (manifest == null) {
+                      return DeletionResult.create(DeletionResult.Status.ALREADY_DELETED, 0);
+                    }
+                    // Doing a sanity check on the date. This is the only place we use the
+                    // CommitLogManifest, so maybe removing this test will improve performance.
+                    // However, unless it's proven that the performance boost is significant (and
+                    // we've tested this enough to be sure it never happens)- the safety of "let's
+                    // not delete stuff we need from prod" is more important.
+                    if (manifest.getCommitTime().isAfter(deletionThreshold)) {
+                      return DeletionResult.create(DeletionResult.Status.AFTER_THRESHOLD, 0);
+                    }
+                    Iterable<Key<CommitLogMutation>> commitLogMutationKeys =
+                        auditedOfy()
+                            .load()
+                            .type(CommitLogMutation.class)
+                            .ancestor(manifestKey)
+                            .keys()
+                            .iterable();
+                    ImmutableList<Key<?>> keysToDelete =
+                        ImmutableList.<Key<?>>builder()
+                            .addAll(commitLogMutationKeys)
+                            .add(manifestKey)
+                            .build();
+                    // Normally in a dry run we would log the entities that would be deleted, but
+                    // those can number in the millions so we skip the logging.
+                    if (!isDryRun) {
+                      auditedOfy().deleteWithoutBackup().keys(keysToDelete);
+                    }
+                    return DeletionResult.create(
+                        DeletionResult.Status.SUCCESS, keysToDelete.size());
+                  });

      switch (deletionResult.status()) {
        case SUCCESS:
--- a/core/src/main/java/google/registry/backup/EntityImports.java
+++ b/core/src/main/java/google/registry/backup/EntityImports.java
@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.apphosting.api.ApiProxy;
+import com.google.storage.onestore.v3.OnestoreEntity;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import com.google.storage.onestore.v3.OnestoreEntity.Path;
+import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
+import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+/** Utilities for handling imported Datastore entities. */
+public class EntityImports {
+
+  /**
+   * Transitively sets the {@code appId} of all keys in a foreign entity to that of the current
+   * system.
+   */
+  public static EntityProto fixEntity(EntityProto entityProto) {
+    String currentAappId = ApiProxy.getCurrentEnvironment().getAppId();
+    if (Objects.equals(currentAappId, entityProto.getKey().getApp())) {
+      return entityProto;
+    }
+    return fixEntity(entityProto, currentAappId);
+  }
+
+  private static EntityProto fixEntity(EntityProto entityProto, String appId) {
+    if (entityProto.hasKey()) {
+      fixKey(entityProto, appId);
+    }
+
+    for (OnestoreEntity.Property property : entityProto.mutablePropertys()) {
+      fixProperty(property, appId);
+    }
+
+    for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
+      fixProperty(property, appId);
+    }
+
+    // CommitLogMutation embeds an entity as bytes, which needs additional fixes.
+    if (isCommitLogMutation(entityProto)) {
+      fixMutationEntityProtoBytes(entityProto, appId);
+    }
+    return entityProto;
+  }
+
+  private static boolean isCommitLogMutation(EntityProto entityProto) {
+    if (!entityProto.hasKey()) {
+      return false;
+    }
+    Path path = entityProto.getKey().getPath();
+    if (path.elementSize() == 0) {
+      return false;
+    }
+    return Objects.equals(
+        path.getElement(path.elementSize() - 1).getType(StandardCharsets.UTF_8),
+        "CommitLogMutation");
+  }
+
+  private static void fixMutationEntityProtoBytes(EntityProto entityProto, String appId) {
+    for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
+      if (Objects.equals(property.getName(), "entityProtoBytes")) {
+        OnestoreEntity.PropertyValue value = property.getValue();
+        EntityProto fixedProto =
+            fixEntity(bytesToEntityProto(value.getStringValueAsBytes()), appId);
+        value.setStringValueAsBytes(fixedProto.toByteArray());
+        return;
+      }
+    }
+  }
+
+  private static void fixKey(EntityProto entityProto, String appId) {
+    entityProto.getMutableKey().setApp(appId);
+  }
+
+  private static void fixKey(ReferenceValue referenceValue, String appId) {
+    referenceValue.setApp(appId);
+  }
+
+  private static void fixProperty(OnestoreEntity.Property property, String appId) {
+    OnestoreEntity.PropertyValue value = property.getMutableValue();
+    if (value.hasReferenceValue()) {
+      fixKey(value.getMutableReferenceValue(), appId);
+      return;
+    }
+    if (property.getMeaningEnum().equals(Meaning.ENTITY_PROTO)) {
+      EntityProto embeddedProto = bytesToEntityProto(value.getStringValueAsBytes());
+      fixEntity(embeddedProto, appId);
+      value.setStringValueAsBytes(embeddedProto.toByteArray());
+    }
+  }
+
+  private static EntityProto bytesToEntityProto(byte[] bytes) {
+    EntityProto entityProto = new EntityProto();
+    boolean isParsed = entityProto.parseFrom(bytes);
+    if (!isParsed) {
+      throw new IllegalStateException("Failed to parse raw bytes as EntityProto.");
+    }
+    return entityProto;
+  }
+}
--- a/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java
+++ b/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java
@@ -25,21 +25,20 @@ import static google.registry.backup.BackupUtils.GcsMetadataKeys.NUM_TRANSACTION
 import static google.registry.backup.BackupUtils.GcsMetadataKeys.UPPER_BOUND_CHECKPOINT;
 import static google.registry.backup.BackupUtils.serializeEntity;
 import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
-import static java.nio.channels.Channels.newOutputStream;
 import static java.util.Comparator.comparingLong;

-import com.google.appengine.tools.cloudstorage.GcsFileOptions;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobId;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
@@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable {

  public static final String DIFF_FILE_PREFIX = "commit_diff_until_";

-  @Inject GcsService gcsService;
+  @Inject GcsUtils gcsUtils;
+
  @Inject @Config("commitLogGcsBucket") String gcsBucket;
  @Inject @Config("commitLogDiffExportBatchSize") int batchSize;
  @Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
@@ -89,23 +89,26 @@ public final class ExportCommitLogDiffAction implements Runnable {
    checkArgument(lowerCheckpointTime.isBefore(upperCheckpointTime));
    // Load the boundary checkpoints - lower is exclusive and may not exist (on the first export,
    // when lowerCheckpointTime is START_OF_TIME), whereas the upper is inclusive and must exist.
-    CommitLogCheckpoint lowerCheckpoint = lowerCheckpointTime.isAfter(START_OF_TIME)
-        ? verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
-        : null;
+    CommitLogCheckpoint lowerCheckpoint =
+        lowerCheckpointTime.isAfter(START_OF_TIME)
+            ? verifyNotNull(
+                auditedOfy().load().key(CommitLogCheckpoint.createKey(lowerCheckpointTime)).now())
+            : null;
    CommitLogCheckpoint upperCheckpoint =
-        verifyNotNull(ofy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());
+        verifyNotNull(
+            auditedOfy().load().key(CommitLogCheckpoint.createKey(upperCheckpointTime)).now());

    // Load the keys of all the manifests to include in this diff.
    List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
    logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
    // Open an output channel to GCS, wrapped in a stream for convenience.
-    try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
-        new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
-        new GcsFileOptions.Builder()
-            .addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
-            .addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
-            .addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
-            .build()))) {
+    try (OutputStream gcsStream =
+        gcsUtils.openOutputStream(
+            BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
+            ImmutableMap.of(
+                LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
+                UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
+                NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
      // Export the upper checkpoint itself.
      serializeEntity(upperCheckpoint, gcsStream);
      // If there are no manifests to export, stop early, now that we've written out the file with
@@ -117,7 +120,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
      // asynchronously load the entities for the next one.
      List<List<Key<CommitLogManifest>>> keyChunks = partition(sortedKeys, batchSize);
      // Objectify's map return type is asynchronous. Calling .values() will block until it loads.
-      Map<?, CommitLogManifest> nextChunkToExport = ofy().load().keys(keyChunks.get(0));
+      Map<?, CommitLogManifest> nextChunkToExport = auditedOfy().load().keys(keyChunks.get(0));
      for (int i = 0; i < keyChunks.size(); i++) {
        // Force the async load to finish.
        Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
@@ -125,10 +128,10 @@ public final class ExportCommitLogDiffAction implements Runnable {
        // Since there is no hard bound on how much data this might be, take care not to let the
        // Objectify session cache fill up and potentially run out of memory. This is the only safe
        // point to do this since at this point there is no async load in progress.
-        ofy().clearSessionCache();
+        auditedOfy().clearSessionCache();
        // Kick off the next async load, which can happen in parallel to the current GCS export.
        if (i + 1 < keyChunks.size()) {
-          nextChunkToExport = ofy().load().keys(keyChunks.get(i + 1));
+          nextChunkToExport = auditedOfy().load().keys(keyChunks.get(i + 1));
        }
        exportChunk(gcsStream, chunkValues);
        logger.atInfo().log("Exported %d manifests", chunkValues.size());
@@ -192,7 +195,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
      return ImmutableSet.of();
    }
    Key<CommitLogBucket> bucketKey = getBucketKey(bucketNum);
-    return ofy().load()
+    return auditedOfy()
+        .load()
        .type(CommitLogManifest.class)
        .ancestor(bucketKey)
        .filterKey(">=", CommitLogManifest.createKey(bucketKey, lowerBound))
@@ -208,7 +212,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
        new ImmutableList.Builder<>();
    for (CommitLogManifest manifest : chunk) {
      entities.add(ImmutableList.of(manifest));
-      entities.add(ofy().load().type(CommitLogMutation.class).ancestor(manifest));
+      entities.add(auditedOfy().load().type(CommitLogMutation.class).ancestor(manifest));
    }
    for (ImmutableObject entity : concat(entities.build())) {
      serializeEntity(entity, gcsStream);
--- a/core/src/main/java/google/registry/backup/GcsDiffFileLister.java
+++ b/core/src/main/java/google/registry/backup/GcsDiffFileLister.java
@@ -15,30 +15,32 @@
 package google.registry.backup;

 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
 import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;

-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
-import com.google.appengine.tools.cloudstorage.ListItem;
-import com.google.appengine.tools.cloudstorage.ListOptions;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.UncheckedExecutionException;
 import google.registry.backup.BackupModule.Backups;
-import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
 import java.io.IOException;
-import java.util.Iterator;
+import java.time.Duration;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.ScheduledExecutorService;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
+import javax.inject.Provider;
 import org.joda.time.DateTime;

 /** Utility class to list commit logs diff files stored on GCS. */
@@ -46,33 +48,49 @@ class GcsDiffFileLister {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

-  @Inject GcsService gcsService;
-  @Inject @Config("commitLogGcsBucket") String gcsBucket;
-  @Inject @Backups ListeningExecutorService executor;
-  @Inject GcsDiffFileLister() {}
+  /** Timeout for retrieving per-file information from GCS. */
+  private static final Duration FILE_INFO_TIMEOUT_DURATION = Duration.ofMinutes(1);
+
+  @Inject GcsUtils gcsUtils;
+
+  @Inject @Backups Provider<ListeningExecutorService> executorProvider;
+  @Inject ScheduledExecutorService scheduledExecutorService;
+
+  @Inject
+  GcsDiffFileLister() {}

  /**
   * Traverses the sequence of diff files backwards from checkpointTime and inserts the file
-   * metadata into "sequence".  Returns true if a complete sequence was discovered, false if one or
+   * metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
   * more files are missing.
+   *
+   * @throws UncheckedExecutionException wrapping a {@link java.util.concurrent.TimeoutException} if
+   *     the GCS call fails to finish within one minute, or wrapping any other exception if
+   *     something else goes wrong.
   */
  private boolean constructDiffSequence(
-      Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
+      String gcsBucket,
+      Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
      DateTime fromTime,
      DateTime lastTime,
-      TreeMap<DateTime, GcsFileMetadata> sequence) {
+      TreeMap<DateTime, BlobInfo> sequence) {
    DateTime checkpointTime = lastTime;
    while (isBeforeOrAt(fromTime, checkpointTime)) {
-      GcsFileMetadata metadata;
-      if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
-        metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
+      BlobInfo blobInfo;
+      if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
+        blobInfo =
+            Futures.getUnchecked(
+                Futures.withTimeout(
+                    upperBoundTimesToBlobInfo.get(checkpointTime),
+                    FILE_INFO_TIMEOUT_DURATION,
+                    scheduledExecutorService));
      } else {
        String filename = DIFF_FILE_PREFIX + checkpointTime;
        logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
-        metadata = getMetadata(filename);
+        blobInfo = getBlobInfo(gcsBucket, filename);

        // If we hit a gap, quit.
-        if (metadata == null) {
+        if (blobInfo == null) {
          logger.atInfo().log(
              "Gap discovered in sequence terminating at %s, missing file: %s",
              sequence.lastKey(), filename);
@@ -80,14 +98,15 @@ class GcsDiffFileLister {
          return false;
        }
      }
-      sequence.put(checkpointTime, metadata);
-      checkpointTime = getLowerBoundTime(metadata);
+      sequence.put(checkpointTime, blobInfo);
+      checkpointTime = getLowerBoundTime(blobInfo);
    }
    logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
    return true;
  }

-  ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
+  ImmutableList<BlobInfo> listDiffFiles(
+      String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
    logger.atInfo().log("Requested restore from time: %s", fromTime);
    if (toTime != null) {
      logger.atInfo().log("  Until time: %s", toTime);
@@ -95,66 +114,74 @@ class GcsDiffFileLister {
    // List all of the diff files on GCS and build a map from each file's upper checkpoint time
    // (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
    // an upper checkpoint time > fromTime.
-    TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
-        = new TreeMap<>();
-    Iterator<ListItem> listItems;
+    TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
+    String commitLogDiffPrefix = getCommitLogDiffPrefix(fromTime, toTime);
+    ImmutableList<String> filenames;
    try {
-      // TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
-      listItems = gcsService.list(
-          gcsBucket,
-          new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
+      filenames =
+          gcsUtils.listFolderObjects(gcsBucket, commitLogDiffPrefix).stream()
+              .map(s -> commitLogDiffPrefix + s)
+              .collect(toImmutableList());
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    DateTime lastUpperBoundTime = START_OF_TIME;
-    while (listItems.hasNext()) {
-      final String filename = listItems.next().getName();
-      DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
-      if (isInRange(upperBoundTime, fromTime, toTime)) {
-        upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
-        lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
-      }
-    }
-    if (upperBoundTimesToMetadata.isEmpty()) {
-      logger.atInfo().log("No files found");
-      return ImmutableList.of();
-    }

-    // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
-    // last file that we found) and finding its previous file until we either run out of files or
-    // get to one that precedes "fromTime".
-    //
-    // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
-    // metadata of a file is sufficient to identify the preceding file, so if we start from the
-    // last file and work backwards we can verify that we have no holes in our chain (although we
-    // may be missing files at the end).
-    TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
-    logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
-    boolean inconsistentFileSet = !constructDiffSequence(
-        upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
-
-    // Verify that all of the elements in the original set are represented in the sequence.  If we
-    // find anything that's not represented, construct a sequence for it.
-    boolean checkForMoreExtraDiffs = true;  // Always loop at least once.
-    while (checkForMoreExtraDiffs) {
-      checkForMoreExtraDiffs = false;
-      for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
-        if (!isInRange(key, fromTime, toTime)) {
-          break;
-        }
-        if (!sequence.containsKey(key)) {
-          constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
-          checkForMoreExtraDiffs = true;
-          inconsistentFileSet = true;
-          break;
+    TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
+    ListeningExecutorService executor = executorProvider.get();
+    try {
+      for (String filename : filenames) {
+        String strippedFilename = filename.replaceFirst(DIFF_FILE_PREFIX, "");
+        DateTime upperBoundTime = DateTime.parse(strippedFilename);
+        if (isInRange(upperBoundTime, fromTime, toTime)) {
+          upperBoundTimesToBlobInfo.put(
+              upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
+          lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
        }
      }
-    }
+      if (upperBoundTimesToBlobInfo.isEmpty()) {
+        logger.atInfo().log("No files found");
+        return ImmutableList.of();
+      }

-    checkState(
-        !inconsistentFileSet,
-        "Unable to compute commit diff history, there are either gaps or forks in the history "
-        + "file set.  Check log for details.");
+      // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e.
+      // the last file that we found) and finding its previous file until we either run out of files
+      // or get to one that precedes "fromTime".
+      //
+      // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
+      // metadata of a file is sufficient to identify the preceding file, so if we start from the
+      // last file and work backwards we can verify that we have no holes in our chain (although we
+      // may be missing files at the end).
+      logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
+      boolean inconsistentFileSet =
+          !constructDiffSequence(
+              gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
+
+      // Verify that all of the elements in the original set are represented in the sequence.  If we
+      // find anything that's not represented, construct a sequence for it.
+      boolean checkForMoreExtraDiffs = true; // Always loop at least once.
+      while (checkForMoreExtraDiffs) {
+        checkForMoreExtraDiffs = false;
+        for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
+          if (!isInRange(key, fromTime, toTime)) {
+            break;
+          }
+          if (!sequence.containsKey(key)) {
+            constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
+            checkForMoreExtraDiffs = true;
+            inconsistentFileSet = true;
+            break;
+          }
+        }
+      }
+
+      checkState(
+          !inconsistentFileSet,
+          "Unable to compute commit diff history, there are either gaps or forks in the history "
+              + "file set.  Check log for details.");
+    } finally {
+      executor.shutdown();
+    }

    logger.atInfo().log(
        "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
@@ -171,15 +198,43 @@ class GcsDiffFileLister {
    return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
  }

-  private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
-    return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
+  private DateTime getLowerBoundTime(BlobInfo blobInfo) {
+    return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
  }

-  private GcsFileMetadata getMetadata(String filename) {
-    try {
-      return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
+  private BlobInfo getBlobInfo(String gcsBucket, String filename) {
+    return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
+  }
+
+  /**
+   * Returns a prefix guaranteed to cover all commit log diff files in the given range.
+   *
+   * <p>The listObjects call can be fairly slow if we search over many thousands or tens of
+   * thousands of files, so we restrict the search space. The commit logs have a file format of
+   * "commit_diff_until_2021-05-11T06:48:00.070Z" so we can often filter down as far as the hour.
+   *
+   * <p>Here, we get the longest prefix possible based on which fields (year, month, day, hour) the
+   * times in question have in common.
+   */
+  @VisibleForTesting
+  static String getCommitLogDiffPrefix(DateTime from, @Nullable DateTime to) {
+    StringBuilder result = new StringBuilder(DIFF_FILE_PREFIX);
+    if (to == null || from.getYear() != to.getYear()) {
+      return result.toString();
    }
+    result.append(from.getYear()).append('-');
+    if (from.getMonthOfYear() != to.getMonthOfYear()) {
+      return result.toString();
+    }
+    result.append(String.format("%02d-", from.getMonthOfYear()));
+    if (from.getDayOfMonth() != to.getDayOfMonth()) {
+      return result.toString();
+    }
+    result.append(String.format("%02dT", from.getDayOfMonth()));
+    if (from.getHourOfDay() != to.getHourOfDay()) {
+      return result.toString();
+    }
+    result.append(String.format("%02d:", from.getHourOfDay()));
+    return result.toString();
  }
 }
--- a/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java
+++ b/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java
@@ -14,70 +14,98 @@

 package google.registry.backup;

+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
+import static google.registry.backup.RestoreCommitLogsAction.DRY_RUN_PARAM;
 import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.DateTimeUtils.isAtOrAfter;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardHours;

 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.Key;
-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobInfo;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.replay.DatastoreEntity;
+import google.registry.model.replay.DatastoreOnlyEntity;
+import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.ReplaySpecializer;
+import google.registry.model.replay.SqlReplayCheckpoint;
 import google.registry.model.server.Lock;
 import google.registry.model.translators.VKeyTranslatorFactory;
 import google.registry.persistence.VKey;
 import google.registry.request.Action;
+import google.registry.request.Action.Method;
+import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.DatastoreOnlyEntity;
-import google.registry.schema.replay.NonReplicatedEntity;
-import google.registry.schema.replay.ReplaySpecializer;
-import google.registry.schema.replay.SqlReplayCheckpoint;
+import google.registry.util.Clock;
 import google.registry.util.RequestStatusChecker;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.channels.Channels;
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.servlet.http.HttpServletResponse;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
+import org.joda.time.Seconds;

 /** Action that replays commit logs to Cloud SQL to keep it up to date. */
@Action(
    service = Action.Service.BACKEND,
    path = ReplayCommitLogsToSqlAction.PATH,
-    method = Action.Method.POST,
+    method = Method.POST,
    automaticallyPrintOk = true,
    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public class ReplayCommitLogsToSqlAction implements Runnable {

  static final String PATH = "/_dr/task/replayCommitLogsToSql";

-  private static final int BLOCK_SIZE =
-      1024 * 1024; // Buffer 1mb at a time, for no particular reason.
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final Duration LEASE_LENGTH = standardHours(1);

-  @Inject GcsService gcsService;
+  private static final Duration LEASE_LENGTH = standardHours(1);
+  // Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
+  // request timeouts
+  private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
+
+  @Inject GcsUtils gcsUtils;
  @Inject Response response;
  @Inject RequestStatusChecker requestStatusChecker;
  @Inject GcsDiffFileLister diffLister;
+  @Inject Clock clock;
+
+  @Inject
+  @Config("commitLogGcsBucket")
+  String gcsBucket;
+
+  /** If true, will exit after logging the commit log files that would otherwise be replayed. */
+  @Inject
+  @Parameter(DRY_RUN_PARAM)
+  boolean dryRun;

  @Inject
  ReplayCommitLogsToSqlAction() {}

  @Override
  public void run() {
-    if (!RegistryConfig.getCloudSqlReplayCommitLogs()) {
-      String message = "ReplayCommitLogsToSqlAction was called but disabled in the config.";
-      logger.atWarning().log(message);
+    DateTime startTime = clock.nowUtc();
+    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(startTime);
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      String message =
+          String.format(
+              "Skipping ReplayCommitLogsToSqlAction because we are in migration phase %s.", state);
+      logger.atInfo().log(message);
      // App Engine will retry on any non-2xx status code, which we don't want in this case.
      response.setStatus(SC_NO_CONTENT);
      response.setPayload(message);
@@ -96,44 +124,120 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
      return;
    }
    try {
-      replayFiles();
-      response.setStatus(HttpServletResponse.SC_OK);
-      logger.atInfo().log("ReplayCommitLogsToSqlAction completed successfully.");
+      logger.atInfo().log("Beginning replay of commit logs.");
+      String resultMessage;
+      if (dryRun) {
+        resultMessage = executeDryRun();
+      } else {
+        resultMessage = replayFiles(startTime);
+      }
+      response.setStatus(SC_OK);
+      response.setPayload(resultMessage);
+      logger.atInfo().log(resultMessage);
+    } catch (Throwable t) {
+      String message = "Errored out replaying files.";
+      logger.atSevere().withCause(t).log(message);
+      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(message);
    } finally {
      lock.ifPresent(Lock::release);
    }
  }

-  private void replayFiles() {
+  private String executeDryRun() {
    // Start at the first millisecond we haven't seen yet
-    DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
-    // If there's an inconsistent file set, this will throw IllegalStateException and the job
-    // will try later -- this is likely because an export hasn't finished yet.
-    ImmutableList<GcsFileMetadata> commitLogFiles =
-        diffLister.listDiffFiles(fromTime, /* current time */ null);
-    for (GcsFileMetadata metadata : commitLogFiles) {
-      // One transaction per GCS file
-      jpaTm().transact(() -> processFile(metadata));
-    }
-    logger.atInfo().log("Replayed %d commit log files to SQL successfully.", commitLogFiles.size());
+    DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    // Search through the end of the hour
+    DateTime searchEndTime =
+        searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
+    ImmutableList<String> fileBatch =
+        diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime).stream()
+            .map(BlobInfo::getName)
+            .collect(toImmutableList());
+    return String.format(
+        "Running in dry-run mode, the first set of commit log files processed would be from "
+            + "searching from %s to %s and would contain %d file(s). They are (limit 10): \n%s",
+        searchStartTime,
+        searchEndTime,
+        fileBatch.size(),
+        fileBatch.stream().limit(10).collect(toImmutableList()));
  }

-  private void processFile(GcsFileMetadata metadata) {
-    try (InputStream input =
-        Channels.newInputStream(
-            gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+  private String replayFiles(DateTime startTime) {
+    DateTime replayTimeoutTime = startTime.plus(REPLAY_TIMEOUT_DURATION);
+    DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    int filesProcessed = 0;
+    int transactionsProcessed = 0;
+    // Starting from one millisecond after the last file we processed, search for and import files
+    // one hour at a time until we catch up to the current time or we hit the replay timeout (in
+    // which case the next run will pick up from where we leave off).
+    //
+    // We use hour-long batches because GCS supports filename prefix-based searches.
+    while (true) {
+      if (isAtOrAfter(clock.nowUtc(), replayTimeoutTime)) {
+        return createResponseString(
+            "Reached max execution time", startTime, filesProcessed, transactionsProcessed);
+      }
+      if (isBeforeOrAt(clock.nowUtc(), searchStartTime)) {
+        return createResponseString(
+            "Caught up to current time", startTime, filesProcessed, transactionsProcessed);
+      }
+      // Search through the end of the hour
+      DateTime searchEndTime =
+          searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
+      ImmutableList<BlobInfo> fileBatch =
+          diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime);
+      if (fileBatch.isEmpty()) {
+        logger.atInfo().log(
+            "No remaining files found in hour %s, continuing search in the next hour.",
+            searchStartTime.toString("yyyy-MM-dd HH"));
+      }
+      for (BlobInfo file : fileBatch) {
+        transactionsProcessed += processFile(file);
+        filesProcessed++;
+        if (clock.nowUtc().isAfter(replayTimeoutTime)) {
+          return createResponseString(
+              "Reached max execution time", startTime, filesProcessed, transactionsProcessed);
+        }
+      }
+      searchStartTime = searchEndTime.plusMillis(1);
+    }
+  }
+
+  private String createResponseString(
+      String msg, DateTime startTime, int filesProcessed, int transactionsProcessed) {
+    double tps =
+        (double) transactionsProcessed
+            / (double) Seconds.secondsBetween(startTime, clock.nowUtc()).getSeconds();
+    return String.format(
+        "%s after replaying %d file(s) containing %d total transaction(s) (%.2f tx/s).",
+        msg, filesProcessed, transactionsProcessed, tps);
+  }
+
+  /**
+   * Replays the commit logs in the given commit log file and returns the number of transactions
+   * committed.
+   */
+  private int processFile(BlobInfo metadata) {
+    try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
      // Load and process the Datastore transactions one at a time
      ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
          CommitLogImports.loadEntitiesByTransaction(input);
-      allTransactions.forEach(this::replayTransaction);
+      try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+          UpdateAutoTimestamp.disableAutoUpdate()) {
+        allTransactions.forEach(
+            transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
+      }
      // if we succeeded, set the last-seen time
-      DateTime checkpoint =
-          DateTime.parse(
-              metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
-      SqlReplayCheckpoint.set(checkpoint);
-      logger.atInfo().log("Replayed %d transactions from commit log file.", allTransactions.size());
+      DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
+      jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
+      logger.atInfo().log(
+          "Replayed %d transactions from commit log file %s with size %d B.",
+          allTransactions.size(), metadata.getName(), metadata.getSize());
+      return allTransactions.size();
    } catch (IOException e) {
-      throw new RuntimeException(e);
+      throw new RuntimeException(
+          "Errored out while replaying commit log file " + metadata.getName(), e);
    }
  }

@@ -151,21 +255,26 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
  }

  private void handleEntityPut(Entity entity) {
-    Object ofyPojo = ofy().toPojo(entity);
-    if (ofyPojo instanceof DatastoreEntity) {
-      DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
-      datastoreEntity
-          .toSqlEntity()
-          .ifPresent(
-              sqlEntity -> {
-                ReplaySpecializer.beforeSqlSave(sqlEntity);
-                jpaTm().put(sqlEntity);
-              });
-    } else {
-      // this should never happen, but we shouldn't fail on it
-      logger.atSevere().log(
-          "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
-          ofyPojo.getClass());
+    Object ofyPojo = auditedOfy().toPojo(entity);
+    try {
+      if (ofyPojo instanceof DatastoreEntity) {
+        DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
+        datastoreEntity
+            .toSqlEntity()
+            .ifPresent(
+                sqlEntity -> {
+                  sqlEntity.beforeSqlSaveOnReplay();
+                  jpaTm().putIgnoringReadOnly(sqlEntity);
+                });
+      } else {
+        // this should never happen, but we shouldn't fail on it
+        logger.atSevere().log(
+            "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
+            ofyPojo.getClass());
+      }
+    } catch (Throwable t) {
+      logger.atSevere().log("Error when replaying object %s", ofyPojo);
+      throw t;
    }
  }

@@ -181,13 +290,18 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
          "Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
      return;
    }
-    Class<?> entityClass = entityVKey.getKind();
-    // Delete the key iff the class represents a JPA entity that is replicated
-    if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
-        && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
-        && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
-      ReplaySpecializer.beforeSqlDelete(entityVKey);
-      jpaTm().delete(entityVKey);
+    try {
+      Class<?> entityClass = entityVKey.getKind();
+      // Delete the key iff the class represents a JPA entity that is replicated
+      if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
+          && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
+          && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
+        ReplaySpecializer.beforeSqlDelete(entityVKey);
+        jpaTm().deleteIgnoringReadOnly(entityVKey);
+      }
+    } catch (Throwable t) {
+      logger.atSevere().log("Error when deleting key %s", entityVKey);
+      throw t;
    }
  }

--- a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java
+++ b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java
@@ -18,14 +18,14 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterators.peekingIterator;
 import static google.registry.backup.BackupUtils.createDeserializingIterator;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;

 import com.google.appengine.api.datastore.DatastoreService;
 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.EntityTranslator;
-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobInfo;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.PeekingIterator;
 import com.google.common.collect.Streams;
@@ -33,7 +33,9 @@ import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.Result;
 import com.googlecode.objectify.util.ResultNow;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
+import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
@@ -46,10 +48,10 @@ import google.registry.request.auth.Auth;
 import google.registry.util.Retrier;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.channels.Channels;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Stream;
 import javax.inject.Inject;
@@ -66,45 +68,57 @@ public class RestoreCommitLogsAction implements Runnable {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

-  static final int BLOCK_SIZE = 1024 * 1024;  // Buffer 1mb at a time, for no particular reason.
-
  public static final String PATH = "/_dr/task/restoreCommitLogs";
  static final String DRY_RUN_PARAM = "dryRun";
  static final String FROM_TIME_PARAM = "fromTime";
  static final String TO_TIME_PARAM = "toTime";
+  static final String BUCKET_OVERRIDE_PARAM = "gcsBucket";
+
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
+
+  @Inject GcsUtils gcsUtils;

-  @Inject GcsService gcsService;
  @Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
  @Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
  @Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
+
+  @Inject
+  @Parameter(BUCKET_OVERRIDE_PARAM)
+  Optional<String> gcsBucketOverride;
+
  @Inject DatastoreService datastoreService;
  @Inject GcsDiffFileLister diffLister;
+
+  @Inject
+  @Config("commitLogGcsBucket")
+  String defaultGcsBucket;
+
  @Inject Retrier retrier;
  @Inject RestoreCommitLogsAction() {}

  @Override
  public void run() {
    checkArgument(
-        RegistryEnvironment.get() == RegistryEnvironment.ALPHA
-            || RegistryEnvironment.get() == RegistryEnvironment.CRASH
-            || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
-        "DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
+        !FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
+        "DO NOT RUN IN PRODUCTION OR SANDBOX.");
    if (dryRun) {
      logger.atInfo().log("Running in dryRun mode");
    }
-    List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
+    String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
+    logger.atInfo().log("Restoring from %s.", gcsBucket);
+    List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
    if (diffFiles.isEmpty()) {
      logger.atInfo().log("Nothing to restore");
      return;
    }
    Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
    CommitLogCheckpoint lastCheckpoint = null;
-    for (GcsFileMetadata metadata : diffFiles) {
-      logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
-      try (InputStream input = Channels.newInputStream(
-          gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+    for (BlobInfo metadata : diffFiles) {
+      logger.atInfo().log("Restoring: %s", metadata.getName());
+      try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
        PeekingIterator<ImmutableObject> commitLogs =
-            peekingIterator(createDeserializingIterator(input));
+            peekingIterator(createDeserializingIterator(input, true));
        lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
        saveOfy(ImmutableList.of(lastCheckpoint));  // Save the checkpoint itself.
        while (commitLogs.hasNext()) {
@@ -146,10 +160,10 @@ public class RestoreCommitLogsAction implements Runnable {
  private CommitLogManifest restoreOneTransaction(PeekingIterator<ImmutableObject> commitLogs) {
    final CommitLogManifest manifest = (CommitLogManifest) commitLogs.next();
    Result<?> deleteResult = deleteAsync(manifest.getDeletions());
-    List<Entity> entitiesToSave = Lists.newArrayList(ofy().save().toEntity(manifest));
+    List<Entity> entitiesToSave = Lists.newArrayList(auditedOfy().save().toEntity(manifest));
    while (commitLogs.hasNext() && commitLogs.peek() instanceof CommitLogMutation) {
      CommitLogMutation mutation = (CommitLogMutation) commitLogs.next();
-      entitiesToSave.add(ofy().save().toEntity(mutation));
+      entitiesToSave.add(auditedOfy().save().toEntity(mutation));
      entitiesToSave.add(EntityTranslator.createFromPbBytes(mutation.getEntityProtoBytes()));
    }
    saveRaw(entitiesToSave);
@@ -176,7 +190,8 @@ public class RestoreCommitLogsAction implements Runnable {
      return;
    }
    retrier.callWithRetry(
-        () -> ofy().saveWithoutBackup().entities(objectsToSave).now(), RuntimeException.class);
+        () -> auditedOfy().saveWithoutBackup().entities(objectsToSave).now(),
+        RuntimeException.class);
  }

  private Result<?> deleteAsync(Set<Key<?>> keysToDelete) {
@@ -185,7 +200,7 @@ public class RestoreCommitLogsAction implements Runnable {
    }
    return dryRun || keysToDelete.isEmpty()
        ? new ResultNow<Void>(null)
-        : ofy().deleteWithoutBackup().keys(keysToDelete);
+        : auditedOfy().deleteWithoutBackup().keys(keysToDelete);
  }

 }
--- a/core/src/main/java/google/registry/backup/VersionedEntity.java
+++ b/core/src/main/java/google/registry/backup/VersionedEntity.java
@@ -47,7 +47,7 @@ import javax.annotation.Nullable;
 *
 * <ul>
 *   <li>Convert an Objectify entity to a Datastore {@link Entity}: {@code
- *       ofy().save().toEntity(..)}
+ *       auditedOfy().save().toEntity(..)}
 *   <li>Entity is serializable, but the more efficient approach is to convert an Entity to a
 *       ProtocolBuffer ({@link com.google.storage.onestore.v3.OnestoreEntity.EntityProto}) and then
 *       to raw bytes.
--- a/core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java
+++ b/core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java
@@ -28,10 +28,10 @@ import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
-import google.registry.schema.domain.RegistryLock;
 import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.Retrier;
 import javax.inject.Inject;
@@ -126,18 +126,18 @@ public final class AsyncTaskEnqueuer {
  public void enqueueAsyncDelete(
      EppResource resourceToDelete,
      DateTime now,
-      String requestingClientId,
+      String requestingRegistrarId,
      Trid trid,
      boolean isSuperuser) {
    Key<EppResource> resourceKey = Key.create(resourceToDelete);
    logger.atInfo().log(
        "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingClientId);
+        resourceKey, requestingRegistrarId);
    TaskOptions task =
        TaskOptions.Builder.withMethod(Method.PULL)
            .countdownMillis(asyncDeleteDelay.getMillis())
            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
-            .param(PARAM_REQUESTING_CLIENT_ID, requestingClientId)
+            .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
            .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
            .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))
            .param(PARAM_REQUESTED_TIME, now.toString());
--- a/core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java
@@ -34,7 +34,7 @@ import static google.registry.model.ResourceTransferUtils.denyPendingTransfer;
 import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete;
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
 import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE;
 import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE_FAILURE;
 import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE;
@@ -109,6 +109,7 @@ import org.joda.time.Duration;
 * A mapreduce that processes batch asynchronous deletions of contact and host resources by mapping
 * over all domains and checking for any references to the contacts/hosts in pending deletion.
 */
+@Deprecated
@Action(
    service = Action.Service.BACKEND,
    path = "/_dr/task/deleteContactsAndHosts",
@@ -335,22 +336,22 @@ public class DeleteContactsAndHostsAction implements Runnable {
        DeletionRequest deletionRequest, boolean hasNoActiveReferences) {
      DateTime now = tm().getTransactionTime();
      EppResource resource =
-          ofy().load().key(deletionRequest.key()).now().cloneProjectedAtTime(now);
+          auditedOfy().load().key(deletionRequest.key()).now().cloneProjectedAtTime(now);
      // Double-check transactionally that the resource is still active and in PENDING_DELETE.
      if (!doesResourceStateAllowDeletion(resource, now)) {
        return DeletionResult.create(Type.ERRORED, "");
      }
      // Contacts and external hosts have a direct client id. For subordinate hosts it needs to be
      // read off of the superordinate domain.
-      String resourceClientId = resource.getPersistedCurrentSponsorClientId();
+      String resourceRegistrarId = resource.getPersistedCurrentSponsorRegistrarId();
      if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
-        resourceClientId =
+        resourceRegistrarId =
            tm().loadByKey(((HostResource) resource).getSuperordinateDomain())
                .cloneProjectedAtTime(now)
-                .getCurrentSponsorClientId();
+                .getCurrentSponsorRegistrarId();
      }
      boolean requestedByCurrentOwner =
-          resourceClientId.equals(deletionRequest.requestingClientId());
+          resourceRegistrarId.equals(deletionRequest.requestingClientId());

      boolean deleteAllowed =
          hasNoActiveReferences && (requestedByCurrentOwner || deletionRequest.isSuperuser());
@@ -369,16 +370,15 @@ public class DeleteContactsAndHostsAction implements Runnable {
                      : "it was transferred prior to deletion");

      HistoryEntry historyEntry =
-          new HistoryEntry.Builder()
-              .setClientId(deletionRequest.requestingClientId())
+          HistoryEntry.createBuilderForResource(resource)
+              .setRegistrarId(deletionRequest.requestingClientId())
              .setModificationTime(now)
              .setType(getHistoryEntryType(resource, deleteAllowed))
-              .setParent(deletionRequest.key())
              .build();

      PollMessage.OneTime pollMessage =
          new PollMessage.OneTime.Builder()
-              .setClientId(deletionRequest.requestingClientId())
+              .setRegistrarId(deletionRequest.requestingClientId())
              .setMsg(pollMessageText)
              .setParent(historyEntry)
              .setEventTime(now)
@@ -408,7 +408,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
      } else {
        resourceToSave = resource.asBuilder().removeStatusValue(PENDING_DELETE).build();
      }
-      ofy().save().<ImmutableObject>entities(resourceToSave, historyEntry, pollMessage);
+      auditedOfy()
+          .save()
+          .<ImmutableObject>entities(resourceToSave, historyEntry.asHistoryEntry(), pollMessage);
      return DeletionResult.create(
          deleteAllowed ? Type.DELETED : Type.NOT_DELETED, pollMessageText);
    }
@@ -525,7 +527,8 @@ public class DeleteContactsAndHostsAction implements Runnable {
          Key.create(
              checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
      EppResource resource =
-          checkNotNull(ofy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+          checkNotNull(
+              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
      checkState(
          resource instanceof ContactResource || resource instanceof HostResource,
          "Cannot delete a %s via this action",
--- a/core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java
@@ -17,8 +17,8 @@ package google.registry.batch;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.flows.FlowUtils.marshalWithLenientRetry;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -36,6 +36,7 @@ import google.registry.flows.StatelessRequestSessionMetadata;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.ProtocolDefinition;
 import google.registry.model.eppoutput.EppOutput;
+import google.registry.persistence.transaction.QueryComposer.Comparator;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
@@ -128,12 +129,15 @@ public class DeleteExpiredDomainsAction implements Runnable {
    logger.atInfo().log(
        "Deleting non-renewing domains with autorenew end times up through %s.", runTime);

-    // Note: This query is (and must be) non-transactional, and thus, is only eventually consistent.
+    // Note: in Datastore, this query is (and must be) non-transactional, and thus, is only
+    // eventually consistent.
    ImmutableList<DomainBase> domainsToDelete =
-        ofy().load().type(DomainBase.class).filter("autorenewEndTime <=", runTime).list().stream()
-            // Datastore can't do two inequalities in one query, so the second happens in-memory.
-            .filter(d -> d.getDeletionTime().isEqual(END_OF_TIME))
-            .collect(toImmutableList());
+        transactIfJpaTm(
+            () ->
+                tm().createQueryComposer(DomainBase.class)
+                    .where("autorenewEndTime", Comparator.LTE, runTime)
+                    .where("deletionTime", Comparator.EQ, END_OF_TIME)
+                    .list());
    if (domainsToDelete.isEmpty()) {
      logger.atInfo().log("Found 0 domains to delete.");
      response.setPayload("Found 0 domains to delete.");
--- a/core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java
@@ -15,12 +15,14 @@
 package google.registry.batch;

 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryEnvironment.PRODUCTION;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;

 import com.google.appengine.tools.mapreduce.Mapper;
 import com.google.common.collect.ImmutableList;
@@ -28,16 +30,24 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryEnvironment;
+import google.registry.flows.poll.PollFlowUtils;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.model.EppResource;
+import google.registry.model.EppResourceUtils;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.ForeignKeyIndex;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
 import java.util.List;
 import javax.inject.Inject;

@@ -46,8 +56,8 @@ import javax.inject.Inject;
 * the associated ForeignKey and EppResourceIndex entities.
 *
 * <p>This only deletes contacts and hosts, NOT domains. To delete domains, use {@link
- * DeleteLoadTestDataAction} and pass it the TLD(s) that the load test domains were created on. Note
- * that DeleteLoadTestDataAction is safe enough to run in production whereas this mapreduce is not,
+ * DeleteProberDataAction} and pass it the TLD(s) that the load test domains were created on. Note
+ * that DeleteProberDataAction is safe enough to run in production whereas this mapreduce is not,
 * but this one does not need to be runnable in production because load testing isn't run against
 * production.
 */
@@ -68,15 +78,22 @@ public class DeleteLoadTestDataAction implements Runnable {
   */
  private static final ImmutableSet<String> LOAD_TEST_REGISTRARS = ImmutableSet.of("proxy");

-  @Inject
-  @Parameter(PARAM_DRY_RUN)
-  boolean isDryRun;
-
-  @Inject MapreduceRunner mrRunner;
-  @Inject Response response;
+  private final boolean isDryRun;
+  private final MapreduceRunner mrRunner;
+  private final Response response;
+  private final Clock clock;

  @Inject
-  DeleteLoadTestDataAction() {}
+  DeleteLoadTestDataAction(
+      @Parameter(PARAM_DRY_RUN) boolean isDryRun,
+      MapreduceRunner mrRunner,
+      Response response,
+      Clock clock) {
+    this.isDryRun = isDryRun;
+    this.mrRunner = mrRunner;
+    this.response = response;
+    this.clock = clock;
+  }

  @Override
  public void run() {
@@ -87,14 +104,85 @@ public class DeleteLoadTestDataAction implements Runnable {
        !RegistryEnvironment.get().equals(PRODUCTION),
        "This mapreduce is not safe to run on PRODUCTION.");

-    mrRunner
-        .setJobName("Delete load test data")
-        .setModuleName("backend")
-        .runMapOnly(
-            new DeleteLoadTestDataMapper(isDryRun),
-            ImmutableList.of(
-                createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
-        .sendLinkToMapreduceConsole(response);
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Delete load test data")
+          .setModuleName("backend")
+          .runMapOnly(
+              new DeleteLoadTestDataMapper(isDryRun),
+              ImmutableList.of(
+                  createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      tm().transact(
+              () -> {
+                LOAD_TEST_REGISTRARS.forEach(this::deletePollMessages);
+                tm().loadAllOfStream(ContactResource.class).forEach(this::deleteContact);
+                tm().loadAllOfStream(HostResource.class).forEach(this::deleteHost);
+              });
+    }
+  }
+
+  private void deletePollMessages(String registrarId) {
+    ImmutableList<PollMessage> pollMessages =
+        PollFlowUtils.createPollMessageQuery(registrarId, END_OF_TIME).list();
+    if (isDryRun) {
+      logger.atInfo().log(
+          "Would delete %d poll messages for registrar %s.", pollMessages.size(), registrarId);
+    } else {
+      pollMessages.forEach(tm()::delete);
+    }
+  }
+
+  private void deleteContact(ContactResource contact) {
+    if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorRegistrarId())) {
+      return;
+    }
+    // We cannot remove contacts from domains in the general case, so we cannot delete contacts
+    // that are linked to domains (since it would break the foreign keys)
+    if (EppResourceUtils.isLinked(contact.createVKey(), clock.nowUtc())) {
+      logger.atWarning().log(
+          "Cannot delete contact with repo ID %s since it is referenced from a domain",
+          contact.getRepoId());
+      return;
+    }
+    deleteResource(contact);
+  }
+
+  private void deleteHost(HostResource host) {
+    if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorRegistrarId())) {
+      return;
+    }
+    VKey<HostResource> hostVKey = host.createVKey();
+    // We can remove hosts from linked domains, so we should do so then delete the hosts
+    ImmutableSet<VKey<DomainBase>> linkedDomains =
+        EppResourceUtils.getLinkedDomainKeys(hostVKey, clock.nowUtc(), null);
+    tm().loadByKeys(linkedDomains)
+        .values()
+        .forEach(
+            domain -> {
+              ImmutableSet<VKey<HostResource>> remainingHosts =
+                  domain.getNsHosts().stream()
+                      .filter(vkey -> !vkey.equals(hostVKey))
+                      .collect(toImmutableSet());
+              tm().put(domain.asBuilder().setNameservers(remainingHosts).build());
+            });
+    deleteResource(host);
+  }
+
+  private void deleteResource(EppResource eppResource) {
+    // In SQL, the only objects parented on the resource are poll messages (deleted above) and
+    // history objects.
+    ImmutableList<HistoryEntry> historyObjects =
+        HistoryEntryDao.loadHistoryObjectsForResource(eppResource.createVKey());
+    if (isDryRun) {
+      logger.atInfo().log(
+          "Would delete repo ID %s along with %d history objects",
+          eppResource.getRepoId(), historyObjects.size());
+    } else {
+      historyObjects.forEach(tm()::delete);
+      tm().delete(eppResource);
+    }
  }

  /** Provides the map method that runs for each existing contact and host entity. */
@@ -110,7 +198,7 @@ public class DeleteLoadTestDataAction implements Runnable {

    @Override
    public final void map(EppResource resource) {
-      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorClientId())) {
+      if (LOAD_TEST_REGISTRARS.contains(resource.getPersistedCurrentSponsorRegistrarId())) {
        deleteResource(resource);
        getContext()
            .incrementCounter(
@@ -125,12 +213,11 @@ public class DeleteLoadTestDataAction implements Runnable {
          Key.create(EppResourceIndex.create(Key.create(resource)));
      final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(resource);
      int numEntitiesDeleted =
-          tm()
-              .transact(
+          tm().transact(
                  () -> {
                    // This ancestor query selects all descendant entities.
                    List<Key<Object>> resourceAndDependentKeys =
-                        ofy().load().ancestor(resource).keys().list();
+                        auditedOfy().load().ancestor(resource).keys().list();
                    ImmutableSet<Key<?>> allKeys =
                        new ImmutableSet.Builder<Key<?>>()
                            .add(fki)
@@ -140,7 +227,7 @@ public class DeleteLoadTestDataAction implements Runnable {
                    if (isDryRun) {
                      logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
                    } else {
-                      ofy().deleteWithoutBackup().keys(allKeys);
+                      auditedOfy().deleteWithoutBackup().keys(allKeys);
                    }
                    return allKeys.size();
                  });
--- a/core/src/main/java/google/registry/batch/DeleteProberDataAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteProberDataAction.java
@@ -20,9 +20,10 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryEnvironment.PRODUCTION;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.model.registry.Registries.getTldsOfType;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
+import static google.registry.model.tld.Registries.getTldsOfType;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
@@ -42,27 +43,31 @@ import google.registry.config.RegistryEnvironment;
 import google.registry.dns.DnsQueue;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
+import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.EppResourceUtils;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.ForeignKeyIndex;
-import google.registry.model.registry.Registry;
-import google.registry.model.registry.Registry.TldType;
-import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import google.registry.model.tld.Registry.TldType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 import javax.inject.Inject;
+import org.hibernate.CacheMode;
+import org.hibernate.ScrollMode;
+import org.hibernate.ScrollableResults;
+import org.hibernate.query.Query;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;

 /**
- * Deletes all prober DomainBases and their subordinate history entries, poll messages, and
- * billing events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
- *
- * <p>See: https://www.youtube.com/watch?v=xuuv0syoHnM
+ * Deletes all prober DomainBases and their subordinate history entries, poll messages, and billing
+ * events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
 */
@Action(
    service = Action.Service.BACKEND,
@@ -73,10 +78,51 @@ public class DeleteProberDataAction implements Runnable {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

+  /**
+   * The maximum amount of time we allow a prober domain to be in use.
+   *
+   * <p>In practice, the prober's connection will time out well before this duration. This includes
+   * a decent buffer.
+   */
+  private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
+
+  /**
+   * The minimum amount of time we want a domain to be "soft deleted".
+   *
+   * <p>The domain has to remain soft deleted for at least enough time for the DNS task to run and
+   * remove it from DNS itself. This is probably on the order of minutes.
+   */
+  private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
+
+  private static final DnsQueue dnsQueue = DnsQueue.create();
+
+  // Domains to delete must:
+  // 1. Be in one of the prober TLDs
+  // 2. Not be a nic domain
+  // 3. Have no subordinate hosts
+  // 4. Not still be used (within an hour of creation time)
+  // 5. Either be active (creationTime <= now < deletionTime) or have been deleted a while ago (this
+  //    prevents accidental double-map with the same key from immediately deleting active domains)
+  //
+  // Note: creationTime must be compared to a Java object (CreateAutoTimestamp) but deletionTime can
+  // be compared directly to the SQL timestamp (it's a DateTime)
+  private static final String DOMAIN_QUERY_STRING =
+      "FROM Domain d WHERE d.tld IN :tlds AND d.fullyQualifiedDomainName NOT LIKE 'nic.%' AND"
+          + " (d.subordinateHosts IS EMPTY OR d.subordinateHosts IS NULL) AND d.creationTime <"
+          + " :creationTimeCutoff AND ((d.creationTime <= :nowAutoTimestamp AND d.deletionTime >"
+          + " current_timestamp()) OR d.deletionTime < :nowMinusSoftDeleteDelay) ORDER BY d.repoId";
+
+  /** Number of domains to retrieve and delete per SQL transaction. */
+  private static final int BATCH_SIZE = 1000;
+
  @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
  /** List of TLDs to work on. If empty - will work on all TLDs that end with .test. */
  @Inject @Parameter(PARAM_TLDS) ImmutableSet<String> tlds;
-  @Inject @Config("registryAdminClientId") String registryAdminClientId;
+
+  @Inject
+  @Config("registryAdminClientId")
+  String registryAdminRegistrarId;
+
  @Inject MapreduceRunner mrRunner;
  @Inject Response response;
  @Inject DeleteProberDataAction() {}
@@ -84,25 +130,14 @@ public class DeleteProberDataAction implements Runnable {
  @Override
  public void run() {
    checkState(
-        !Strings.isNullOrEmpty(registryAdminClientId),
+        !Strings.isNullOrEmpty(registryAdminRegistrarId),
        "Registry admin client ID must be configured for prober data deletion to work");
-    mrRunner
-        .setJobName("Delete prober data")
-        .setModuleName("backend")
-        .runMapOnly(
-            new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId),
-            ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
-        .sendLinkToMapreduceConsole(response);
-  }
-
-  private ImmutableSet<String> getProberRoidSuffixes() {
    checkArgument(
        !PRODUCTION.equals(RegistryEnvironment.get())
            || tlds.stream().allMatch(tld -> tld.endsWith(".test")),
        "On production, can only work on TLDs that end with .test");
    ImmutableSet<String> deletableTlds =
-        getTldsOfType(TldType.TEST)
-            .stream()
+        getTldsOfType(TldType.TEST).stream()
            .filter(tld -> tlds.isEmpty() ? tld.endsWith(".test") : tlds.contains(tld))
            .collect(toImmutableSet());
    checkArgument(
@@ -110,10 +145,161 @@ public class DeleteProberDataAction implements Runnable {
        "If tlds are given, they must all exist and be TEST tlds. Given: %s, not found: %s",
        tlds,
        Sets.difference(tlds, deletableTlds));
-    return deletableTlds
-        .stream()
-        .map(tld -> Registry.get(tld).getRoidSuffix())
-        .collect(toImmutableSet());
+    ImmutableSet<String> proberRoidSuffixes =
+        deletableTlds.stream()
+            .map(tld -> Registry.get(tld).getRoidSuffix())
+            .collect(toImmutableSet());
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Delete prober data")
+          .setModuleName("backend")
+          .runMapOnly(
+              new DeleteProberDataMapper(proberRoidSuffixes, isDryRun, registryAdminRegistrarId),
+              ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      runSqlJob(deletableTlds);
+    }
+  }
+
+  private void runSqlJob(ImmutableSet<String> deletableTlds) {
+    AtomicInteger softDeletedDomains = new AtomicInteger();
+    AtomicInteger hardDeletedDomains = new AtomicInteger();
+    jpaTm().transact(() -> processDomains(deletableTlds, softDeletedDomains, hardDeletedDomains));
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have soft-deleted" : "Soft-deleted", softDeletedDomains.get());
+    logger.atInfo().log(
+        "%s %d domains.",
+        isDryRun ? "Would have hard-deleted" : "Hard-deleted", hardDeletedDomains.get());
+  }
+
+  private void processDomains(
+      ImmutableSet<String> deletableTlds,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    DateTime now = tm().getTransactionTime();
+    // Scroll through domains, soft-deleting as necessary (very few will be soft-deleted) and
+    // keeping track of which domains to hard-delete (there can be many, so we batch them up)
+    ScrollableResults scrollableResult =
+        jpaTm()
+            .query(DOMAIN_QUERY_STRING, DomainBase.class)
+            .setParameter("tlds", deletableTlds)
+            .setParameter(
+                "creationTimeCutoff", CreateAutoTimestamp.create(now.minus(DOMAIN_USED_DURATION)))
+            .setParameter("nowMinusSoftDeleteDelay", now.minus(SOFT_DELETE_DELAY))
+            .setParameter("nowAutoTimestamp", CreateAutoTimestamp.create(now))
+            .unwrap(Query.class)
+            .setCacheMode(CacheMode.IGNORE)
+            .scroll(ScrollMode.FORWARD_ONLY);
+    ImmutableList.Builder<String> domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+    ImmutableList.Builder<String> hostNamesToHardDelete = new ImmutableList.Builder<>();
+    for (int i = 1; scrollableResult.next(); i = (i + 1) % BATCH_SIZE) {
+      DomainBase domain = (DomainBase) scrollableResult.get(0);
+      processDomain(
+          domain,
+          domainRepoIdsToHardDelete,
+          hostNamesToHardDelete,
+          softDeletedDomains,
+          hardDeletedDomains);
+      // Batch the deletion and DB flush + session clearing so we don't OOM
+      if (i == 0) {
+        hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+        domainRepoIdsToHardDelete = new ImmutableList.Builder<>();
+        hostNamesToHardDelete = new ImmutableList.Builder<>();
+        jpaTm().getEntityManager().flush();
+        jpaTm().getEntityManager().clear();
+      }
+    }
+    // process the remainder
+    hardDeleteDomainsAndHosts(domainRepoIdsToHardDelete.build(), hostNamesToHardDelete.build());
+  }
+
+  private void processDomain(
+      DomainBase domain,
+      ImmutableList.Builder<String> domainRepoIdsToHardDelete,
+      ImmutableList.Builder<String> hostNamesToHardDelete,
+      AtomicInteger softDeletedDomains,
+      AtomicInteger hardDeletedDomains) {
+    // If the domain is still active, that means that the prober encountered a failure and did not
+    // successfully soft-delete the domain (thus leaving its DNS entry published). We soft-delete
+    // it now so that the DNS entry can be handled. The domain will then be hard-deleted the next
+    // time the job is run.
+    if (EppResourceUtils.isActive(domain, tm().getTransactionTime())) {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would soft-delete the active domain: %s (%s)",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue);
+      }
+      softDeletedDomains.incrementAndGet();
+    } else {
+      if (isDryRun) {
+        logger.atInfo().log(
+            "Would hard-delete the non-active domain: %s (%s) and its dependents",
+            domain.getDomainName(), domain.getRepoId());
+      } else {
+        domainRepoIdsToHardDelete.add(domain.getRepoId());
+        hostNamesToHardDelete.addAll(domain.getSubordinateHosts());
+      }
+      hardDeletedDomains.incrementAndGet();
+    }
+  }
+
+  private void hardDeleteDomainsAndHosts(
+      ImmutableList<String> domainRepoIds, ImmutableList<String> hostNames) {
+    jpaTm()
+        .query("DELETE FROM Host WHERE fullyQualifiedHostName IN :hostNames")
+        .setParameter("hostNames", hostNames)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingEvent WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingRecurrence WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM BillingCancellation WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM DomainHistory WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM PollMessage WHERE domainRepoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+    jpaTm()
+        .query("DELETE FROM Domain WHERE repoId IN :repoIds")
+        .setParameter("repoIds", domainRepoIds)
+        .executeUpdate();
+  }
+
+  // Take a DNS queue + admin registrar id as input so that it can be called from the mapper as well
+  private static void softDeleteDomain(
+      DomainBase domain, String registryAdminRegistrarId, DnsQueue localDnsQueue) {
+    DomainBase deletedDomain =
+        domain.asBuilder().setDeletionTime(tm().getTransactionTime()).setStatusValues(null).build();
+    DomainHistory historyEntry =
+        new DomainHistory.Builder()
+            .setDomain(domain)
+            .setType(DOMAIN_DELETE)
+            .setModificationTime(tm().getTransactionTime())
+            .setBySuperuser(true)
+            .setReason("Deletion of prober data")
+            .setRegistrarId(registryAdminRegistrarId)
+            .build();
+    // Note that we don't bother handling grace periods, billing events, pending transfers, poll
+    // messages, or auto-renews because those will all be hard-deleted the next time the job runs
+    // anyway.
+    tm().putAllWithoutBackup(ImmutableList.of(deletedDomain, historyEntry));
+    // updating foreign keys is a no-op in SQL
+    updateForeignKeyIndexDeletionTime(deletedDomain);
+    localDnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
  }

  /** Provides the map method that runs for each existing DomainBase entity. */
@@ -122,32 +308,17 @@ public class DeleteProberDataAction implements Runnable {
    private static final DnsQueue dnsQueue = DnsQueue.create();
    private static final long serialVersionUID = -7724537393697576369L;

-    /**
-     * The maximum amount of time we allow a prober domain to be in use.
-     *
-     * In practice, the prober's connection will time out well before this duration. This includes a
-     * decent buffer.
-     *
-     */
-    private static final Duration DOMAIN_USED_DURATION = Duration.standardHours(1);
-
-    /**
-     * The minimum amount of time we want a domain to be "soft deleted".
-     *
-     * The domain has to remain soft deleted for at least enough time for the DNS task to run and
-     * remove it from DNS itself. This is probably on the order of minutes.
-     */
-    private static final Duration SOFT_DELETE_DELAY = Duration.standardHours(1);
-
    private final ImmutableSet<String> proberRoidSuffixes;
    private final Boolean isDryRun;
-    private final String registryAdminClientId;
+    private final String registryAdminRegistrarId;

    public DeleteProberDataMapper(
-        ImmutableSet<String> proberRoidSuffixes, Boolean isDryRun, String registryAdminClientId) {
+        ImmutableSet<String> proberRoidSuffixes,
+        Boolean isDryRun,
+        String registryAdminRegistrarId) {
      this.proberRoidSuffixes = proberRoidSuffixes;
      this.isDryRun = isDryRun;
-      this.registryAdminClientId = registryAdminClientId;
+      this.registryAdminRegistrarId = registryAdminRegistrarId;
    }

    @Override
@@ -166,7 +337,7 @@ public class DeleteProberDataAction implements Runnable {
    }

    private void deleteDomain(final Key<DomainBase> domainKey) {
-      final DomainBase domain = ofy().load().key(domainKey).now();
+      final DomainBase domain = auditedOfy().load().key(domainKey).now();

      DateTime now = DateTime.now(UTC);

@@ -203,7 +374,7 @@ public class DeleteProberDataAction implements Runnable {
          logger.atInfo().log(
              "Would soft-delete the active domain: %s (%s)", domainName, domainKey);
        } else {
-          softDeleteDomain(domain);
+          tm().transact(() -> softDeleteDomain(domain, registryAdminRegistrarId, dnsQueue));
        }
        getContext().incrementCounter("domains soft-deleted");
        return;
@@ -220,14 +391,12 @@ public class DeleteProberDataAction implements Runnable {
      final Key<? extends ForeignKeyIndex<?>> fki = ForeignKeyIndex.createKey(domain);

      int entitiesDeleted =
-          tm()
-              .transact(
+          tm().transact(
                  () -> {
                    // This ancestor query selects all descendant HistoryEntries, BillingEvents,
-                    // PollMessages,
-                    // and TLD-specific entities, as well as the domain itself.
+                    // PollMessages, and TLD-specific entities, as well as the domain itself.
                    List<Key<Object>> domainAndDependentKeys =
-                        ofy().load().ancestor(domainKey).keys().list();
+                        auditedOfy().load().ancestor(domainKey).keys().list();
                    ImmutableSet<Key<?>> allKeys =
                        new ImmutableSet.Builder<Key<?>>()
                            .add(fki)
@@ -237,41 +406,12 @@ public class DeleteProberDataAction implements Runnable {
                    if (isDryRun) {
                      logger.atInfo().log("Would hard-delete the following entities: %s", allKeys);
                    } else {
-                      ofy().deleteWithoutBackup().keys(allKeys);
+                      auditedOfy().deleteWithoutBackup().keys(allKeys);
                    }
                    return allKeys.size();
                  });
      getContext().incrementCounter("domains hard-deleted");
      getContext().incrementCounter("total entities hard-deleted", entitiesDeleted);
    }
-
-    private void softDeleteDomain(final DomainBase domain) {
-      tm().transactNew(
-              () -> {
-                DomainBase deletedDomain =
-                    domain
-                        .asBuilder()
-                        .setDeletionTime(tm().getTransactionTime())
-                        .setStatusValues(null)
-                        .build();
-                HistoryEntry historyEntry =
-                    new HistoryEntry.Builder()
-                        .setParent(domain)
-                        .setType(DOMAIN_DELETE)
-                        .setModificationTime(tm().getTransactionTime())
-                        .setBySuperuser(true)
-                        .setReason("Deletion of prober data")
-                        .setClientId(registryAdminClientId)
-                        .build();
-                // Note that we don't bother handling grace periods, billing events, pending
-                // transfers,
-                // poll messages, or auto-renews because these will all be hard-deleted the next
-                // time the
-                // mapreduce runs anyway.
-                ofy().save().entities(deletedDomain, historyEntry);
-                updateForeignKeyIndexDeletionTime(deletedDomain);
-                dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
-              });
-    }
  }
 }
--- a/core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java
+++ b/core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java
@@ -21,9 +21,12 @@ import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createChildEntityInput;
 import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
 import static google.registry.model.domain.Period.Unit.YEARS;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
+import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
 import static google.registry.util.CollectionUtils.union;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -38,10 +41,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Range;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.NullInput;
-import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
@@ -49,11 +50,12 @@ import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.common.Cursor;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.Period;
-import google.registry.model.registry.Registry;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
-import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -91,30 +93,88 @@ public class ExpandRecurringBillingEventsAction implements Runnable {

  @Override
  public void run() {
-    Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
    DateTime executeTime = clock.nowUtc();
-    DateTime persistedCursorTime = (cursor == null ? START_OF_TIME : cursor.getCursorTime());
+    DateTime persistedCursorTime =
+        transactIfJpaTm(
+            () ->
+                tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
+                    .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                    .getCursorTime());
    DateTime cursorTime = cursorTimeParam.orElse(persistedCursorTime);
    checkArgument(
-        cursorTime.isBefore(executeTime),
-        "Cursor time must be earlier than execution time.");
+        cursorTime.isBefore(executeTime), "Cursor time must be earlier than execution time.");
    logger.atInfo().log(
        "Running Recurring billing event expansion for billing time range [%s, %s).",
        cursorTime, executeTime);
-    mrRunner
-        .setJobName("Expand Recurring billing events into synthetic OneTime events.")
-        .setModuleName("backend")
-        .runMapreduce(
-            new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
-            new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
-            // Add an extra shard that maps over a null recurring event (see the mapper for why).
-            ImmutableList.of(
-                new NullInput<>(),
-                createChildEntityInput(
-                    ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
-        .sendLinkToMapreduceConsole(response);
-  }
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Expand Recurring billing events into synthetic OneTime events.")
+          .setModuleName("backend")
+          .runMapreduce(
+              new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
+              new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
+              // Add an extra shard that maps over a null recurring event (see the mapper for why).
+              ImmutableList.of(
+                  new NullInput<>(),
+                  createChildEntityInput(
+                      ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      int numBillingEventsSaved =
+          jpaTm()
+              .transact(
+                  () ->
+                      jpaTm()
+                          .query(
+                              "FROM BillingRecurrence "
+                                  + "WHERE eventTime <= :executeTime "
+                                  + "AND eventTime < recurrenceEndTime "
+                                  + "ORDER BY id ASC",
+                              Recurring.class)
+                          .setParameter("executeTime", executeTime)
+                          // Need to get a list from the transaction and then convert it to a stream
+                          // for further processing. If we get a stream directly, each elements gets
+                          // processed downstream eagerly but Hibernate returns a
+                          // ScrollableResultsIterator that cannot be advanced outside the
+                          // transaction, resulting in an exception.
+                          .getResultList())
+              .stream()
+              .map(
+                  recurring ->
+                      jpaTm()
+                          .transact(
+                              () ->
+                                  expandBillingEvent(recurring, executeTime, cursorTime, isDryRun)))
+              .reduce(0, Integer::sum);

+      if (!isDryRun) {
+        logger.atInfo().log("Saved OneTime billing events", numBillingEventsSaved);
+      } else {
+        logger.atInfo().log("Generated OneTime billing events (dry run)", numBillingEventsSaved);
+      }
+      logger.atInfo().log(
+          "Recurring event expansion %s complete for billing event range [%s, %s).",
+          isDryRun ? "(dry run) " : "", cursorTime, executeTime);
+      tm().transact(
+              () -> {
+                // Check for the unlikely scenario where the cursor has been altered during the
+                // expansion.
+                DateTime currentCursorTime =
+                    tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
+                        .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                        .getCursorTime();
+                if (!currentCursorTime.equals(persistedCursorTime)) {
+                  throw new IllegalStateException(
+                      String.format(
+                          "Current cursor position %s does not match persisted cursor position %s.",
+                          currentCursorTime, persistedCursorTime));
+                }
+                if (!isDryRun) {
+                  tm().put(Cursor.createGlobal(RECURRING_BILLING, executeTime));
+                }
+              });
+    }
+  }
  /** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */
  public static class ExpandRecurringBillingEventsMapper
      extends Mapper<Recurring, DateTime, DateTime> {
@@ -153,98 +213,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
      try {
        numBillingEventsSaved =
            tm().transactNew(
-                    () -> {
-                      ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
-                          new ImmutableSet.Builder<>();
-                      final Registry tld =
-                          Registry.get(getTldFromDomainName(recurring.getTargetId()));
-
-                      // Determine the complete set of times at which this recurring event should
-                      // occur (up to and including the runtime of the mapreduce).
-                      Iterable<DateTime> eventTimes =
-                          recurring
-                              .getRecurrenceTimeOfYear()
-                              .getInstancesInRange(
-                                  Range.closed(
-                                      recurring.getEventTime(),
-                                      earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
-
-                      // Convert these event times to billing times
-                      final ImmutableSet<DateTime> billingTimes =
-                          getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
-
-                      Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
-                      Iterable<OneTime> oneTimesForDomain =
-                          ofy().load().type(OneTime.class).ancestor(domainKey);
-
-                      // Determine the billing times that already have OneTime events persisted.
-                      ImmutableSet<DateTime> existingBillingTimes =
-                          getExistingBillingTimes(oneTimesForDomain, recurring);
-
-                      ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
-                          new ImmutableSet.Builder<>();
-                      // Create synthetic OneTime events for all billing times that do not yet have
-                      // an event persisted.
-                      for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
-                        // Construct a new HistoryEntry that parents over the OneTime
-                        HistoryEntry historyEntry =
-                            new HistoryEntry.Builder()
-                                .setBySuperuser(false)
-                                .setClientId(recurring.getClientId())
-                                .setModificationTime(tm().getTransactionTime())
-                                .setParent(domainKey)
-                                .setPeriod(Period.create(1, YEARS))
-                                .setReason(
-                                    "Domain autorenewal by ExpandRecurringBillingEventsAction")
-                                .setRequestedByRegistrar(false)
-                                .setType(DOMAIN_AUTORENEW)
-                                // Don't write a domain transaction record if the recurrence was
-                                // ended prior to the billing time (i.e. a domain was deleted
-                                // during the autorenew grace period).
-                                .setDomainTransactionRecords(
-                                    recurring.getRecurrenceEndTime().isBefore(billingTime)
-                                        ? ImmutableSet.of()
-                                        : ImmutableSet.of(
-                                            DomainTransactionRecord.create(
-                                                tld.getTldStr(),
-                                                // We report this when the autorenew grace period
-                                                // ends
-                                                billingTime,
-                                                TransactionReportField.netRenewsFieldFromYears(1),
-                                                1)))
-                                .build();
-                        historyEntriesBuilder.add(historyEntry);
-
-                        DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
-                        // Determine the cost for a one-year renewal.
-                        Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
-                        syntheticOneTimesBuilder.add(
-                            new OneTime.Builder()
-                                .setBillingTime(billingTime)
-                                .setClientId(recurring.getClientId())
-                                .setCost(renewCost)
-                                .setEventTime(eventTime)
-                                .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
-                                .setParent(historyEntry)
-                                .setPeriodYears(1)
-                                .setReason(recurring.getReason())
-                                .setSyntheticCreationTime(executeTime)
-                                .setCancellationMatchingBillingEvent(recurring.createVKey())
-                                .setTargetId(recurring.getTargetId())
-                                .build());
-                      }
-                      Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
-                      Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
-                      if (!isDryRun) {
-                        ImmutableSet<ImmutableObject> entitiesToSave =
-                            new ImmutableSet.Builder<ImmutableObject>()
-                                .addAll(historyEntries)
-                                .addAll(syntheticOneTimes)
-                                .build();
-                        ofy().save().entities(entitiesToSave).now();
-                      }
-                      return syntheticOneTimes.size();
-                    });
+                    () -> expandBillingEvent(recurring, executeTime, cursorTime, isDryRun));
      } catch (Throwable t) {
        getContext().incrementCounter("error: " + t.getClass().getSimpleName());
        getContext().incrementCounter(ERROR_COUNTER);
@@ -256,45 +225,12 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
      if (!isDryRun) {
        getContext().incrementCounter("Saved OneTime billing events", numBillingEventsSaved);
      } else {
-        getContext().incrementCounter(
-            "Generated OneTime billing events (dry run)", numBillingEventsSaved);
+        getContext()
+            .incrementCounter("Generated OneTime billing events (dry run)", numBillingEventsSaved);
      }
    }
-
-    /**
-     * Filters a set of {@link DateTime}s down to event times that are in scope for a particular
-     * mapreduce run, given the cursor time and the mapreduce execution time.
-     */
-    private ImmutableSet<DateTime> getBillingTimesInScope(
-        Iterable<DateTime> eventTimes,
-        DateTime cursorTime,
-        DateTime executeTime,
-        final Registry tld) {
-      return Streams.stream(eventTimes)
-          .map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
-          .filter(Range.closedOpen(cursorTime, executeTime))
-          .collect(toImmutableSet());
-    }
-
-    /**
-     * Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted
-     * for a given recurring billing event.
-     */
-    private ImmutableSet<DateTime> getExistingBillingTimes(
-        Iterable<BillingEvent.OneTime> oneTimesForDomain,
-        final BillingEvent.Recurring recurringEvent) {
-      return Streams.stream(oneTimesForDomain)
-          .filter(
-              billingEvent ->
-                  recurringEvent
-                      .createVKey()
-                      .equals(billingEvent.getCancellationMatchingBillingEvent()))
-          .map(OneTime::getBillingTime)
-          .collect(toImmutableSet());
-    }
  }

-
  /**
   * "Reducer" to advance the cursor after all map jobs have been completed. The NullInput into the
   * mapper will cause the mapper to emit one timestamp pair (current cursor and execution time),
@@ -327,7 +263,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
          isDryRun ? "(dry run) " : "", cursorTime, executionTime);
      tm().transact(
              () -> {
-                Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
+                Cursor cursor =
+                    auditedOfy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
                DateTime currentCursorTime =
                    (cursor == null ? START_OF_TIME : cursor.getCursorTime());
                if (!currentCursorTime.equals(expectedPersistedCursorTime)) {
@@ -342,4 +279,135 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
              });
    }
  }
+
+  private static int expandBillingEvent(
+      Recurring recurring, DateTime executeTime, DateTime cursorTime, boolean isDryRun) {
+    ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder = new ImmutableSet.Builder<>();
+    final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
+
+    // Determine the complete set of times at which this recurring event should
+    // occur (up to and including the runtime of the mapreduce).
+    Iterable<DateTime> eventTimes =
+        recurring
+            .getRecurrenceTimeOfYear()
+            .getInstancesInRange(
+                Range.closed(
+                    recurring.getEventTime(),
+                    earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
+
+    // Convert these event times to billing times
+    final ImmutableSet<DateTime> billingTimes =
+        getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
+
+    VKey<DomainBase> domainKey =
+        VKey.create(
+            DomainBase.class, recurring.getDomainRepoId(), recurring.getParentKey().getParent());
+    Iterable<OneTime> oneTimesForDomain;
+    if (tm().isOfy()) {
+      oneTimesForDomain = auditedOfy().load().type(OneTime.class).ancestor(domainKey.getOfyKey());
+    } else {
+      oneTimesForDomain =
+          tm().createQueryComposer(OneTime.class)
+              .where("domainRepoId", EQ, recurring.getDomainRepoId())
+              .list();
+    }
+
+    // Determine the billing times that already have OneTime events persisted.
+    ImmutableSet<DateTime> existingBillingTimes =
+        getExistingBillingTimes(oneTimesForDomain, recurring);
+
+    ImmutableSet.Builder<DomainHistory> historyEntriesBuilder = new ImmutableSet.Builder<>();
+    // Create synthetic OneTime events for all billing times that do not yet have
+    // an event persisted.
+    for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
+      // Construct a new HistoryEntry that parents over the OneTime
+      DomainHistory historyEntry =
+          new DomainHistory.Builder()
+              .setBySuperuser(false)
+              .setRegistrarId(recurring.getRegistrarId())
+              .setModificationTime(tm().getTransactionTime())
+              .setDomain(tm().loadByKey(domainKey))
+              .setPeriod(Period.create(1, YEARS))
+              .setReason("Domain autorenewal by ExpandRecurringBillingEventsAction")
+              .setRequestedByRegistrar(false)
+              .setType(DOMAIN_AUTORENEW)
+              // Don't write a domain transaction record if the recurrence was
+              // ended prior to the billing time (i.e. a domain was deleted
+              // during the autorenew grace period).
+              .setDomainTransactionRecords(
+                  recurring.getRecurrenceEndTime().isBefore(billingTime)
+                      ? ImmutableSet.of()
+                      : ImmutableSet.of(
+                          DomainTransactionRecord.create(
+                              tld.getTldStr(),
+                              // We report this when the autorenew grace period
+                              // ends
+                              billingTime,
+                              TransactionReportField.netRenewsFieldFromYears(1),
+                              1)))
+              .build();
+      historyEntriesBuilder.add(historyEntry);
+
+      DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
+      // Determine the cost for a one-year renewal.
+      Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
+      syntheticOneTimesBuilder.add(
+          new OneTime.Builder()
+              .setBillingTime(billingTime)
+              .setRegistrarId(recurring.getRegistrarId())
+              .setCost(renewCost)
+              .setEventTime(eventTime)
+              .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
+              .setParent(historyEntry)
+              .setPeriodYears(1)
+              .setReason(recurring.getReason())
+              .setSyntheticCreationTime(executeTime)
+              .setCancellationMatchingBillingEvent(recurring.createVKey())
+              .setTargetId(recurring.getTargetId())
+              .build());
+    }
+    Set<DomainHistory> historyEntries = historyEntriesBuilder.build();
+    Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
+    if (!isDryRun) {
+      ImmutableSet<ImmutableObject> entitiesToSave =
+          new ImmutableSet.Builder<ImmutableObject>()
+              .addAll(historyEntries)
+              .addAll(syntheticOneTimes)
+              .build();
+      tm().putAll(entitiesToSave);
+    }
+    return syntheticOneTimes.size();
+  }
+
+  /**
+   * Filters a set of {@link DateTime}s down to event times that are in scope for a particular
+   * mapreduce run, given the cursor time and the mapreduce execution time.
+   */
+  protected static ImmutableSet<DateTime> getBillingTimesInScope(
+      Iterable<DateTime> eventTimes,
+      DateTime cursorTime,
+      DateTime executeTime,
+      final Registry tld) {
+    return Streams.stream(eventTimes)
+        .map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
+        .filter(Range.closedOpen(cursorTime, executeTime))
+        .collect(toImmutableSet());
+  }
+
+  /**
+   * Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted for a
+   * given recurring billing event.
+   */
+  private static ImmutableSet<DateTime> getExistingBillingTimes(
+      Iterable<BillingEvent.OneTime> oneTimesForDomain,
+      final BillingEvent.Recurring recurringEvent) {
+    return Streams.stream(oneTimesForDomain)
+        .filter(
+            billingEvent ->
+                recurringEvent
+                    .createVKey()
+                    .equals(billingEvent.getCancellationMatchingBillingEvent()))
+        .map(OneTime::getBillingTime)
+        .collect(toImmutableSet());
+  }
 }
--- a/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java
+++ b/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java
@@ -23,6 +23,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
+import static google.registry.model.EppResourceUtils.getLinkedDomainKeys;
 import static google.registry.model.EppResourceUtils.isActive;
 import static google.registry.model.EppResourceUtils.isDeleted;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -69,6 +70,7 @@ import java.util.logging.Level;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Named;
+import org.apache.http.HttpStatus;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;

@@ -86,6 +88,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
  @Inject Clock clock;
  @Inject MapreduceRunner mrRunner;
  @Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue;
+
+  @Inject DnsQueue dnsQueue;
  @Inject RequestStatusChecker requestStatusChecker;
  @Inject Response response;
  @Inject Retrier retrier;
@@ -153,7 +157,39 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
    } else {
      logger.atInfo().log(
          "Processing asynchronous DNS refresh for renamed hosts: %s", hostKeys.build());
-      runMapreduce(refreshRequests, lock);
+      if (tm().isOfy()) {
+        runMapreduce(refreshRequests, lock);
+      } else {
+        try {
+          refreshRequests.stream()
+              .flatMap(
+                  request ->
+                      getLinkedDomainKeys(request.hostKey(), request.lastUpdateTime(), null)
+                          .stream())
+              .distinct()
+              .map(domainKey -> tm().transact(() -> tm().loadByKey(domainKey).getDomainName()))
+              .forEach(
+                  domainName -> {
+                    retrier.callWithRetry(
+                        () -> dnsQueue.addDomainRefreshTask(domainName),
+                        TransientFailureException.class);
+                    logger.atInfo().log("Enqueued DNS refresh for domain %s.", domainName);
+                  });
+          deleteTasksWithRetry(
+              refreshRequests,
+              getQueue(QUEUE_ASYNC_HOST_RENAME),
+              asyncTaskMetrics,
+              retrier,
+              OperationResult.SUCCESS);
+        } catch (Throwable t) {
+          String message = "Error refreshing DNS on host rename.";
+          logger.atSevere().withCause(t).log(message);
+          response.setPayload(message);
+          response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR);
+        } finally {
+          lock.get().release();
+        }
+      }
    }
  }

--- a/core/src/main/java/google/registry/batch/RelockDomainAction.java
+++ b/core/src/main/java/google/registry/batch/RelockDomainAction.java
@@ -28,16 +28,16 @@ import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
-import google.registry.model.registry.RegistryLockDao;
+import google.registry.model.tld.RegistryLockDao;
 import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.schema.domain.RegistryLock;
 import google.registry.tools.DomainLockUtils;
 import google.registry.util.DateTimeUtils;
 import google.registry.util.EmailMessage;
@@ -203,11 +203,11 @@ public class RelockDomainAction implements Runnable {
        "Domain %s has a pending transfer.",
        domainName);
    checkArgument(
-        domain.getCurrentSponsorClientId().equals(oldLock.getRegistrarId()),
+        domain.getCurrentSponsorRegistrarId().equals(oldLock.getRegistrarId()),
        "Domain %s has been transferred from registrar %s to registrar %s since the unlock.",
        domainName,
        oldLock.getRegistrarId(),
-        domain.getCurrentSponsorClientId());
+        domain.getCurrentSponsorRegistrarId());
  }

  private void handleNonRetryableFailure(RegistryLock oldLock, Throwable t) {
@@ -293,7 +293,7 @@ public class RelockDomainAction implements Runnable {

  private ImmutableSet<InternetAddress> getEmailRecipients(String registrarId) {
    Registrar registrar =
-        Registrar.loadByClientIdCached(registrarId)
+        Registrar.loadByRegistrarIdCached(registrarId)
            .orElseThrow(
                () ->
                    new IllegalStateException(String.format("Unknown registrar %s", registrarId)));
--- a/core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java
+++ b/core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java
@@ -15,7 +15,7 @@
 package google.registry.batch;

 import static google.registry.mapreduce.MapreduceRunner.PARAM_FAST;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;

 import com.google.appengine.tools.mapreduce.Mapper;
@@ -54,6 +54,8 @@ import javax.inject.Inject;
    service = Action.Service.BACKEND,
    path = "/_dr/task/resaveAllEppResources",
    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+// No longer needed in SQL. Subject to future removal.
+@Deprecated
 public class ResaveAllEppResourcesAction implements Runnable {

  @Inject MapreduceRunner mrRunner;
@@ -104,13 +106,13 @@ public class ResaveAllEppResourcesAction implements Runnable {
      boolean resaved =
          tm().transact(
                  () -> {
-                    EppResource originalResource = ofy().load().key(resourceKey).now();
+                    EppResource originalResource = auditedOfy().load().key(resourceKey).now();
                    EppResource projectedResource =
                        originalResource.cloneProjectedAtTime(tm().getTransactionTime());
                    if (isFast && originalResource.equals(projectedResource)) {
                      return false;
                    } else {
-                      ofy().save().entity(projectedResource).now();
+                      auditedOfy().save().entity(projectedResource).now();
                      return true;
                    }
                  });
--- a/core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java
+++ b/core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java
@@ -0,0 +1,342 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.joda.time.DateTimeZone.UTC;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.flows.certs.CertificateChecker;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registrar.RegistrarContact.Type;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.EmailMessage;
+import google.registry.util.SendEmailService;
+import java.util.Date;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.mail.internet.AddressException;
+import javax.mail.internet.InternetAddress;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/** An action that sends notification emails to registrars whose certificates are expiring soon. */
+@Action(
+    service = Action.Service.BACKEND,
+    path = SendExpiringCertificateNotificationEmailAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class SendExpiringCertificateNotificationEmailAction implements Runnable {
+  public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
+  /**
+   * Used as an offset when storing the last notification email sent date.
+   *
+   * <p>This is used to handle edges cases when the update happens in between the day switch. For
+   * instance,if the job starts at 2:00 am every day and it finishes at 2:03 of the same day, then
+   * next day at 2am, the date difference will be less than a day, which will lead to the date
+   * difference between two successive email sent date being the expected email interval days + 1;
+   */
+  protected static final Duration UPDATE_TIME_OFFSET = Duration.standardMinutes(10);
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd");
+
+  private final CertificateChecker certificateChecker;
+  private final String expirationWarningEmailBodyText;
+  private final SendEmailService sendEmailService;
+  private final String expirationWarningEmailSubjectText;
+  private final InternetAddress gSuiteOutgoingEmailAddress;
+  private final Response response;
+
+  @Inject
+  public SendExpiringCertificateNotificationEmailAction(
+      @Config("expirationWarningEmailBodyText") String expirationWarningEmailBodyText,
+      @Config("expirationWarningEmailSubjectText") String expirationWarningEmailSubjectText,
+      @Config("gSuiteOutgoingEmailAddress") InternetAddress gSuiteOutgoingEmailAddress,
+      SendEmailService sendEmailService,
+      CertificateChecker certificateChecker,
+      Response response) {
+    this.certificateChecker = certificateChecker;
+    this.expirationWarningEmailSubjectText = expirationWarningEmailSubjectText;
+    this.sendEmailService = sendEmailService;
+    this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress;
+    this.expirationWarningEmailBodyText = expirationWarningEmailBodyText;
+    this.response = response;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      sendNotificationEmails();
+      response.setStatus(SC_OK);
+    } catch (Exception e) {
+      logger.atWarning().withCause(e).log(
+          "Exception thrown when sending expiring certificate notification emails.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(String.format("Exception thrown with cause: %s", e));
+    }
+  }
+
+  /**
+   * Returns a list of registrars that should receive expiring notification emails. There are two
+   * certificates that should be considered (the main certificate and failOver certificate). The
+   * registrars should receive notifications if one of the certificate checks returns true.
+   */
+  @VisibleForTesting
+  ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
+    logger.atInfo().log(
+        "Getting a list of registrars that should receive expiring notification emails.");
+    return Streams.stream(Registrar.loadAllCached())
+        .map(
+            registrar ->
+                RegistrarInfo.create(
+                    registrar,
+                    registrar.getClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringCertNotificationSentDate(),
+                            registrar.getClientCertificate().get()),
+                    registrar.getFailoverClientCertificate().isPresent()
+                        && certificateChecker.shouldReceiveExpiringNotification(
+                            registrar.getLastExpiringFailoverCertNotificationSentDate(),
+                            registrar.getFailoverClientCertificate().get())))
+        .filter(
+            registrarInfo ->
+                registrarInfo.isCertExpiring() || registrarInfo.isFailOverCertExpiring())
+        .collect(toImmutableList());
+  }
+
+  /**
+   * Sends a notification email to the registrar regarding the expiring certificate and returns true
+   * if it's sent successfully.
+   */
+  @VisibleForTesting
+  boolean sendNotificationEmail(
+      Registrar registrar,
+      DateTime lastExpiringCertNotificationSentDate,
+      CertificateType certificateType,
+      Optional<String> certificate) {
+    if (!certificate.isPresent()
+        || !certificateChecker.shouldReceiveExpiringNotification(
+            lastExpiringCertNotificationSentDate, certificate.get())) {
+      return false;
+    }
+    try {
+      ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
+      logger.atInfo().log(
+          "Registrar %s should receive an email that its %s SSL certificate will expire on %s.",
+          registrar.getRegistrarName(),
+          certificateType.getDisplayName(),
+          expirationDate.toString());
+      if (recipients.isEmpty()) {
+        logger.atWarning().log(
+            "Registrar %s contains no email addresses to receive notification email.",
+            registrar.getRegistrarName());
+        return false;
+      }
+      sendEmailService.sendEmail(
+          EmailMessage.newBuilder()
+              .setFrom(gSuiteOutgoingEmailAddress)
+              .setSubject(expirationWarningEmailSubjectText)
+              .setBody(
+                  getEmailBody(
+                      registrar.getRegistrarName(),
+                      certificateType,
+                      expirationDate,
+                      registrar.getRegistrarId()))
+              .setRecipients(recipients)
+              .setCcs(getEmailAddresses(registrar, Type.ADMIN))
+              .build());
+      /*
+       * A duration time offset is used here to ensure that date comparison between two
+       * successive dates is always greater than 1 day. This date is set as last updated date,
+       * for applicable certificate.
+       */
+      updateLastNotificationSentDate(
+          registrar,
+          DateTime.now(UTC).minusMinutes((int) UPDATE_TIME_OFFSET.getStandardMinutes()),
+          certificateType);
+      return true;
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to send expiring certificate notification email to registrar %s.",
+              registrar.getRegistrarName()));
+    }
+  }
+
+  /** Updates the last notification sent date in database. */
+  @VisibleForTesting
+  void updateLastNotificationSentDate(
+      Registrar registrar, DateTime now, CertificateType certificateType) {
+    try {
+      tm().transact(
+              () -> {
+                Registrar.Builder newRegistrar = tm().loadByEntity(registrar).asBuilder();
+                switch (certificateType) {
+                  case PRIMARY:
+                    newRegistrar.setLastExpiringCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  case FAILOVER:
+                    newRegistrar.setLastExpiringFailoverCertNotificationSentDate(now);
+                    tm().put(newRegistrar.build());
+                    logger.atInfo().log(
+                        "Updated last notification email sent date to %s for %s certificate of "
+                            + "registrar %s.",
+                        DATE_FORMATTER.print(now),
+                        certificateType.getDisplayName(),
+                        registrar.getRegistrarName());
+                    break;
+                  default:
+                    throw new IllegalArgumentException(
+                        String.format(
+                            "Unsupported certificate type: %s being passed in when updating "
+                                + "the last notification sent date to registrar %s.",
+                            certificateType.toString(), registrar.getRegistrarName()));
+                }
+              });
+    } catch (Exception e) {
+      throw new RuntimeException(
+          String.format(
+              "Failed to update the last notification sent date to Registrar %s for the %s "
+                  + "certificate.",
+              registrar.getRegistrarName(), certificateType.getDisplayName()));
+    }
+  }
+
+  /** Sends notification emails to registrars with expiring certificates. */
+  @VisibleForTesting
+  int sendNotificationEmails() {
+    int emailsSent = 0;
+    for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
+      Registrar registrar = registrarInfo.registrar();
+      if (registrarInfo.isCertExpiring()) {
+        sendNotificationEmail(
+            registrar,
+            registrar.getLastExpiringCertNotificationSentDate(),
+            CertificateType.PRIMARY,
+            registrar.getClientCertificate());
+        emailsSent++;
+      }
+      if (registrarInfo.isFailOverCertExpiring()) {
+        sendNotificationEmail(
+            registrar,
+            registrar.getLastExpiringFailoverCertNotificationSentDate(),
+            CertificateType.FAILOVER,
+            registrar.getFailoverClientCertificate());
+        emailsSent++;
+      }
+    }
+    logger.atInfo().log(
+        "Attempted to send %d expiring certificate notification emails.", emailsSent);
+    return emailsSent;
+  }
+
+  /** Returns a list of email addresses of the registrar that should receive a notification email */
+  @VisibleForTesting
+  ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
+    ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
+    ImmutableSet.Builder<InternetAddress> recipientEmails = new ImmutableSet.Builder<>();
+    for (RegistrarContact contact : contacts) {
+      try {
+        recipientEmails.add(new InternetAddress(contact.getEmailAddress()));
+      } catch (AddressException e) {
+        logger.atWarning().withCause(e).log(
+            "Registrar Contact email address %s of Registrar %s is invalid; skipping.",
+            contact.getEmailAddress(), registrar.getRegistrarName());
+      }
+    }
+    return recipientEmails.build();
+  }
+
+  /**
+   * Generates email content by taking registrar name, certificate type and expiration date as
+   * parameters.
+   */
+  @VisibleForTesting
+  @SuppressWarnings("lgtm[java/dereferenced-value-may-be-null]")
+  String getEmailBody(
+      String registrarName, CertificateType type, Date expirationDate, String registrarId) {
+    checkArgumentNotNull(expirationDate, "Expiration date cannot be null");
+    checkArgumentNotNull(type, "Certificate type cannot be null");
+    checkArgumentNotNull(registrarId, "Registrar Id cannot be null");
+    return String.format(
+        expirationWarningEmailBodyText,
+        registrarName,
+        type.getDisplayName(),
+        DATE_FORMATTER.print(new DateTime(expirationDate)),
+        registrarId);
+  }
+
+  /**
+   * Certificate types for X509Certificate.
+   *
+   * <p><b>Note:</b> These types are only used to indicate the type of expiring certificate in
+   * notification emails.
+   */
+  protected enum CertificateType {
+    PRIMARY("primary"),
+    FAILOVER("fail-over");
+
+    private final String displayName;
+
+    CertificateType(String displayName) {
+      this.displayName = displayName;
+    }
+
+    public String getDisplayName() {
+      return displayName;
+    }
+  }
+
+  @AutoValue
+  public abstract static class RegistrarInfo {
+    static RegistrarInfo create(
+        Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
+      return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(
+          registrar, isCertExpiring, isFailOverCertExpiring);
+    }
+
+    public abstract Registrar registrar();
+
+    public abstract boolean isCertExpiring();
+
+    public abstract boolean isFailOverCertExpiring();
+  }
+}
--- a/core/src/main/java/google/registry/batch/WipeOutCloudSqlAction.java
+++ b/core/src/main/java/google/registry/batch/WipeOutCloudSqlAction.java
@@ -19,19 +19,21 @@ import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;

+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Retrier;
 import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.function.Supplier;
 import javax.inject.Inject;
-import org.flywaydb.core.api.FlywayException;

 /**
 * Wipes out all Cloud SQL data in a Nomulus GCP environment.
@@ -46,22 +48,18 @@ import org.flywaydb.core.api.FlywayException;
 public class WipeOutCloudSqlAction implements Runnable {
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

-  // As a short-lived class, hardcode allowed projects here instead of using config files.
-  private static final ImmutableSet<String> ALLOWED_PROJECTS =
-      ImmutableSet.of("domain-registry-qa");
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);

-  private final String projectId;
  private final Supplier<Connection> connectionSupplier;
  private final Response response;
  private final Retrier retrier;

  @Inject
  WipeOutCloudSqlAction(
-      @Config("projectId") String projectId,
      @SchemaManagerConnection Supplier<Connection> connectionSupplier,
      Response response,
      Retrier retrier) {
-    this.projectId = projectId;
    this.connectionSupplier = connectionSupplier;
    this.response = response;
    this.retrier = retrier;
@@ -71,28 +69,93 @@ public class WipeOutCloudSqlAction implements Runnable {
  public void run() {
    response.setContentType(PLAIN_TEXT_UTF_8);

-    if (!ALLOWED_PROJECTS.contains(projectId)) {
+    if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
      response.setStatus(SC_FORBIDDEN);
-      response.setPayload("Wipeout is not allowed in " + projectId);
+      response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
      return;
    }

    try {
      retrier.callWithRetry(
          () -> {
-            try (Connection conn = connectionSupplier.get();
-                Statement statement = conn.createStatement()) {
-              statement.execute("drop owned by schema_deployer;");
+            try (Connection conn = connectionSupplier.get()) {
+              dropAllTables(conn, listTables(conn));
+              dropAllSequences(conn, listSequences(conn));
            }
            return null;
          },
-          e -> !(e instanceof FlywayException));
+          e -> !(e instanceof SQLException));
      response.setStatus(SC_OK);
-      response.setPayload("Wiped out Cloud SQL in " + projectId);
+      response.setPayload("Wiped out Cloud SQL in " + RegistryEnvironment.get());
    } catch (RuntimeException e) {
      logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
      response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
+      response.setPayload("Failed to wipe out Cloud SQL in " + RegistryEnvironment.get());
+    }
+  }
+
+  /** Returns a list of all tables in the public schema of a Postgresql database. */
+  static ImmutableList<String> listTables(Connection connection) throws SQLException {
+    try (ResultSet resultSet =
+        connection.getMetaData().getTables(null, null, null, new String[] {"TABLE"})) {
+      ImmutableList.Builder<String> tables = new ImmutableList.Builder<>();
+      while (resultSet.next()) {
+        String schema = resultSet.getString("TABLE_SCHEM");
+        if (schema == null || !schema.equalsIgnoreCase("public")) {
+          continue;
+        }
+        String tableName = resultSet.getString("TABLE_NAME");
+        tables.add("public.\"" + tableName + "\"");
+      }
+      return tables.build();
+    }
+  }
+
+  static void dropAllTables(Connection conn, ImmutableList<String> tables) throws SQLException {
+    if (tables.isEmpty()) {
+      return;
+    }
+
+    try (Statement statement = conn.createStatement()) {
+      for (String table : tables) {
+        statement.addBatch(String.format("DROP TABLE IF EXISTS %s CASCADE;", table));
+      }
+      for (int code : statement.executeBatch()) {
+        if (code == Statement.EXECUTE_FAILED) {
+          throw new RuntimeException("Failed to drop some tables. Please check.");
+        }
+      }
+    }
+  }
+
+  /** Returns a list of all sequences in a Postgresql database. */
+  static ImmutableList<String> listSequences(Connection conn) throws SQLException {
+    try (Statement statement = conn.createStatement();
+        ResultSet resultSet =
+            statement.executeQuery("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';")) {
+      ImmutableList.Builder<String> sequences = new ImmutableList.Builder<>();
+      while (resultSet.next()) {
+        sequences.add('\"' + resultSet.getString(1) + '\"');
+      }
+      return sequences.build();
+    }
+  }
+
+  static void dropAllSequences(Connection conn, ImmutableList<String> sequences)
+      throws SQLException {
+    if (sequences.isEmpty()) {
+      return;
+    }
+
+    try (Statement statement = conn.createStatement()) {
+      for (String sequence : sequences) {
+        statement.addBatch(String.format("DROP SEQUENCE IF EXISTS %s CASCADE;", sequence));
+      }
+      for (int code : statement.executeBatch()) {
+        if (code == Statement.EXECUTE_FAILED) {
+          throw new RuntimeException("Failed to drop some sequences. Please check.");
+        }
+      }
    }
  }
 }
--- a/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java
+++ b/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
@@ -49,9 +50,8 @@ public class WipeoutDatastoreAction implements Runnable {

  private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";

-  // As a short-lived class, hardcode allowed projects here instead of using config files.
-  private static final ImmutableSet<String> ALLOWED_PROJECTS =
-      ImmutableSet.of("domain-registry-qa");
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);

  private final String projectId;
  private final String jobRegion;
@@ -80,9 +80,9 @@ public class WipeoutDatastoreAction implements Runnable {
  public void run() {
    response.setContentType(PLAIN_TEXT_UTF_8);

-    if (!ALLOWED_PROJECTS.contains(projectId)) {
+    if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
      response.setStatus(SC_FORBIDDEN);
-      response.setPayload("Wipeout is not allowed in " + projectId);
+      response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
      return;
    }

--- a/core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java
+++ b/core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java
@@ -18,17 +18,23 @@ import static com.google.common.base.Verify.verify;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;

 import google.registry.backup.AppEngineEnvironment;
+import google.registry.model.contact.ContactResource;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import java.io.Serializable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;

-/** Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines. */
+/**
+ * Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines.
+ *
+ * <p>This pipeline may also be used as an integration test for {@link RegistryJpaIO.Read} in a
+ * project with realistic data.
+ */
 public class JpaDemoPipeline implements Serializable {

  public static void main(String[] args) {
@@ -38,23 +44,16 @@ public class JpaDemoPipeline implements Serializable {

    Pipeline pipeline = Pipeline.create(options);
    pipeline
-        .apply("Start", Create.of((Void) null))
        .apply(
-            "Generate Elements",
-            ParDo.of(
-                new DoFn<Void, Void>() {
-                  @ProcessElement
-                  public void processElement(OutputReceiver<Void> output) {
-                    for (int i = 0; i < 500; i++) {
-                      output.output(null);
-                    }
-                  }
-                }))
+            "Read contacts",
+            RegistryJpaIO.read(
+                () -> CriteriaQueryBuilder.create(ContactResource.class).build(),
+                ContactResource::getRepoId))
        .apply(
-            "Make Query",
+            "Count Contacts",
            ParDo.of(
-                new DoFn<Void, Void>() {
-                  private Counter counter = Metrics.counter("Demo", "Read");
+                new DoFn<String, Void>() {
+                  private Counter counter = Metrics.counter("Contacts", "Read");

                  @ProcessElement
                  public void processElement() {
--- a/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java
+++ b/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java
@@ -21,21 +21,34 @@ import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
 import google.registry.backup.AppEngineEnvironment;
+import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
 import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
+import java.io.Serializable;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.Nullable;
+import javax.persistence.criteria.CriteriaQuery;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupIntoBatches;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.util.ShardedKey;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;

 /**
@@ -51,10 +64,165 @@ public final class RegistryJpaIO {

  private RegistryJpaIO() {}

+  public static <R> Read<R, R> read(CriteriaQuerySupplier<R> query) {
+    return read(query, x -> x);
+  }
+
+  public static <R, T> Read<R, T> read(
+      CriteriaQuerySupplier<R> query, SerializableFunction<R, T> resultMapper) {
+    return Read.<R, T>builder().criteriaQuery(query).resultMapper(resultMapper).build();
+  }
+
+  public static <R, T> Read<R, T> read(
+      String sql, boolean nativeQuery, SerializableFunction<R, T> resultMapper) {
+    return read(sql, null, nativeQuery, resultMapper);
+  }
+
+  /**
+   * Returns a {@link Read} connector based on the given {@code jpql} query string.
+   *
+   * <p>User should take care to prevent sql-injection attacks.
+   */
+  public static <R, T> Read<R, T> read(
+      String sql,
+      @Nullable Map<String, Object> parameter,
+      boolean nativeQuery,
+      SerializableFunction<R, T> resultMapper) {
+    Read.Builder<R, T> builder = Read.builder();
+    if (nativeQuery) {
+      builder.nativeQuery(sql, parameter);
+    } else {
+      builder.jpqlQuery(sql, parameter);
+    }
+    return builder.resultMapper(resultMapper).build();
+  }
+
+  public static <R, T> Read<R, T> read(
+      String jpql, Class<R> clazz, SerializableFunction<R, T> resultMapper) {
+    return read(jpql, null, clazz, resultMapper);
+  }
+
+  /**
+   * Returns a {@link Read} connector based on the given {@code jpql} typed query string.
+   *
+   * <p>User should take care to prevent sql-injection attacks.
+   */
+  public static <R, T> Read<R, T> read(
+      String jpql,
+      @Nullable Map<String, Object> parameter,
+      Class<R> clazz,
+      SerializableFunction<R, T> resultMapper) {
+    return Read.<R, T>builder()
+        .jpqlQuery(jpql, clazz, parameter)
+        .resultMapper(resultMapper)
+        .build();
+  }
+
  public static <T> Write<T> write() {
    return Write.<T>builder().build();
  }

+  /**
+   * A {@link PTransform transform} that transactionally executes a JPA {@link CriteriaQuery} and
+   * adds the results to the BEAM pipeline. Users have the option to transform the results before
+   * sending them to the next stages.
+   */
+  @AutoValue
+  public abstract static class Read<R, T> extends PTransform<PBegin, PCollection<T>> {
+
+    public static final String DEFAULT_NAME = "RegistryJpaIO.Read";
+
+    abstract String name();
+
+    abstract RegistryQuery<R> query();
+
+    abstract SerializableFunction<R, T> resultMapper();
+
+    abstract Coder<T> coder();
+
+    abstract Builder<R, T> toBuilder();
+
+    @Override
+    @SuppressWarnings("deprecation") // Reshuffle still recommended by GCP.
+    public PCollection<T> expand(PBegin input) {
+      return input
+          .apply("Starting " + name(), Create.of((Void) null))
+          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
+          .setCoder(coder())
+          .apply("Reshuffle", Reshuffle.viaRandomKey());
+    }
+
+    public Read<R, T> withName(String name) {
+      return toBuilder().name(name).build();
+    }
+
+    public Read<R, T> withResultMapper(SerializableFunction<R, T> mapper) {
+      return toBuilder().resultMapper(mapper).build();
+    }
+
+    public Read<R, T> withCoder(Coder<T> coder) {
+      return toBuilder().coder(coder).build();
+    }
+
+    static <R, T> Builder<R, T> builder() {
+      return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
+          .name(DEFAULT_NAME)
+          .coder(SerializableCoder.of(Serializable.class));
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder<R, T> {
+
+      abstract Builder<R, T> name(String name);
+
+      abstract Builder<R, T> query(RegistryQuery<R> query);
+
+      abstract Builder<R, T> resultMapper(SerializableFunction<R, T> mapper);
+
+      abstract Builder<R, T> coder(Coder coder);
+
+      abstract Read<R, T> build();
+
+      Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
+        return query(RegistryQuery.createQuery(criteriaQuery));
+      }
+
+      Builder<R, T> nativeQuery(String sql, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(sql, parameters, true));
+      }
+
+      Builder<R, T> jpqlQuery(String jpql, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(jpql, parameters, false));
+      }
+
+      Builder<R, T> jpqlQuery(String jpql, Class<R> clazz, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(jpql, parameters, clazz));
+      }
+    }
+
+    static class QueryRunner<R, T> extends DoFn<Void, T> {
+      private final RegistryQuery<R> query;
+      private final SerializableFunction<R, T> resultMapper;
+
+      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+        this.query = query;
+        this.resultMapper = resultMapper;
+      }
+
+      @ProcessElement
+      public void processElement(OutputReceiver<T> outputReceiver) {
+        // AppEngineEnvironment is need for handling VKeys, which involve Ofy keys. Unlike
+        // SqlBatchWriter, it is unnecessary to initialize ObjectifyService in this class.
+        try (AppEngineEnvironment env = new AppEngineEnvironment()) {
+          // TODO(b/187210388): JpaTransactionManager should support non-transactional query.
+          jpaTm()
+              .transactNoRetry(
+                  () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
+        }
+      }
+    }
+  }
+
  /**
   * A {@link PTransform transform} that writes a PCollection of entities to the SQL database using
   * the {@link JpaTransactionManager}.
@@ -108,6 +276,12 @@ public final class RegistryJpaIO {

    public abstract SerializableFunction<T, Object> jpaConverter();

+    /**
+     * Signal to the writer that the {@link UpdateAutoTimestamp} property should be allowed to
+     * manipulate its value before persistence. The default value is {@code true}.
+     */
+    abstract boolean withUpdateAutoTimestamp();
+
    public Write<T> withName(String name) {
      return toBuilder().name(name).build();
    }
@@ -128,6 +302,10 @@ public final class RegistryJpaIO {
      return toBuilder().jpaConverter(jpaConverter).build();
    }

+    public Write<T> disableUpdateAutoTimestamp() {
+      return toBuilder().withUpdateAutoTimestamp(false).build();
+    }
+
    abstract Builder<T> toBuilder();

    @Override
@@ -144,7 +322,7 @@ public final class RegistryJpaIO {
              GroupIntoBatches.<Integer, T>ofSize(batchSize()).withShardedKey())
          .apply(
              "Write in batch for " + name(),
-              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter())));
+              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter(), withUpdateAutoTimestamp())));
    }

    static <T> Builder<T> builder() {
@@ -152,7 +330,8 @@ public final class RegistryJpaIO {
          .name(DEFAULT_NAME)
          .batchSize(DEFAULT_BATCH_SIZE)
          .shards(DEFAULT_SHARDS)
-          .jpaConverter(x -> x);
+          .jpaConverter(x -> x)
+          .withUpdateAutoTimestamp(true);
    }

    @AutoValue.Builder
@@ -166,6 +345,8 @@ public final class RegistryJpaIO {

      abstract Builder<T> jpaConverter(SerializableFunction<T, Object> jpaConverter);

+      abstract Builder<T> withUpdateAutoTimestamp(boolean withUpdateAutoTimestamp);
+
      abstract Write<T> build();
    }
  }
@@ -174,16 +355,20 @@ public final class RegistryJpaIO {
  private static class SqlBatchWriter<T> extends DoFn<KV<ShardedKey<Integer>, Iterable<T>>, Void> {
    private final Counter counter;
    private final SerializableFunction<T, Object> jpaConverter;
+    private final boolean withAutoTimestamp;

-    SqlBatchWriter(String type, SerializableFunction<T, Object> jpaConverter) {
+    SqlBatchWriter(
+        String type, SerializableFunction<T, Object> jpaConverter, boolean withAutoTimestamp) {
      counter = Metrics.counter("SQL_WRITE", type);
      this.jpaConverter = jpaConverter;
+      this.withAutoTimestamp = withAutoTimestamp;
    }

    @Setup
    public void setup() {
-      // Below is needed as long as Objectify keys are still involved in the handling of SQL
-      // entities (e.g., in VKeys).
+      // AppEngineEnvironment is needed as long as Objectify keys are still involved in the handling
+      // of SQL entities (e.g., in VKeys). ObjectifyService needs to be initialized when conversion
+      // between Ofy entity and Datastore entity is needed.
      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
        ObjectifyService.initOfy();
      }
@@ -191,18 +376,28 @@ public final class RegistryJpaIO {

    @ProcessElement
    public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
+      if (withAutoTimestamp) {
+        actuallyProcessElement(kv);
+        return;
+      }
+      try (DisableAutoUpdateResource disable = UpdateAutoTimestamp.disableAutoUpdate()) {
+        actuallyProcessElement(kv);
+      }
+    }
+
+    private void actuallyProcessElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ImmutableList<Object> ofyEntities =
+        ImmutableList<Object> entities =
            Streams.stream(kv.getValue())
                .map(this.jpaConverter::apply)
                // TODO(b/177340730): post migration delete the line below.
                .filter(Objects::nonNull)
                .collect(ImmutableList.toImmutableList());
        try {
-          jpaTm().transact(() -> jpaTm().putAll(ofyEntities));
-          counter.inc(ofyEntities.size());
+          jpaTm().transact(() -> jpaTm().putAll(entities));
+          counter.inc(entities.size());
        } catch (RuntimeException e) {
-          processSingly(ofyEntities);
+          processSingly(entities);
        }
      }
    }
@@ -211,19 +406,22 @@ public final class RegistryJpaIO {
     * Writes entities in a failed batch one by one to identify the first bad entity and throws a
     * {@link RuntimeException} on it.
     */
-    private void processSingly(ImmutableList<Object> ofyEntities) {
-      for (Object ofyEntity : ofyEntities) {
+    private void processSingly(ImmutableList<Object> entities) {
+      for (Object entity : entities) {
        try {
-          jpaTm().transact(() -> jpaTm().put(ofyEntity));
+          jpaTm().transact(() -> jpaTm().put(entity));
          counter.inc();
        } catch (RuntimeException e) {
-          throw new RuntimeException(toOfyKey(ofyEntity).toString(), e);
+          throw new RuntimeException(toEntityKeyString(entity), e);
        }
      }
    }

-    private com.googlecode.objectify.Key<?> toOfyKey(Object ofyEntity) {
-      return com.googlecode.objectify.Key.create(ofyEntity);
+    private String toEntityKeyString(Object entity) {
+      if (entity instanceof SqlEntity) {
+        return ((SqlEntity) entity).getPrimaryKeyString();
+      }
+      return "Non-SqlEntity: " + String.valueOf(entity);
    }
  }
 }
--- a/core/src/main/java/google/registry/beam/common/RegistryQuery.java
+++ b/core/src/main/java/google/registry/beam/common/RegistryQuery.java
@@ -0,0 +1,139 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import google.registry.persistence.transaction.JpaTransactionManager;
+import java.io.Serializable;
+import java.util.Map;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaQuery;
+
+/** Interface for query instances used by {@link RegistryJpaIO.Read}. */
+public interface RegistryQuery<T> extends Serializable {
+
+  /**
+   * Number of JPA entities to fetch in each batch during a query.
+   *
+   * <p>With Hibernate, for result streaming to work, a query's fetchSize property must be set to a
+   * non-zero value.
+   */
+  int QUERY_FETCH_SIZE = 1000;
+
+  Stream<T> stream();
+
+  interface CriteriaQuerySupplier<T> extends Supplier<CriteriaQuery<T>>, Serializable {}
+
+  /**
+   * Returns a {@link RegistryQuery} that creates a string query from constant text.
+   *
+   * @param nativeQuery whether the given string is to be interpreted as a native query or JPQL.
+   * @param parameters parameters to be substituted in the query.
+   * @param <T> Type of each row in the result set, {@link Object} in single-select queries, and
+   *     {@code Object[]} in multi-select queries.
+   */
+  static <T> RegistryQuery<T> createQuery(
+      String sql, @Nullable Map<String, Object> parameters, boolean nativeQuery) {
+    return () -> {
+      EntityManager entityManager = jpaTm().getEntityManager();
+      Query query =
+          nativeQuery ? entityManager.createNativeQuery(sql) : entityManager.createQuery(sql);
+      if (parameters != null) {
+        parameters.forEach(query::setParameter);
+      }
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      @SuppressWarnings("unchecked")
+      Stream<T> resultStream = query.getResultStream();
+      return nativeQuery ? resultStream : resultStream.map(e -> detach(entityManager, e));
+    };
+  }
+
+  /**
+   * Returns a {@link RegistryQuery} that creates a typed JPQL query from constant text.
+   *
+   * @param parameters parameters to be substituted in the query.
+   * @param <T> Type of each row in the result set.
+   */
+  static <T> RegistryQuery<T> createQuery(
+      String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
+    return () -> {
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(jpql, clazz);
+      if (parameters != null) {
+        parameters.forEach(query::setParameter);
+      }
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
+    };
+  }
+
+  /**
+   * Returns a {@link RegistryQuery} from a {@link CriteriaQuery} supplier.
+   *
+   * <p>A serializable supplier is needed in because {@link CriteriaQuery} itself must be created
+   * within a transaction, and we are not in a transaction yet when this function is called to set
+   * up the pipeline.
+   *
+   * @param <T> Type of each row in the result set.
+   */
+  static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
+    return () -> {
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(criteriaQuery.get());
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
+    };
+  }
+
+  /**
+   * Removes an object from the JPA session cache if applicable.
+   *
+   * @param object An object that represents a row in the result set. It may be a JPA entity, a
+   *     non-entity object, or an array that holds JPA entities and/or non-entities.
+   */
+  static <T> T detach(EntityManager entityManager, T object) {
+    if (object.getClass().isArray()) {
+      for (Object arrayElement : (Object[]) object) {
+        detachObject(entityManager, arrayElement);
+      }
+    } else {
+      detachObject(entityManager, object);
+    }
+    return object;
+  }
+
+  static void detachObject(EntityManager entityManager, Object object) {
+    Class<?> objectClass = object.getClass();
+    if (objectClass.isPrimitive() || objectClass == String.class) {
+      return;
+    }
+    try {
+      // TODO(b/193662898): choose detach() or clear() based on the type of transaction.
+      // For context, EntityManager.detach() does not remove all metadata about loaded entities.
+      // See b/193925312 or https://hibernate.atlassian.net/browse/HHH-14735 for details.
+      entityManager.clear();
+    } catch (IllegalArgumentException e) {
+      // Not an entity. Do nothing.
+    }
+  }
+}
--- a/core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java
+++ b/core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java
@@ -87,20 +87,18 @@ public class BulkDeleteDatastorePipeline {

  private final BulkDeletePipelineOptions options;

-  private final Pipeline pipeline;
-
  BulkDeleteDatastorePipeline(BulkDeletePipelineOptions options) {
    this.options = options;
-    pipeline = Pipeline.create(options);
  }

  public void run() {
-    setupPipeline();
+    Pipeline pipeline = Pipeline.create(options);
+    setupPipeline(pipeline);
    pipeline.run();
  }

  @SuppressWarnings("deprecation") // org.apache.beam.sdk.transforms.Reshuffle
-  private void setupPipeline() {
+  private void setupPipeline(Pipeline pipeline) {
    checkState(
        !FORBIDDEN_PROJECTS.contains(options.getProject()),
        "Bulk delete is forbidden in %s",
--- a/core/src/main/java/google/registry/beam/initsql/CloudSqlCredentialDecryptor.java
+++ b/core/src/main/java/google/registry/beam/initsql/CloudSqlCredentialDecryptor.java
@@ -1,50 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.initsql;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.api.services.cloudkms.v1.model.DecryptRequest;
-import com.google.common.base.Strings;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.keyring.kms.KmsConnection;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import javax.inject.Inject;
-
-/**
- * Decrypts data using Cloud KMS, with the same crypto key with which Cloud SQL credential files on
- * GCS was encrypted. See {@link BackupPaths#getCloudSQLCredentialFilePatterns} for more
- * information.
- */
-public class CloudSqlCredentialDecryptor {
-
-  private static final String CRYPTO_KEY_NAME = "nomulus-tool-key";
-  private final KmsConnection kmsConnection;
-
-  @Inject
-  CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
-    this.kmsConnection = kmsConnection;
-  }
-
-  public String decrypt(String data) {
-    checkArgument(!Strings.isNullOrEmpty(data), "Null or empty data.");
-    byte[] ciphertext = Base64.getDecoder().decode(data);
-    // Re-encode for Cloud KMS JSON REST API, invoked through kmsConnection.
-    String urlSafeCipherText = new DecryptRequest().encodeCiphertext(ciphertext).getCiphertext();
-    return new String(
-        kmsConnection.decrypt(CRYPTO_KEY_NAME, urlSafeCipherText), StandardCharsets.UTF_8);
-  }
-}
--- a/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java
+++ b/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java
@@ -32,8 +32,8 @@ import google.registry.model.host.HostResource;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
-import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.io.Serializable;
 import java.util.Collection;
@@ -120,26 +120,22 @@ public class InitSqlPipeline implements Serializable {

  private final InitSqlPipelineOptions options;

-  private final Pipeline pipeline;
-
  InitSqlPipeline(InitSqlPipelineOptions options) {
    this.options = options;
-    pipeline = Pipeline.create(options);
+  }
+
+  PipelineResult run() {
+    return run(Pipeline.create(options));
  }

  @VisibleForTesting
-  InitSqlPipeline(InitSqlPipelineOptions options, Pipeline pipeline) {
-    this.options = options;
-    this.pipeline = pipeline;
-  }
-
-  public PipelineResult run() {
-    setupPipeline();
+  PipelineResult run(Pipeline pipeline) {
+    setupPipeline(pipeline);
    return pipeline.run();
  }

  @VisibleForTesting
-  void setupPipeline() {
+  void setupPipeline(Pipeline pipeline) {
    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_UNCOMMITTED);
    PCollectionTuple datastoreSnapshot =
        pipeline.apply(
@@ -223,7 +219,8 @@ public class InitSqlPipeline implements Serializable {
            .withName(transformId)
            .withBatchSize(options.getSqlWriteBatchSize())
            .withShards(options.getSqlWriteShards())
-            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity));
+            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity)
+            .disableUpdateAutoTimestamp());
  }

  private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
--- a/core/src/main/java/google/registry/beam/initsql/SqlAccessInfo.java
+++ b/core/src/main/java/google/registry/beam/initsql/SqlAccessInfo.java
@@ -1,45 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.initsql;
-
-import com.google.auto.value.AutoValue;
-import java.util.Optional;
-
-/**
- * Information needed to connect to a database, including JDBC URL, user name, password, and in the
- * case of Cloud SQL, the database instance's name.
- */
-@AutoValue
-abstract class SqlAccessInfo {
-
-  abstract String jdbcUrl();
-
-  abstract String user();
-
-  abstract String password();
-
-  abstract Optional<String> cloudSqlInstanceName();
-
-  public static SqlAccessInfo createCloudSqlAccessInfo(
-      String sqlInstanceName, String username, String password) {
-    return new AutoValue_SqlAccessInfo(
-        "jdbc:postgresql://google/postgres", username, password, Optional.of(sqlInstanceName));
-  }
-
-  public static SqlAccessInfo createLocalSqlAccessInfo(
-      String jdbcUrl, String username, String password) {
-    return new AutoValue_SqlAccessInfo(jdbcUrl, username, password, Optional.empty());
-  }
-}
--- a/core/src/main/java/google/registry/beam/initsql/Transforms.java
+++ b/core/src/main/java/google/registry/beam/initsql/Transforms.java
@@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static java.util.Comparator.comparing;
@@ -32,13 +32,16 @@ import com.google.appengine.api.datastore.EntityTranslator;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
+import google.registry.model.billing.BillingEvent.Flag;
+import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.replay.DatastoreAndSqlEntity;
+import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.schema.replay.DatastoreAndSqlEntity;
-import google.registry.schema.replay.SqlEntity;
 import google.registry.tools.LevelDbLogReader;
 import java.io.Serializable;
 import java.util.Collection;
@@ -256,7 +259,58 @@ public final class Transforms {
                    .iterator()));
  }

+  // Production data repair configs go below. See b/185954992.
+
+  // Prober domains in bad state, without associated contacts, hosts, billings, and history.
+  // They can be safely ignored.
+  private static final ImmutableSet<String> IGNORED_DOMAINS =
+      ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT");
+
+  // Prober hosts referencing phantom registrars. They and their associated history entries can be
+  // safely ignored.
+  private static final ImmutableSet<String> IGNORED_HOSTS =
+      ImmutableSet.of(
+          "4E21_WJ0TEST-GOOGLE",
+          "4E21_WJ1TEST-GOOGLE",
+          "4E21_WJ2TEST-GOOGLE",
+          "4E21_WJ3TEST-GOOGLE");
+
+  // Prober contacts referencing phantom registrars. They and their associated history entries can
+  // be safely ignored.
+  private static final ImmutableSet IGNORED_CONTACTS =
+      ImmutableSet.of(
+          "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE");
+
  private static boolean isMigratable(Entity entity) {
+    // Checks specific to production data. See b/185954992 for details.
+    // The names of these bad entities in production do not conflict with other environments. For
+    // simplicities sake we apply them regardless of the source of the data.
+    if (entity.getKind().equals("DomainBase")
+        && IGNORED_DOMAINS.contains(entity.getKey().getName())) {
+      return false;
+    }
+    if (entity.getKind().equals("ContactResource")) {
+      String roid = entity.getKey().getName();
+      return !IGNORED_CONTACTS.contains(roid);
+    }
+    if (entity.getKind().equals("HostResource")) {
+      String roid = entity.getKey().getName();
+      return !IGNORED_HOSTS.contains(roid);
+    }
+    if (entity.getKind().equals("HistoryEntry")) {
+      // Remove production bad data: History of the contacts to be ignored:
+      com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent();
+      if (parentKey.getKind().equals("ContactResource")) {
+        String contactRoid = parentKey.getName();
+        return !IGNORED_CONTACTS.contains(contactRoid);
+      }
+      if (parentKey.getKind().equals("HostResource")) {
+        String hostRoid = parentKey.getName();
+        return !IGNORED_HOSTS.contains(hostRoid);
+      }
+    }
+    // End of production-specific checks.
+
    if (entity.getKind().equals("HistoryEntry")) {
      // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated.
      // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot
@@ -266,6 +320,18 @@ public final class Transforms {
    return true;
  }

+  private static Entity repairBadData(Entity entity) {
+    if (entity.getKind().equals("Cancellation")
+        && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) {
+      // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the
+      // AUTO_RENEW flag. Note: all affected entities have empty flags so we can simply assign
+      // instead of append. See b/185954992.
+      entity.setUnindexedProperty("reason", Reason.RENEW.name());
+      entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name()));
+    }
+    return entity;
+  }
+
  private static SqlEntity toSqlEntity(Object ofyEntity) {
    if (ofyEntity instanceof HistoryEntry) {
      HistoryEntry ofyHistory = (HistoryEntry) ofyEntity;
@@ -286,7 +352,8 @@ public final class Transforms {
    return dsEntity
        .getEntity()
        .filter(Transforms::isMigratable)
-        .map(e -> ofy().toPojo(e))
+        .map(Transforms::repairBadData)
+        .map(e -> auditedOfy().toPojo(e))
        .map(Transforms::toSqlEntity)
        .orElse(null);
  }
--- a/core/src/main/java/google/registry/beam/invoicing/BillingEvent.java
+++ b/core/src/main/java/google/registry/beam/invoicing/BillingEvent.java
@@ -251,7 +251,14 @@ public abstract class BillingEvent implements Serializable {
  InvoiceGroupingKey getInvoiceGroupingKey() {
    return new AutoValue_BillingEvent_InvoiceGroupingKey(
        billingTime().toLocalDate().withDayOfMonth(1).toString(),
-        billingTime().toLocalDate().withDayOfMonth(1).plusYears(years()).minusDays(1).toString(),
+        years() == 0
+            ? ""
+            : billingTime()
+                .toLocalDate()
+                .withDayOfMonth(1)
+                .plusYears(years())
+                .minusDays(1)
+                .toString(),
        billingId(),
        String.format("%s - %s", registrarId(), tld()),
        String.format("%s | TLD: %s | TERM: %d-year", action(), tld(), years()),
--- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java
+++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java
@@ -14,19 +14,29 @@

 package google.registry.beam.invoicing;

+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;

-import com.google.common.annotations.VisibleForTesting;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryJpaIO.Read;
 import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey;
 import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
+import google.registry.model.billing.BillingEvent.Flag;
+import google.registry.model.registrar.Registrar;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.reporting.billing.BillingModule;
+import google.registry.util.DateTimeUtils;
+import google.registry.util.DomainNameUtils;
 import google.registry.util.SqlTemplate;
 import java.io.Serializable;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.YearMonth;
+import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Optional;
+import java.util.regex.Pattern;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.SerializableCoder;
@@ -59,40 +69,76 @@ public class InvoicingPipeline implements Serializable {
  private static final DateTimeFormatter TIMESTAMP_FORMATTER =
      DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");

-  private final InvoicingPipelineOptions options;
-  private final Pipeline pipeline;
+  private static final Pattern SQL_COMMENT_REGEX =
+      Pattern.compile("^\\s*--.*\\n", Pattern.MULTILINE);

-  @VisibleForTesting
-  InvoicingPipeline(InvoicingPipelineOptions options, Pipeline pipeline) {
-    this.options = options;
-    this.pipeline = pipeline;
-  }
+  private final InvoicingPipelineOptions options;

  InvoicingPipeline(InvoicingPipelineOptions options) {
-    this(options, Pipeline.create(options));
+    this.options = options;
  }

  PipelineResult run() {
-    setupPipeline();
+    Pipeline pipeline = Pipeline.create(options);
+    setupPipeline(pipeline);
    return pipeline.run();
  }

-  void setupPipeline() {
+  void setupPipeline(Pipeline pipeline) {
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
    PCollection<BillingEvent> billingEvents =
-        pipeline.apply(
-            "Read BillingEvents from Bigquery",
-            BigQueryIO.read(BillingEvent::parseFromRecord)
-                .fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
-                .withCoder(SerializableCoder.of(BillingEvent.class))
-                .usingStandardSql()
-                .withoutValidation()
-                .withTemplateCompatibility());
+        options.getDatabase().equals("DATASTORE")
+            ? readFromBigQuery(options, pipeline)
+            : readFromCloudSql(options, pipeline);

    saveInvoiceCsv(billingEvents, options);

    saveDetailedCsv(billingEvents, options);
  }

+  static PCollection<BillingEvent> readFromBigQuery(
+      InvoicingPipelineOptions options, Pipeline pipeline) {
+    return pipeline.apply(
+        "Read BillingEvents from Bigquery",
+        BigQueryIO.read(BillingEvent::parseFromRecord)
+            .fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
+            .withCoder(SerializableCoder.of(BillingEvent.class))
+            .usingStandardSql()
+            .withoutValidation()
+            .withTemplateCompatibility());
+  }
+
+  static PCollection<BillingEvent> readFromCloudSql(
+      InvoicingPipelineOptions options, Pipeline pipeline) {
+    Read<Object[], BillingEvent> read =
+        RegistryJpaIO.read(
+            makeCloudSqlQuery(options.getYearMonth()), false, InvoicingPipeline::parseRow);
+
+    return pipeline.apply("Read BillingEvents from Cloud SQL", read);
+  }
+
+  private static BillingEvent parseRow(Object[] row) {
+    google.registry.model.billing.BillingEvent.OneTime oneTime =
+        (google.registry.model.billing.BillingEvent.OneTime) row[0];
+    Registrar registrar = (Registrar) row[1];
+    return BillingEvent.create(
+        oneTime.getId(),
+        DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
+        DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
+        registrar.getRegistrarId(),
+        registrar.getBillingIdentifier().toString(),
+        registrar.getPoNumber().orElse(""),
+        DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
+        oneTime.getReason().toString(),
+        oneTime.getTargetId(),
+        oneTime.getDomainRepoId(),
+        Optional.ofNullable(oneTime.getPeriodYears()).orElse(0),
+        oneTime.getCost().getCurrencyUnit().toString(),
+        oneTime.getCost().getAmount().doubleValue(),
+        String.join(
+            " ", oneTime.getFlags().stream().map(Flag::toString).collect(toImmutableSet())));
+  }
+
  /** Transform that converts a {@code BillingEvent} into an invoice CSV row. */
  private static class GenerateInvoiceRows
      extends PTransform<PCollection<BillingEvent>, PCollection<String>> {
@@ -178,6 +224,21 @@ public class InvoicingPipeline implements Serializable {
        .build();
  }

+  /** Create the Cloud SQL query for a given yearMonth at runtime. */
+  static String makeCloudSqlQuery(String yearMonth) {
+    YearMonth endMonth = YearMonth.parse(yearMonth).plusMonths(1);
+    String queryWithComments =
+        SqlTemplate.create(
+                getQueryFromFile(InvoicingPipeline.class, "cloud_sql_billing_events.sql"))
+            .put("FIRST_TIMESTAMP_OF_MONTH", yearMonth.concat("-01"))
+            .put(
+                "LAST_TIMESTAMP_OF_MONTH",
+                String.format("%d-%d-01", endMonth.getYear(), endMonth.getMonthValue()))
+            .build();
+    // Remove the comments from the query string
+    return SQL_COMMENT_REGEX.matcher(queryWithComments).replaceAll("");
+  }
+
  public static void main(String[] args) {
    PipelineOptionsFactory.register(InvoicingPipelineOptions.class);
    InvoicingPipelineOptions options =
--- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java
+++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java
@@ -30,6 +30,11 @@ public interface InvoicingPipelineOptions extends RegistryPipelineOptions {

  void setInvoiceFilePrefix(String value);

+  @Description("The database to read data from.")
+  String getDatabase();
+
+  void setDatabase(String value);
+
  @Description("The GCS bucket URL for invoices and detailed  reports to be uploaded.")
  String getBillingBucketUrl();

--- a/core/src/main/java/google/registry/beam/rde/RdeIO.java
+++ b/core/src/main/java/google/registry/beam/rde/RdeIO.java
@@ -0,0 +1,273 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.rde;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Verify.verify;
+import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.common.flogger.FluentLogger;
+import google.registry.gcs.GcsUtils;
+import google.registry.keyring.api.PgpHelper;
+import google.registry.model.common.Cursor;
+import google.registry.model.rde.RdeMode;
+import google.registry.model.rde.RdeNamingUtils;
+import google.registry.model.rde.RdeRevision;
+import google.registry.model.tld.Registry;
+import google.registry.rde.DepositFragment;
+import google.registry.rde.Ghostryde;
+import google.registry.rde.PendingDeposit;
+import google.registry.rde.RdeCounter;
+import google.registry.rde.RdeMarshaller;
+import google.registry.rde.RdeResourceType;
+import google.registry.rde.RdeUtil;
+import google.registry.tldconfig.idn.IdnTableEnum;
+import google.registry.xjc.rdeheader.XjcRdeHeader;
+import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
+import google.registry.xml.ValidationMode;
+import google.registry.xml.XmlException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.security.Security;
+import java.util.Optional;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.openpgp.PGPPublicKey;
+import org.joda.time.DateTime;
+
+public class RdeIO {
+
+  @AutoValue
+  abstract static class Write
+      extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
+
+    abstract GcsUtils gcsUtils();
+
+    abstract String rdeBucket();
+
+    // It's OK to return a primitive array because we are only using it to construct the
+    // PGPPublicKey, which is not serializable.
+    @SuppressWarnings("mutable")
+    abstract byte[] stagingKeyBytes();
+
+    abstract ValidationMode validationMode();
+
+    static Builder builder() {
+      return new AutoValue_RdeIO_Write.Builder();
+    }
+
+    @AutoValue.Builder
+    abstract static class Builder {
+      abstract Builder setGcsUtils(GcsUtils gcsUtils);
+
+      abstract Builder setRdeBucket(String value);
+
+      abstract Builder setStagingKeyBytes(byte[] value);
+
+      abstract Builder setValidationMode(ValidationMode value);
+
+      abstract Write build();
+    }
+
+    @Override
+    public PDone expand(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
+      input
+          .apply(
+              "Write to GCS",
+              ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
+          .apply("Update cursors", ParDo.of(new CursorUpdater()));
+      return PDone.in(input.getPipeline());
+    }
+  }
+
+  private static class RdeWriter
+      extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
+
+    private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+    private final GcsUtils gcsUtils;
+    private final String rdeBucket;
+    private final byte[] stagingKeyBytes;
+    private final RdeMarshaller marshaller;
+
+    protected RdeWriter(
+        GcsUtils gcsUtils,
+        String rdeBucket,
+        byte[] stagingKeyBytes,
+        ValidationMode validationMode) {
+      this.gcsUtils = gcsUtils;
+      this.rdeBucket = rdeBucket;
+      this.stagingKeyBytes = stagingKeyBytes;
+      this.marshaller = new RdeMarshaller(validationMode);
+    }
+
+    @Setup
+    public void setup() {
+      Security.addProvider(new BouncyCastleProvider());
+    }
+
+    @ProcessElement
+    public void processElement(
+        @Element KV<PendingDeposit, Iterable<DepositFragment>> kv,
+        PipelineOptions options,
+        OutputReceiver<KV<PendingDeposit, Integer>> outputReceiver) {
+      PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
+      PendingDeposit key = kv.getKey();
+      Iterable<DepositFragment> fragments = kv.getValue();
+      RdeCounter counter = new RdeCounter();
+
+      // Determine some basic things about the deposit.
+      final RdeMode mode = key.mode();
+      final String tld = key.tld();
+      final DateTime watermark = key.watermark();
+      final int revision =
+          Optional.ofNullable(key.revision())
+              .orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
+      String id = RdeUtil.timestampToId(watermark);
+      String prefix = options.getJobName();
+      String basename = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
+      if (key.manual()) {
+        checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
+        prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
+      } else {
+        prefix = prefix + "/" + basename;
+      }
+      BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
+      // This file will contain the byte length (ASCII) of the raw unencrypted XML.
+      //
+      // This is necessary because RdeUploadAction creates a tar file which requires that the length
+      // be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
+      // length, so we just save it separately.
+      BlobId xmlLengthFilename = BlobId.of(rdeBucket, prefix + ".xml.length");
+      BlobId reportFilename = BlobId.of(rdeBucket, prefix + "-report.xml.ghostryde");
+
+      // These variables will be populated as we write the deposit XML and used for other files.
+      boolean failed = false;
+      XjcRdeHeader header;
+
+      // Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
+
+      logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
+      try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
+          OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
+          OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
+          Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
+
+        // Output the top portion of the XML document.
+        output.write(marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
+
+        // Output XML fragments while counting them.
+        for (DepositFragment fragment : fragments) {
+          if (!fragment.xml().isEmpty()) {
+            output.write(fragment.xml());
+            counter.increment(fragment.type());
+          }
+          if (!fragment.error().isEmpty()) {
+            failed = true;
+            logger.atSevere().log("Fragment error: %s", fragment.error());
+          }
+        }
+
+        // Don't write the IDN elements for BRDA.
+        if (mode == RdeMode.FULL) {
+          for (IdnTableEnum idn : IdnTableEnum.values()) {
+            output.write(marshaller.marshalIdn(idn.getTable()));
+            counter.increment(RdeResourceType.IDN);
+          }
+        }
+
+        // Output XML that says how many resources were emitted.
+        header = counter.makeHeader(tld, mode);
+        output.write(marshaller.marshalOrDie(new XjcRdeHeaderElement(header)));
+
+        // Output the bottom of the XML document.
+        output.write(marshaller.makeFooter());
+
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+
+      // If an entity was broken, abort after writing as much logs/deposit data as possible.
+      verify(!failed, "RDE staging failed for TLD %s", tld);
+
+      // Write a tiny XML file to GCS containing some information about the deposit.
+      //
+      // This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
+      if (mode == RdeMode.FULL) {
+        logger.atInfo().log("Writing %s", reportFilename);
+        try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
+            OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
+          counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
+        } catch (IOException | XmlException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      // Now that we're done, output roll the cursor forward.
+      if (key.manual()) {
+        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
+      } else {
+        outputReceiver.output(KV.of(key, revision));
+      }
+    }
+  }
+
+  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+    private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+    @ProcessElement
+    public void processElement(@Element KV<PendingDeposit, Integer> input) {
+      tm().transact(
+              () -> {
+                PendingDeposit key = input.getKey();
+                int revision = input.getValue();
+                Registry registry = Registry.get(key.tld());
+                Optional<Cursor> cursor =
+                    transactIfJpaTm(
+                        () ->
+                            tm().loadByKeyIfPresent(
+                                    Cursor.createVKey(key.cursor(), registry.getTldStr())));
+                DateTime position = getCursorTimeOrStartOfTime(cursor);
+                checkState(key.interval() != null, "Interval must be present");
+                DateTime newPosition = key.watermark().plus(key.interval());
+                if (!position.isBefore(newPosition)) {
+                  logger.atWarning().log("Cursor has already been rolled forward.");
+                  return;
+                }
+                verify(
+                    position.equals(key.watermark()),
+                    "Partial ordering of RDE deposits broken: %s %s",
+                    position,
+                    key);
+                tm().put(Cursor.create(key.cursor(), newPosition, registry));
+                logger.atInfo().log(
+                    "Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
+                RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+              });
+    }
+  }
+}
--- a/core/src/main/java/google/registry/beam/rde/RdePipeline.java
+++ b/core/src/main/java/google/registry/beam/rde/RdePipeline.java
@@ -0,0 +1,303 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.rde;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.io.BaseEncoding;
+import dagger.BindsInstance;
+import dagger.Component;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.EppResource;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.host.HostResource;
+import google.registry.model.rde.RdeMode;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.Registrar.Type;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
+import google.registry.rde.DepositFragment;
+import google.registry.rde.PendingDeposit;
+import google.registry.rde.PendingDeposit.PendingDepositCoder;
+import google.registry.rde.RdeFragmenter;
+import google.registry.rde.RdeMarshaller;
+import google.registry.xml.ValidationMode;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.persistence.Entity;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.FlatMapElements;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.joda.time.DateTime;
+
+/**
+ * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
+ *
+ * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ *
+ * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
+ *
+ * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
+ *     Flex Templates</a>
+ */
+@Singleton
+public class RdePipeline implements Serializable {
+
+  private final transient RdePipelineOptions options;
+  private final ValidationMode mode;
+  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final String rdeBucket;
+  private final byte[] stagingKeyBytes;
+  private final GcsUtils gcsUtils;
+
+  // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
+  // if sneaks into production we would get an extra signal.
+  private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
+      Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
+
+  private static final String EPP_RESOURCE_QUERY =
+      "SELECT id FROM %entity% "
+          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
+          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
+          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
+
+  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
+    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
+        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
+  }
+
+  @Inject
+  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
+    this.options = options;
+    this.mode = ValidationMode.valueOf(options.getValidationMode());
+    this.pendings = decodePendings(options.getPendings());
+    this.rdeBucket = options.getGcsBucket();
+    this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
+    this.gcsUtils = gcsUtils;
+  }
+
+  PipelineResult run() {
+    Pipeline pipeline = Pipeline.create(options);
+    PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> fragments =
+        createFragments(pipeline);
+    persistData(fragments);
+    return pipeline.run();
+  }
+
+  PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
+    return PCollectionList.of(processRegistrars(pipeline))
+        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
+        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
+        .and(processNonRegistrarEntities(pipeline, HostResource.class))
+        .apply(Flatten.pCollections())
+        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
+        .apply("Group by PendingDeposit", GroupByKey.create());
+  }
+
+  void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
+    input.apply(
+        "Write to GCS and update cursors",
+        RdeIO.Write.builder()
+            .setRdeBucket(rdeBucket)
+            .setGcsUtils(gcsUtils)
+            .setValidationMode(mode)
+            .setStagingKeyBytes(stagingKeyBytes)
+            .build());
+  }
+
+  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    return pipeline
+        .apply(
+            "Read all production Registrar entities",
+            RegistryJpaIO.read(
+                "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
+                ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
+                String.class,
+                // TODO: consider adding coders for entities and pass them directly instead of using
+                // VKeys.
+                id -> VKey.createSql(Registrar.class, id)))
+        .apply(
+            "Marshal Registrar into DepositFragment",
+            FlatMapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (VKey<Registrar> key) -> {
+                      Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
+                      DepositFragment fragment =
+                          new RdeMarshaller(mode).marshalRegistrar(registrar);
+                      return pendings.values().stream()
+                          .map(pending -> KV.of(pending, fragment))
+                          .collect(toImmutableSet());
+                    }));
+  }
+
+  @SuppressWarnings("deprecation") // Reshuffle is still recommended by Dataflow.
+  <T extends EppResource>
+      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
+          Pipeline pipeline, Class<T> clazz) {
+    return createInputs(pipeline, clazz)
+        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
+        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
+        .apply(
+            "Reshuffle KV<PendingDeposit, DepositFragment> of "
+                + clazz.getSimpleName()
+                + " to prevent fusion",
+            Reshuffle.of());
+  }
+
+  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
+    return pipeline.apply(
+        "Read all production " + clazz.getSimpleName() + " entities",
+        RegistryJpaIO.read(
+            createEppResourceQuery(clazz),
+            clazz.equals(DomainBase.class)
+                ? ImmutableMap.of("tlds", pendings.keySet())
+                : ImmutableMap.of(),
+            String.class,
+            // TODO: consider adding coders for entities and pass them directly instead of using
+            // VKeys.
+            x -> VKey.create(clazz, x)));
+  }
+
+  <T extends EppResource>
+      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
+    return FlatMapElements.into(
+            TypeDescriptors.kvs(
+                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
+        .via(
+            (VKey<T> key) -> {
+              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
+              // The set of all TLDs to which this resource should be emitted.
+              ImmutableSet<String> tlds =
+                  clazz.equals(DomainBase.class)
+                      ? ImmutableSet.of(((DomainBase) resource).getTld())
+                      : pendings.keySet();
+              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
+              ImmutableSet<DateTime> dates =
+                  tlds.stream()
+                      .map(pendings::get)
+                      .flatMap(ImmutableSet::stream)
+                      .map(PendingDeposit::watermark)
+                      .collect(toImmutableSet());
+              // Launch asynchronous fetches of point-in-time representations of resource.
+              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
+                  ImmutableMap.copyOf(
+                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
+              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
+              // the result.
+              RdeFragmenter fragmenter =
+                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
+              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
+              for (String tld : tlds) {
+                for (PendingDeposit pending : pendings.get(tld)) {
+                  // Hosts and contacts don't get included in BRDA deposits.
+                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
+                    continue;
+                  }
+                  Optional<DepositFragment> fragment =
+                      fragmenter.marshal(pending.watermark(), pending.mode());
+                  fragment.ifPresent(
+                      depositFragment -> results.add(KV.of(pending, depositFragment)));
+                }
+              }
+              return results;
+            });
+  }
+
+  /**
+   * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
+   * the original TLD to pending deposit map.
+   */
+  @SuppressWarnings("unchecked")
+  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+    try (ObjectInputStream ois =
+        new ObjectInputStream(
+            new ByteArrayInputStream(
+                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
+      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+    } catch (IOException | ClassNotFoundException e) {
+      throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
+    }
+  }
+
+  /**
+   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
+   * worker by the pipeline launcher as a pipeline option.
+   */
+  static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+      throws IOException {
+    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+      ObjectOutputStream oos = new ObjectOutputStream(baos);
+      oos.writeObject(pendings);
+      oos.flush();
+      return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
+    }
+  }
+
+  public static void main(String[] args) throws IOException, ClassNotFoundException {
+    PipelineOptionsFactory.register(RdePipelineOptions.class);
+    RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
+    DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
+  }
+
+  @Singleton
+  @Component(modules = {CredentialModule.class, ConfigModule.class})
+  interface RdePipelineComponent {
+    RdePipeline rdePipeline();
+
+    @Component.Builder
+    interface Builder {
+      @BindsInstance
+      Builder options(RdePipelineOptions options);
+
+      RdePipelineComponent build();
+    }
+  }
+}
--- a/core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java
+++ b/core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java
@@ -0,0 +1,42 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.rde;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import org.apache.beam.sdk.options.Description;
+
+/** Custom options for running the spec11 pipeline. */
+public interface RdePipelineOptions extends RegistryPipelineOptions {
+
+  @Description("The Base64-encoded serialized map of TLDs to PendingDeposit.")
+  String getPendings();
+
+  void setPendings(String value);
+
+  @Description("The validation mode (LENIENT|STRICT) that the RDE marshaller uses.")
+  String getValidationMode();
+
+  void setValidationMode(String value);
+
+  @Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
+  String getGcsBucket();
+
+  void setGcsBucket(String value);
+
+  @Description("The Base64-encoded PGP public key to encrypt the deposits.")
+  String getStagingKey();
+
+  void setStagingKey(String value);
+}
--- a/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java
+++ b/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java
@@ -25,7 +25,7 @@ import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;

 /**
- * A POJO representing a single subdomain, parsed from a {@code SchemaAndRecord}.
+ * A POJO representing a domain name and associated info, parsed from a {@code SchemaAndRecord}.
 *
 * <p>This is a trivially serializable class that allows Beam to transform the results of a Bigquery
 * query into a standard Java representation, giving us the type guarantees and ease of manipulation
@@ -33,28 +33,31 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 * function.
 */
@AutoValue
-public abstract class Subdomain implements Serializable {
+public abstract class DomainNameInfo implements Serializable {

  private static final ImmutableList<String> FIELD_NAMES =
      ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");

  /** Returns the fully qualified domain name. */
  abstract String domainName();
+
  /** Returns the domain repo ID (the primary key of the domain table). */
  abstract String domainRepoId();
+
  /** Returns the registrar ID of the associated registrar for this domain. */
  abstract String registrarId();
+
  /** Returns the email address of the registrar associated with this domain. */
  abstract String registrarEmailAddress();

  /**
-   * Constructs a {@link Subdomain} from an Apache Avro {@code SchemaAndRecord}.
+   * Constructs a {@link DomainNameInfo} from an Apache Avro {@code SchemaAndRecord}.
   *
   * @see <a
   *     href=http://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/generic/GenericData.Record.html>
   *     Apache AVRO GenericRecord</a>
   */
-  static Subdomain parseFromRecord(SchemaAndRecord schemaAndRecord) {
+  static DomainNameInfo parseFromRecord(SchemaAndRecord schemaAndRecord) {
    checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
    GenericRecord record = schemaAndRecord.getRecord();
    return create(
@@ -65,18 +68,15 @@ public abstract class Subdomain implements Serializable {
  }

  /**
-   * Creates a concrete {@link Subdomain}.
+   * Creates a concrete {@link DomainNameInfo}.
   *
-   * <p>This should only be used outside this class for testing- instances of {@link Subdomain}
+   * <p>This should only be used outside this class for testing- instances of {@link DomainNameInfo}
   * should otherwise come from {@link #parseFromRecord}.
   */
  @VisibleForTesting
-  static Subdomain create(
-      String domainName,
-      String domainRepoId,
-      String registrarId,
-      String registrarEmailAddress) {
-    return new AutoValue_Subdomain(
+  static DomainNameInfo create(
+      String domainName, String domainRepoId, String registrarId, String registrarEmailAddress) {
+    return new AutoValue_DomainNameInfo(
        domainName, domainRepoId, registrarId, registrarEmailAddress);
  }
 }
--- a/core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java
+++ b/core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java
@@ -55,13 +55,14 @@ public class SafeBrowsingTransforms {
      "https://safebrowsing.googleapis.com/v4/threatMatches:find";

  /**
-   * {@link DoFn} mapping a {@link Subdomain} to its evaluation report from SafeBrowsing.
+   * {@link DoFn} mapping a {@link DomainNameInfo} to its evaluation report from SafeBrowsing.
   *
   * <p>Refer to the Lookup API documentation for the request/response format and other details.
   *
   * @see <a href=https://developers.google.com/safe-browsing/v4/lookup-api>Lookup API</a>
   */
-  static class EvaluateSafeBrowsingFn extends DoFn<Subdomain, KV<Subdomain, ThreatMatch>> {
+  static class EvaluateSafeBrowsingFn
+      extends DoFn<DomainNameInfo, KV<DomainNameInfo, ThreatMatch>> {

    /**
     * Max number of urls we can check in a single query.
@@ -74,10 +75,11 @@ public class SafeBrowsingTransforms {
    private final String apiKey;

    /**
-     * Maps a subdomain's {@code fullyQualifiedDomainName} to its corresponding {@link Subdomain} to
-     * facilitate batching SafeBrowsing API requests.
+     * Maps a domain name's {@code fullyQualifiedDomainName} to its corresponding {@link
+     * DomainNameInfo} to facilitate batching SafeBrowsing API requests.
     */
-    private final Map<String, Subdomain> subdomainBuffer = new LinkedHashMap<>(BATCH_SIZE);
+    private final Map<String, DomainNameInfo> domainNameInfoBuffer =
+        new LinkedHashMap<>(BATCH_SIZE);

    /**
     * Provides the HTTP client we use to interact with the SafeBrowsing API.
@@ -119,37 +121,38 @@ public class SafeBrowsingTransforms {
      closeableHttpClientSupplier = clientSupplier;
    }

-    /** Evaluates any buffered {@link Subdomain} objects upon completing the bundle. */
+    /** Evaluates any buffered {@link DomainNameInfo} objects upon completing the bundle. */
    @FinishBundle
    public void finishBundle(FinishBundleContext context) {
-      if (!subdomainBuffer.isEmpty()) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      if (!domainNameInfoBuffer.isEmpty()) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
        results.forEach((kv) -> context.output(kv, Instant.now(), GlobalWindow.INSTANCE));
      }
    }

    /**
-     * Buffers {@link Subdomain} objects until we reach the batch size, then bulk-evaluate the URLs
-     * with the SafeBrowsing API.
+     * Buffers {@link DomainNameInfo} objects until we reach the batch size, then bulk-evaluate the
+     * URLs with the SafeBrowsing API.
     */
    @ProcessElement
    public void processElement(ProcessContext context) {
-      Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.domainName(), subdomain);
-      if (subdomainBuffer.size() >= BATCH_SIZE) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      DomainNameInfo domainNameInfo = context.element();
+      domainNameInfoBuffer.put(domainNameInfo.domainName(), domainNameInfo);
+      if (domainNameInfoBuffer.size() >= BATCH_SIZE) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
        results.forEach(context::output);
      }
    }

    /**
-     * Evaluates all {@link Subdomain} objects in the buffer and returns a list of key-value pairs
-     * from {@link Subdomain} to its SafeBrowsing report.
+     * Evaluates all {@link DomainNameInfo} objects in the buffer and returns a list of key-value
+     * pairs from {@link DomainNameInfo} to its SafeBrowsing report.
     *
-     * <p>If a {@link Subdomain} is safe according to the API, it will not emit a report.
+     * <p>If a {@link DomainNameInfo} is safe according to the API, it will not emit a report.
     */
-    private ImmutableSet<KV<Subdomain, ThreatMatch>> evaluateAndFlush() {
-      ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder = new ImmutableSet.Builder<>();
+    private ImmutableSet<KV<DomainNameInfo, ThreatMatch>> evaluateAndFlush() {
+      ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder =
+          new ImmutableSet.Builder<>();
      try {
        URIBuilder uriBuilder = new URIBuilder(SAFE_BROWSING_URL);
        // Add the API key param
@@ -174,7 +177,7 @@ public class SafeBrowsingTransforms {
        throw new RuntimeException("Caught parsing exception, failing pipeline.", e);
      } finally {
        // Flush the buffer
-        subdomainBuffer.clear();
+        domainNameInfoBuffer.clear();
      }
      return resultBuilder.build();
    }
@@ -183,7 +186,7 @@ public class SafeBrowsingTransforms {
    private JSONObject createRequestBody() throws JSONException {
      // Accumulate all domain names to evaluate.
      JSONArray threatArray = new JSONArray();
-      for (String fullyQualifiedDomainName : subdomainBuffer.keySet()) {
+      for (String fullyQualifiedDomainName : domainNameInfoBuffer.keySet()) {
        threatArray.put(new JSONObject().put("url", fullyQualifiedDomainName));
      }
      // Construct the JSON request body
@@ -211,7 +214,7 @@ public class SafeBrowsingTransforms {
     */
    private void processResponse(
        CloseableHttpResponse response,
-        ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder)
+        ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder)
        throws JSONException, IOException {
      int statusCode = response.getStatusLine().getStatusCode();
      if (statusCode != SC_OK) {
@@ -226,16 +229,17 @@ public class SafeBrowsingTransforms {
        if (responseBody.length() == 0) {
          logger.atInfo().log("Response was empty, no threats detected");
        } else {
-          // Emit all Subdomains with their API results.
+          // Emit all DomainNameInfos with their API results.
          JSONArray threatMatches = responseBody.getJSONArray("matches");
          for (int i = 0; i < threatMatches.length(); i++) {
            JSONObject match = threatMatches.getJSONObject(i);
            String url = match.getJSONObject("threat").getString("url");
-            Subdomain subdomain = subdomainBuffer.get(url);
+            DomainNameInfo domainNameInfo = domainNameInfoBuffer.get(url);
            resultBuilder.add(
                KV.of(
-                    subdomain,
-                    ThreatMatch.create(match.getString("threatType"), subdomain.domainName())));
+                    domainNameInfo,
+                    ThreatMatch.create(
+                        match.getString("threatType"), domainNameInfo.domainName())));
          }
        }
      }
--- a/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java
+++ b/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java
@@ -18,16 +18,18 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.beam.BeamUtils.getQueryFromFile;

 import com.google.auto.value.AutoValue;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import dagger.Component;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryJpaIO.Read;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
 import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.reporting.Spec11ThreatMatch;
 import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
 import google.registry.util.UtilsModule;
@@ -84,75 +86,101 @@ public class Spec11Pipeline implements Serializable {

  private final Spec11PipelineOptions options;
  private final EvaluateSafeBrowsingFn safeBrowsingFn;
-  private final Pipeline pipeline;
-
-  @VisibleForTesting
-  Spec11Pipeline(
-      Spec11PipelineOptions options, EvaluateSafeBrowsingFn safeBrowsingFn, Pipeline pipeline) {
-    this.options = options;
-    this.safeBrowsingFn = safeBrowsingFn;
-    this.pipeline = pipeline;
-  }

  Spec11Pipeline(Spec11PipelineOptions options, EvaluateSafeBrowsingFn safeBrowsingFn) {
-    this(options, safeBrowsingFn, Pipeline.create(options));
+    this.options = options;
+    this.safeBrowsingFn = safeBrowsingFn;
  }

  PipelineResult run() {
-    setupPipeline();
+    Pipeline pipeline = Pipeline.create(options);
+    setupPipeline(pipeline);
    return pipeline.run();
  }

-  void setupPipeline() {
-    PCollection<Subdomain> domains =
-        pipeline.apply(
-            "Read active domains from BigQuery",
-            BigQueryIO.read(Subdomain::parseFromRecord)
-                .fromQuery(
-                    SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
-                        .put("PROJECT_ID", options.getProject())
-                        .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
-                        .put("REGISTRAR_TABLE", "Registrar")
-                        .put("DOMAIN_BASE_TABLE", "DomainBase")
-                        .build())
-                .withCoder(SerializableCoder.of(Subdomain.class))
-                .usingStandardSql()
-                .withoutValidation()
-                .withTemplateCompatibility());
+  void setupPipeline(Pipeline pipeline) {
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
+    PCollection<DomainNameInfo> domains =
+        options.getDatabase().equals("DATASTORE")
+            ? readFromBigQuery(options, pipeline)
+            : readFromCloudSql(pipeline);

-    PCollection<KV<Subdomain, ThreatMatch>> threatMatches =
+    PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches =
        domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn));

    saveToSql(threatMatches, options);
    saveToGcs(threatMatches, options);
  }

+  static PCollection<DomainNameInfo> readFromCloudSql(Pipeline pipeline) {
+    Read<Object[], DomainNameInfo> read =
+        RegistryJpaIO.read(
+            "select d, r.emailAddress from Domain d join Registrar r on"
+                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
+                + " and d.deletionTime > now()",
+            false,
+            Spec11Pipeline::parseRow);
+
+    return pipeline.apply("Read active domains from Cloud SQL", read);
+  }
+
+  static PCollection<DomainNameInfo> readFromBigQuery(
+      Spec11PipelineOptions options, Pipeline pipeline) {
+    return pipeline.apply(
+        "Read active domains from BigQuery",
+        BigQueryIO.read(DomainNameInfo::parseFromRecord)
+            .fromQuery(
+                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "domain_name_infos.sql"))
+                    .put("PROJECT_ID", options.getProject())
+                    .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
+                    .put("REGISTRAR_TABLE", "Registrar")
+                    .put("DOMAIN_BASE_TABLE", "DomainBase")
+                    .build())
+            .withCoder(SerializableCoder.of(DomainNameInfo.class))
+            .usingStandardSql()
+            .withoutValidation()
+            .withTemplateCompatibility());
+  }
+
+  private static DomainNameInfo parseRow(Object[] row) {
+    DomainBase domainBase = (DomainBase) row[0];
+    String emailAddress = (String) row[1];
+    if (emailAddress == null) {
+      emailAddress = "";
+    }
+    return DomainNameInfo.create(
+        domainBase.getDomainName(),
+        domainBase.getRepoId(),
+        domainBase.getCurrentSponsorRegistrarId(),
+        emailAddress);
+  }
+
  static void saveToSql(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
    String transformId = "Spec11 Threat Matches";
    LocalDate date = LocalDate.parse(options.getDate(), ISODateTimeFormat.date());
    threatMatches.apply(
        "Write to Sql: " + transformId,
-        RegistryJpaIO.<KV<Subdomain, ThreatMatch>>write()
+        RegistryJpaIO.<KV<DomainNameInfo, ThreatMatch>>write()
            .withName(transformId)
            .withBatchSize(options.getSqlWriteBatchSize())
            .withShards(options.getSqlWriteShards())
            .withJpaConverter(
                (kv) -> {
-                  Subdomain subdomain = kv.getKey();
+                  DomainNameInfo domainNameInfo = kv.getKey();
                  return new Spec11ThreatMatch.Builder()
                      .setThreatTypes(
                          ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
                      .setCheckDate(date)
-                      .setDomainName(subdomain.domainName())
-                      .setDomainRepoId(subdomain.domainRepoId())
-                      .setRegistrarId(subdomain.registrarId())
+                      .setDomainName(domainNameInfo.domainName())
+                      .setDomainRepoId(domainNameInfo.domainRepoId())
+                      .setRegistrarId(domainNameInfo.registrarId())
                      .build();
                }));
  }

  static void saveToGcs(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
    threatMatches
        .apply(
            "Map registrar ID to email/ThreatMatch pair",
@@ -160,7 +188,7 @@ public class Spec11Pipeline implements Serializable {
                    TypeDescriptors.kvs(
                        TypeDescriptors.strings(), TypeDescriptor.of(EmailAndThreatMatch.class)))
                .via(
-                    (KV<Subdomain, ThreatMatch> kv) ->
+                    (KV<DomainNameInfo, ThreatMatch> kv) ->
                        KV.of(
                            kv.getKey().registrarId(),
                            EmailAndThreatMatch.create(
@@ -171,15 +199,15 @@ public class Spec11Pipeline implements Serializable {
            MapElements.into(TypeDescriptors.strings())
                .via(
                    (KV<String, Iterable<EmailAndThreatMatch>> kv) -> {
-                      String clientId = kv.getKey();
+                      String registrarId = kv.getKey();
                      checkArgument(
                          kv.getValue().iterator().hasNext(),
                          String.format(
-                              "Registrar with ID %s had no corresponding threats", clientId));
+                              "Registrar with ID %s had no corresponding threats", registrarId));
                      String email = kv.getValue().iterator().next().email();
                      JSONObject output = new JSONObject();
                      try {
-                        output.put(REGISTRAR_CLIENT_ID_FIELD, clientId);
+                        output.put(REGISTRAR_CLIENT_ID_FIELD, registrarId);
                        output.put(REGISTRAR_EMAIL_FIELD, email);
                        JSONArray threatMatchArray = new JSONArray();
                        for (EmailAndThreatMatch emailAndThreatMatch : kv.getValue()) {
@@ -203,7 +231,7 @@ public class Spec11Pipeline implements Serializable {
                        options.getReportingBucketUrl(),
                        getSpec11ReportFilePath(LocalDate.parse(options.getDate()))))
                .withoutSharding()
-                .withHeader("Map from registrar email / name to detected subdomain threats:"));
+                .withHeader("Map from registrar email / name to detected domain name threats:"));
  }

  public static void main(String[] args) {
--- a/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java
+++ b/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java
@@ -34,4 +34,9 @@ public interface Spec11PipelineOptions extends RegistryPipelineOptions {
  String getReportingBucketUrl();

  void setReportingBucketUrl(String value);
+
+  @Description("The database to read data from.")
+  String getDatabase();
+
+  void setDatabase(String value);
 }
--- a/core/src/main/java/google/registry/config/CloudTasksUtilsModule.java
+++ b/core/src/main/java/google/registry/config/CloudTasksUtilsModule.java
@@ -0,0 +1,65 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.config;
+
+import com.google.api.gax.core.FixedCredentialsProvider;
+import com.google.cloud.tasks.v2.CloudTasksClient;
+import com.google.cloud.tasks.v2.CloudTasksSettings;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.CloudTasksUtils;
+import google.registry.util.GoogleCredentialsBundle;
+import google.registry.util.Retrier;
+import java.io.IOException;
+import javax.inject.Provider;
+import javax.inject.Singleton;
+
+/**
+ * A {@link Module} that provides {@link CloudTasksUtils}.
+ *
+ * <p>The class itself cannot be annotated as {@code Inject} because its requested dependencies use
+ * the {@link Config} qualifier which is not available in the {@code util} package.
+ */
+@Module
+public abstract class CloudTasksUtilsModule {
+
+  @Singleton
+  @Provides
+  public static CloudTasksUtils provideCloudTasksUtils(
+      @Config("projectId") String projectId,
+      @Config("locationId") String locationId,
+      // Use a provider so that we can use try-with-resources with the client, which implements
+      // Autocloseable.
+      Provider<CloudTasksClient> clientProvider,
+      Retrier retrier) {
+    return new CloudTasksUtils(retrier, projectId, locationId, clientProvider);
+  }
+
+  @Provides
+  public static CloudTasksClient provideCloudTasksClient(
+      @DefaultCredential GoogleCredentialsBundle credentials) {
+    try {
+      return CloudTasksClient.create(
+          CloudTasksSettings.newBuilder()
+              .setCredentialsProvider(
+                  FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
+              .build());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}
--- a/core/src/main/java/google/registry/config/RegistryConfig.java
+++ b/core/src/main/java/google/registry/config/RegistryConfig.java
@@ -60,7 +60,7 @@ import org.joda.time.Duration;
 * <p>This class does not represent the total configuration of the Nomulus service. It's <b>only
 * meant for settings that need to be configured <i>once</i></b>. Settings which may be subject to
 * change in the future, should instead be retrieved from Datastore. The {@link
- * google.registry.model.registry.Registry Registry} class is one such example of this.
+ * google.registry.model.tld.Registry Registry} class is one such example of this.
 *
 * <p>Note: Only settings that are actually configurable belong in this file. It's not a catch-all
 * for anything widely used throughout the code base.
@@ -105,6 +105,12 @@ public final class RegistryConfig {
      return config.appEngine.projectId;
    }

+    @Provides
+    @Config("locationId")
+    public static String provideLocationId(RegistryConfigSettings config) {
+      return config.appEngine.locationId;
+    }
+
    /**
     * The filename of the logo to be displayed in the header of the registrar console.
     *
@@ -203,7 +209,7 @@ public final class RegistryConfig {
     * Configuration for analytics services installed in the web console.
     *
     * @see google.registry.ui.server.registrar.ConsoleUiAction
-     * @see google.registry.ui.soy.AnalyticsSoyInfo
+     * @see google.registry.ui.soy.registrar.AnalyticsSoyInfo
     */
    @Provides
    @Config("analyticsConfig")
@@ -416,17 +422,6 @@ public final class RegistryConfig {
      return Optional.ofNullable(config.cloudDns.servicePath);
    }

-    /**
-     * Returns size of Google Cloud Storage client connection buffer in bytes.
-     *
-     * @see google.registry.gcs.GcsUtils
-     */
-    @Provides
-    @Config("gcsBufferSize")
-    public static int provideGcsBufferSize() {
-      return 1024 * 1024;
-    }
-
    /**
     * Returns the email address of the admin account on the G Suite app used to perform
     * administrative actions.
@@ -1143,8 +1138,8 @@ public final class RegistryConfig {
    }

    /**
-     * Returns the clientId of the registrar that admins are automatically logged in as if they
-     * aren't otherwise associated with one.
+     * Returns the ID of the registrar that admins are automatically logged in as if they aren't
+     * otherwise associated with one.
     */
    @Provides
    @Config("registryAdminClientId")
@@ -1278,16 +1273,34 @@ public final class RegistryConfig {

    @Provides
    @Config("expirationWarningDays")
-    public static int provideDaysToExpiration(RegistryConfigSettings config) {
+    public static int provideExpirationWarningDays(RegistryConfigSettings config) {
      return config.sslCertificateValidation.expirationWarningDays;
    }

+    @Provides
+    @Config("expirationWarningIntervalDays")
+    public static int provideExpirationWarningIntervalDays(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningIntervalDays;
+    }
+
    @Provides
    @Config("minimumRsaKeyLength")
    public static int provideMinimumRsaKeyLength(RegistryConfigSettings config) {
      return config.sslCertificateValidation.minimumRsaKeyLength;
    }

+    @Provides
+    @Config("expirationWarningEmailBodyText")
+    public static String provideExpirationWarningEmailBodyText(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningEmailBodyText;
+    }
+
+    @Provides
+    @Config("expirationWarningEmailSubjectText")
+    public static String provideExpirationWarningEmailSubjectText(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningEmailSubjectText;
+    }
+
    @Provides
    @Config("allowedEcdsaCurves")
    public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
@@ -1387,8 +1400,8 @@ public final class RegistryConfig {
  /**
   * Returns the amount of time a domain label list should be cached in memory before expiring.
   *
-   * @see google.registry.model.registry.label.ReservedList
-   * @see google.registry.model.registry.label.PremiumList
+   * @see google.registry.model.tld.label.ReservedList
+   * @see google.registry.model.tld.label.PremiumList
   */
  public static Duration getDomainLabelListCacheDuration() {
    return Duration.standardSeconds(CONFIG_SETTINGS.get().caching.domainLabelCachingSeconds);
@@ -1487,37 +1500,6 @@ public final class RegistryConfig {
    return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
  }

-  /**
-   * Returns whether to replicate cloud SQL transactions to datastore.
-   *
-   * <p>If true, all cloud SQL transactions will be persisted as TransactionEntity objects in the
-   * Transaction table and replayed against datastore in a cron job.
-   */
-  public static boolean getCloudSqlReplicateTransactions() {
-    return CONFIG_SETTINGS.get().cloudSql.replicateTransactions;
-  }
-
-  @VisibleForTesting
-  public static void overrideCloudSqlReplicateTransactions(boolean replicateTransactions) {
-    CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
-  }
-
-  /**
-   * Returns whether or not to replay commit logs to the SQL database after export to GCS.
-   *
-   * <p>If true, we will trigger the {@link google.registry.backup.ReplayCommitLogsToSqlAction}
-   * after the {@link google.registry.backup.ExportCommitLogDiffAction} to load the commit logs and
-   * replay them to SQL.
-   */
-  public static boolean getCloudSqlReplayCommitLogs() {
-    return CONFIG_SETTINGS.get().cloudSql.replayCommitLogs;
-  }
-
-  @VisibleForTesting
-  public static void overrideCloudSqlReplayCommitLogs(boolean replayCommitLogs) {
-    CONFIG_SETTINGS.get().cloudSql.replayCommitLogs = replayCommitLogs;
-  }
-
  /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
  public static String getContactAndHostRoidSuffix() {
    return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;
--- a/core/src/main/java/google/registry/config/RegistryConfigSettings.java
+++ b/core/src/main/java/google/registry/config/RegistryConfigSettings.java
@@ -45,6 +45,7 @@ public class RegistryConfigSettings {
  /** Configuration options that apply to the entire App Engine project. */
  public static class AppEngine {
    public String projectId;
+    public String locationId;
    public boolean isLocal;
    public String defaultServiceUrl;
    public String backendServiceUrl;
@@ -126,8 +127,6 @@ public class RegistryConfigSettings {
    // TODO(05012021): remove username field after it is removed from all yaml files.
    public String username;
    public String instanceConnectionName;
-    public boolean replicateTransactions;
-    public boolean replayCommitLogs;
  }

  /** Configuration for Apache Beam (Cloud Dataflow). */
@@ -229,7 +228,10 @@ public class RegistryConfigSettings {
  public static class SslCertificateValidation {
    public Map<String, Integer> maxValidityDaysSchedule;
    public int expirationWarningDays;
+    public int expirationWarningIntervalDays;
    public int minimumRsaKeyLength;
    public Set<String> allowedEcdsaCurves;
+    public String expirationWarningEmailBodyText;
+    public String expirationWarningEmailSubjectText;
  }
 }
--- a/core/src/main/java/google/registry/config/files/default-config.yaml
+++ b/core/src/main/java/google/registry/config/files/default-config.yaml
@@ -8,6 +8,10 @@
 appEngine:
  # Globally unique App Engine project ID
  projectId: registry-project-id
+  # Location of the App engine project, note that us-central1 and europe-west1 are special in that
+  # they are used without the trailing number in App Engine commands and Google Cloud Console.
+  # See: https://cloud.google.com/appengine/docs/locations
+  locationId: registry-location-id

  # whether to use local/test credentials when connecting to the servers
  isLocal: true
@@ -227,11 +231,6 @@ cloudSql:
  jdbcUrl: jdbc:postgresql://localhost
  # This name is used by Cloud SQL when connecting to the database.
  instanceConnectionName: project-id:region:instance-id
-  # Set this to true to replicate cloud SQL transactions to datastore in the
-  # background.
-  replicateTransactions: false
-  # Set this to true to enable replay of commit logs to SQL
-  replayCommitLogs: false

 cloudDns:
  # Set both properties to null in Production.
@@ -454,6 +453,39 @@ sslCertificateValidation:
  # The number of days before a certificate expires that indicates the
  # certificate is nearing expiration and warnings should be sent.
  expirationWarningDays: 30
+  # The minimum number of days between two successive expiring notification emails.
+  expirationWarningIntervalDays: 15
+  # Text for expiring certificate notification email subject.
+  expirationWarningEmailSubjectText: "[Important] Expiring SSL certificate for Google Registry EPP connection"
+  # Text for expiring certificate notification email body that accepts 3 parameters:
+  # registrar name, certificate type, and expiration date, respectively.
+  expirationWarningEmailBodyText: |
+    Dear %1$s,
+
+    We would like to inform you that your %2$s SSL certificate will expire at %3$s. Please take note that using expired certificates will prevent successful Registry login.
+
+    Kindly update your production account certificate within the support console using the following steps:
+
+      1. Navigate to support.registry.google and login using your %4$s@registry.google credentials.
+          * If this is your first time logging in, you will be prompted to reset your password, so please keep your new password safe.
+          * If you are already logged in with some other Google account(s) but not your %4$s@registry.google account, you need to click on
+          “Add Account” and login using your %4$s@registry.google credentials.
+      2. Select “Settings > Security” from the left navigation bar.
+      3. Click “Edit” on the top left corner.
+      4. Enter your full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----) in the box.
+      5. Click “Save”. If there are validation issues with the form, you will be prompted to fix them and click “Save” again.
+
+    A failover SSL certificate can also be added in order to prevent connection issues once your main certificate expires. Connecting with either of the certificates will work with our production EPP server.
+
+    Further information about our EPP connection requirements can be found in section 9.2 in the updated Technical Guide in your Google Drive folder.
+
+    Note that account certificate changes take a few minutes to become effective and that the existing connections will remain unaffected by the change.
+
+    If you also would like to update your OT&E account certificate, please send an email from your primary or technical contact to registry-support@google.com and include the full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).
+
+    Regards,
+    Google Registry
+
  # The minimum number of bits an RSA key must contain.
  minimumRsaKeyLength: 2048
  # The ECDSA curves that are allowed for public keys.
--- a/Show More
+++ b/Show More