Remove recursive load in DBMSS cache (#1286 )

* Remove recursive load in DBMSS cache This occurs because if we do a standard transaction, the JpaTxnManager checks to see if we should be doing backups, which involves loading the migration state schedule (causing the recursion). When starting the transaction to load the schedule, we should explicitly transactWithoutBackup so there's no need to check. This wasn't hit in tests because we previously manually set the replication to not occur in the JpaTransactionManagerExtension -- we remove that and related setters.
Add instructions for two-step DB schema updates (#1283 )
2026-02-04 20:12:23 +00:00 · 2021-08-14 12:34:23 -06:00 · 2021-08-13 17:21:38 -04:00 · 2021-08-13 16:22:54 -04:00 · 2021-08-13 15:32:18 -04:00 · 2021-08-13 14:16:58 -04:00
781 changed files with 19321 additions and 18462 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 ######################################################################
 # Java Ignores

+gjf.out
 *.class

 # Mobile Tools for Java (J2ME)
@@ -111,3 +112,7 @@ core/**/registrar_bin*.js
 core/**/registrar_dbg*.js
 core/**/registrar_bin*.css
 core/**/registrar_dbg*.css
+
+# Appengine generated files
+core/WEB-INF/appengine-generated/*.bin
+core/WEB-INF/appengine-generated/*.xml
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -0,0 +1,4 @@
+To report a security issue, please use http://g.co/vulnz. We use
+http://g.co/vulnz for our intake, and do coordination and disclosure here on
+GitHub (including using GitHub Security Advisory). The Google Security Team will
+respond within 5 working days of your report on g.co/vulnz.
--- a/build.gradle
+++ b/build.gradle
@@ -318,7 +318,7 @@ subprojects {
  // expose to users.
  if (project.name != 'docs') {
    javadocSource << project.sourceSets.main.allJava
-    javadocClasspath << project.sourceSets.main.compileClasspath
+    javadocClasspath << project.sourceSets.main.runtimeClasspath
    javadocClasspath << "${buildDir}/generated/sources/annotationProcessor/java/main"
    javadocDependentTasks << project.tasks.compileJava
  }
@@ -457,6 +457,8 @@ task javaIncrementalFormatApply {
 task javadoc(type: Javadoc) {
  source javadocSource
  classpath = files(javadocClasspath)
+  // Exclude the misbehaving generated-by-Soy Java files
+  exclude "**/*SoyInfo.java"
  destinationDir = file("${buildDir}/docs/javadoc")
  options.encoding = "UTF-8"
  // In a lot of places we don't write @return so suppress warnings about that.
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -72,6 +72,7 @@ dependencies {
  compile deps['com.google.auth:google-auth-library-credentials']
  compile deps['com.google.auth:google-auth-library-oauth2-http']
  compile deps['com.google.auto.value:auto-value-annotations']
+  compile deps['com.google.common.html.types:types']
  compile deps['com.google.cloud:google-cloud-core']
  compile deps['com.google.cloud:google-cloud-storage']
  compile deps['com.google.guava:guava']
--- a/buildSrc/gradle/dependency-locks/compile.lockfile
+++ b/buildSrc/gradle/dependency-locks/compile.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/compileClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/compileClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/deploy_jar.lockfile
+++ b/buildSrc/gradle/dependency-locks/deploy_jar.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/runtime.lockfile
+++ b/buildSrc/gradle/dependency-locks/runtime.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 commons-codec:commons-codec:1.11
 commons-logging:commons-logging:1.2
@@ -47,17 +48,16 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.apache.commons:commons-lang3:3.8.1
 org.apache.commons:commons-text:1.6
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testCompile.lockfile
+++ b/buildSrc/gradle/dependency-locks/testCompile.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testRuntime.lockfile
+++ b/buildSrc/gradle/dependency-locks/testRuntime.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile
+++ b/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile
@@ -20,12 +20,12 @@ com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.http-client:google-http-client-apache-v2:1.39.0
 com.google.http-client:google-http-client-appengine:1.39.0
 com.google.http-client:google-http-client-gson:1.39.0
@@ -34,10 +34,11 @@ com.google.http-client:google-http-client:1.39.0
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.oauth-client:google-oauth-client:1.31.4
 com.google.protobuf:protobuf-java-util:3.15.3
 com.google.protobuf:protobuf-java:3.15.3
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.ibm.icu:icu4j:57.1
@@ -49,7 +50,6 @@ io.opencensus:opencensus-contrib-http-util:0.28.0
 javax.annotation:javax.annotation-api:1.3.2
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 junit:junit:4.13.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -70,9 +70,9 @@ org.junit:junit-bom:5.6.2
 org.mockito:mockito-core:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
 org.ow2.asm:asm:9.0
 org.threeten:threetenbp:1.5.0
--- a/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java
+++ b/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java
@@ -23,6 +23,7 @@ import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.html.types.TrustedResourceUrls;
 import com.google.template.soy.SoyFileSet;
 import com.google.template.soy.tofu.SoyTofu;
 import google.registry.gradle.plugin.ProjectData.TaskData;
@@ -118,7 +119,7 @@ final class CoverPageGenerator {

    builder.put("projectState", state.toString());
    builder.put("title", title);
-    builder.put("cssFiles", ImmutableSet.of("css/style.css"));
+    builder.put("cssFiles", ImmutableSet.of(TrustedResourceUrls.fromConstant("css/style.css")));
    builder.put("invocation", getInvocation());
    builder.put("tasksByState", getTasksByStateSoyData());
    return builder.build();
--- a/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy
+++ b/buildSrc/src/main/resources/google/registry/gradle/plugin/soy/coverpage.soy
@@ -16,7 +16,7 @@

 {template .coverPage}
  {@param title: string}
-  {@param cssFiles: list<string>}
+  {@param cssFiles: list<trusted_resource_uri>}
  {@param projectState: string}
  {@param invocation: string}
  {@param tasksByState: map<string, list<[uniqueName: string, description: string, log: string, reports: map<string, string>]>>}
--- a/common/gradle/dependency-locks/compile.lockfile
+++ b/common/gradle/dependency-locks/compile.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/compileClasspath.lockfile
+++ b/common/gradle/dependency-locks/compileClasspath.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/default.lockfile
+++ b/common/gradle/dependency-locks/default.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/deploy_jar.lockfile
+++ b/common/gradle/dependency-locks/deploy_jar.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/runtime.lockfile
+++ b/common/gradle/dependency-locks/runtime.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/runtimeClasspath.lockfile
+++ b/common/gradle/dependency-locks/runtimeClasspath.lockfile
@@ -2,11 +2,11 @@
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
 com.google.code.findbugs:jsr305:3.0.2
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 javax.inject:javax.inject:1
 joda-time:joda-time:2.9.2
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
--- a/common/gradle/dependency-locks/testCompile.lockfile
+++ b/common/gradle/dependency-locks/testCompile.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testCompileClasspath.lockfile
+++ b/common/gradle/dependency-locks/testCompileClasspath.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testRuntime.lockfile
+++ b/common/gradle/dependency-locks/testRuntime.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testRuntimeClasspath.lockfile
+++ b/common/gradle/dependency-locks/testRuntimeClasspath.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingCompile.lockfile
+++ b/common/gradle/dependency-locks/testingCompile.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingCompileClasspath.lockfile
+++ b/common/gradle/dependency-locks/testingCompileClasspath.lockfile
@@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
 com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingRuntime.lockfile
+++ b/common/gradle/dependency-locks/testingRuntime.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile
+++ b/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile
@@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.truth:truth:1.1.2
--- a/common/src/main/java/google/registry/util/DateTimeUtils.java
+++ b/common/src/main/java/google/registry/util/DateTimeUtils.java
@@ -100,6 +100,15 @@ public class DateTimeUtils {
    return ZonedDateTime.ofInstant(instant, ZoneId.of(dateTime.getZone().getID()).normalized());
  }

+  /**
+   * Converts a Joda {@link DateTime} object to an equivalent java.time {@link ZonedDateTime}
+   * object.
+   */
+  public static ZonedDateTime toZonedDateTime(DateTime dateTime, ZoneId zoneId) {
+    java.time.Instant instant = java.time.Instant.ofEpochMilli(dateTime.getMillis());
+    return ZonedDateTime.ofInstant(instant, zoneId);
+  }
+
  /**
   * Converts a java.time {@link ZonedDateTime} object to an equivalent Joda {@link DateTime}
   * object.
--- a/config/presubmits.py
+++ b/config/presubmits.py
@@ -79,9 +79,9 @@ PRESUBMITS = {
        r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
        ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
            ".git", "/build/", "/generated/", "/generated_tests/",
-            "node_modules/", "JUnitBackports.java", "registrar_bin.",
-            "registrar_dbg.", "google-java-format-diff.py",
-            "nomulus.golden.sql", "soyutils_usegoog.js"
+            "node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
+            "registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
+            "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
        }, REQUIRED):
        "File did not include the license header.",

@@ -202,6 +202,8 @@ PRESUBMITS = {
        "java",
        # ActivityReportingQueryBuilder deals with Dremel queries
        {"src/test", "ActivityReportingQueryBuilder.java",
+         # This class contains helper method to make queries in Beam.
+         "RegistryJpaIO.java",
         # TODO(b/179158393): Remove everything below, which should be done
         # using Criteria
         "ForeignKeyIndex.java",
@@ -213,6 +215,7 @@ PRESUBMITS = {
         "RdapDomainSearchAction.java",
         "RdapNameserverSearchAction.java",
         "RdapSearchActionBase.java",
+         "RegistryQuery",
         },
    ):
        "The first String parameter to EntityManager.create(Native)Query "
--- a/core/WEB-INF/appengine-generated/local_db.bin
+++ b/core/WEB-INF/appengine-generated/local_db.bin
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -74,11 +74,12 @@ def fragileTestPatterns = [
    "google/registry/cron/TldFanoutActionTest.*",
    // Test Datastore inexplicably aborts transaction.
    "google/registry/model/tmch/ClaimsListShardTest.*",
-    // Creates large object (64MBytes), occasionally throws OOM error.
-    "google/registry/model/server/KmsSecretRevisionTest.*",
    // Changes cache timeouts and for some reason appears to have contention
    // with other tests.
    "google/registry/whois/WhoisCommandFactoryTest.*",
+    // Currently changes a global configuration parameter that for some reason
+    // results in timestamp inversions for other tests.  TODO(mmuller): fix.
+    "google/registry/flows/host/HostInfoFlowTest.*",
 ] + dockerIncompatibleTestPatterns

 sourceSets {
@@ -194,6 +195,7 @@ dependencies {
  compile deps['com.google.apis:google-api-services-groupssettings']
  compile deps['com.google.apis:google-api-services-monitoring']
  compile deps['com.google.apis:google-api-services-sheets']
+  compile deps['com.google.apis:google-api-services-storage']
  testCompile deps['com.google.appengine:appengine-api-stubs']
  compile deps['com.google.appengine.tools:appengine-gcs-client']
  compile deps['com.google.appengine.tools:appengine-mapreduce']
@@ -218,6 +220,8 @@ dependencies {
  gradleLint.ignore('unused-dependency') {
    compile deps['com.google.gwt:gwt-user']
  }
+  compile deps['com.google.cloud:google-cloud-core']
+  compile deps['com.google.cloud:google-cloud-storage']
  compile deps['com.google.http-client:google-http-client']
  compile deps['com.google.http-client:google-http-client-appengine']
  compile deps['com.google.http-client:google-http-client-jackson2']
@@ -312,11 +316,13 @@ dependencies {
  annotationProcessor project(':processor')
  testAnnotationProcessor project(':processor')

+  testCompile deps['com.google.cloud:google-cloud-nio']
  testCompile deps['com.google.appengine:appengine-testing']
  testCompile deps['com.google.guava:guava-testlib']
  testCompile deps['com.google.monitoring-client:contrib']
  testCompile deps['com.google.truth:truth']
  testCompile deps['com.google.truth.extensions:truth-java8-extension']
+  testCompile deps['org.checkerframework:checker-qual']
  testCompile deps['org.hamcrest:hamcrest']
  testCompile deps['org.hamcrest:hamcrest-core']
  testCompile deps['org.hamcrest:hamcrest-library']
@@ -330,7 +336,6 @@ dependencies {
  testCompile deps['org.junit.platform:junit-platform-suite-api']
  testCompile deps['org.mockito:mockito-core']
  testCompile deps['org.mockito:mockito-junit-jupiter']
-  testCompile 'org.checkerframework:checker-qual:3.9.1'
  runtime deps['org.postgresql:postgresql']

  // Indirect dependency found by undeclared-dependency check. Such
@@ -433,12 +438,9 @@ task soyToJava {
  // Relative paths of soy directories.
  def spec11SoyDir = "google/registry/reporting/spec11/soy"
  def toolsSoyDir = "google/registry/tools/soy"
-  def uiSoyDir = "google/registry/ui/soy"
  def registrarSoyDir = "google/registry/ui/soy/registrar"

-  def soyRelativeDirs = [
-          spec11SoyDir, toolsSoyDir, uiSoyDir, registrarSoyDir,
-  ]
+  def soyRelativeDirs = [spec11SoyDir, toolsSoyDir, registrarSoyDir]
  soyRelativeDirs.each {
    inputs.dir "${resourcesSourceDir}/${it}"
    outputs.dir "${generatedDir}/${it}"
@@ -452,7 +454,8 @@ task soyToJava {
              "--outputDirectory", "${outputDirectory}",
              "--javaClassNameSource", "filename",
              "--allowExternalCalls", "true",
-              "--srcs", "${soyFiles.join(',')}"
+              "--srcs", "${soyFiles.join(',')}",
+              "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
    }
  }

@@ -469,14 +472,6 @@ task soyToJava {
                    dir: "${resourcesSourceDir}/${registrarSoyDir}",
                    include: ['**/*.soy']))

-    soyToJava('google.registry.ui.soy',
-            "${generatedDir}/${uiSoyDir}",
-            files {
-              file("${resourcesSourceDir}/${uiSoyDir}").listFiles()
-            }.filter {
-              it.name.endsWith(".soy")
-            })
-
    soyToJava('google.registry.reporting.spec11.soy',
            "${generatedDir}/${spec11SoyDir}",
            fileTree(
@@ -485,42 +480,24 @@ task soyToJava {
  }
 }

-task soyToJS {
-  def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy"
-  def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy"
+task soyToJS(type: JavaExec) {
+  def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy/registrar"
+  def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy/registrar"
  inputs.dir rootSoyDirectory
  outputs.dir outputSoyDirectory

-  ext.soyToJS = { outputDirectory, soyFiles , deps->
-    javaexec {
-      main = "com.google.template.soy.SoyToJsSrcCompiler"
-      classpath configurations.soy
+  def inputSoyFiles = files {
+    file("${rootSoyDirectory}").listFiles()
+  }.filter {
+    it.name.endsWith(".soy")
+  }

-      args "--outputPathFormat", "${outputDirectory}/{INPUT_FILE_NAME}.js",
+  classpath configurations.soy
+  main = "com.google.template.soy.SoyToJsSrcCompiler"
+  args "--outputPathFormat", "${outputSoyDirectory}/{INPUT_FILE_NAME}.js",
          "--allowExternalCalls", "false",
-          "--srcs", "${soyFiles.join(',')}",
-          "--shouldProvideRequireSoyNamespaces", "true",
+          "--srcs", "${inputSoyFiles.join(',')}",
          "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
-      if (deps != "") {
-        args "--deps", "${deps.join(',')}"
-      }
-    }
-  }
-
-  doLast {
-    def rootSoyFiles =
-      fileTree(
-        dir: "${rootSoyDirectory}",
-        include: ['*.soy'])
-
-    soyToJS("${outputSoyDirectory}", rootSoyFiles, "")
-    soyToJS("${outputSoyDirectory}/registrar",
-      files {
-        file("${rootSoyDirectory}/registrar").listFiles()
-      }.filter {
-        it.name.endsWith(".soy")
-      }, rootSoyFiles)
-  }
 }

 task stylesheetsToJavascript {
@@ -603,8 +580,8 @@ task compileProdJS(type: JavaExec) {
  closureArgs << "--generate_exports"

  // manually include all the required js files
-  closureArgs << "--js=${nodeModulesDir}/google-closure-library/**.js"
-  closureArgs << "--js=${jsDir}/soyutils_usegoog.js"
+  closureArgs << "--js=${nodeModulesDir}/google-closure-library/**/*.js"
+  closureArgs << "--js=${jsDir}/*.js"
  closureArgs << "--js=${cssSourceDir}/registrar_bin.css.js"
  closureArgs << "--js=${jsSourceDir}/**.js"
  closureArgs << "--js=${externsDir}/json.js"
@@ -631,15 +608,6 @@ compileProdJS.dependsOn processResources
 compileProdJS.dependsOn processTestResources
 compileProdJS.dependsOn soyToJS

-task karmaTest(type: Exec) {
-  dependsOn ':npmInstall'
-  workingDir rootProject.projectDir
-  executable '.gradle/nodejs/node-v14.15.5-linux-x64/bin/node'
-  args('node_modules/karma/bin/karma', 'start', "${project.projectDir}/karma.conf.js")
-}
-
-test.dependsOn karmaTest
-
 // Make testing artifacts available to be depended up on by other projects.
 // TODO: factor out google.registry.testing to be a separate project.
 task testJar(type: Jar) {
@@ -814,6 +782,10 @@ if (environment in ['alpha', 'crash']) {
          mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
          metaData: 'google/registry/beam/invoicing_pipeline_metadata.json'
      ],
+      [
+          mainClass: 'google.registry.beam.rde.RdePipeline',
+          metaData: 'google/registry/beam/rde_pipeline_metadata.json'
+      ],
  ]
  project.tasks.create("stage_beam_pipelines") {
    doLast {
--- a/core/gradle/dependency-locks/annotationProcessor.lockfile
+++ b/core/gradle/dependency-locks/annotationProcessor.lockfile
@@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33
 com.google.dagger:dagger-spi:2.33
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotation:2.3.4
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.errorprone:error_prone_check_api:2.3.4
 com.google.errorprone:error_prone_core:2.3.4
 com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
@@ -32,7 +32,7 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.checkerframework:dataflow:3.0.0
 org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
--- a/core/gradle/dependency-locks/closureCompiler.lockfile
+++ b/core/gradle/dependency-locks/closureCompiler.lockfile
@@ -1,15 +1,4 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-args4j:args4j:2.0.26
-com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.7
-com.google.errorprone:error_prone_annotations:2.3.1
-com.google.guava:guava:25.1-jre
-com.google.j2objc:j2objc-annotations:1.1
-com.google.javascript:closure-compiler-externs:v20190301
-com.google.javascript:closure-compiler:v20190301
-com.google.jsinterop:jsinterop-annotations:1.0.0
-com.google.protobuf:protobuf-java:3.0.2
-org.checkerframework:checker-qual:2.0.0
-org.codehaus.mojo:animal-sniffer-annotations:1.14
+com.google.javascript:closure-compiler:v20210505
--- a/core/gradle/dependency-locks/compile.lockfile
+++ b/core/gradle/dependency-locks/compile.lockfile
@@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -97,22 +97,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -132,10 +134,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -220,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/compileClasspath.lockfile
+++ b/core/gradle/dependency-locks/compileClasspath.lockfile
@@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -96,21 +96,23 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -130,10 +132,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -214,7 +216,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
--- a/core/gradle/dependency-locks/default.lockfile
+++ b/core/gradle/dependency-locks/default.lockfile
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -102,22 +102,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -137,10 +139,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -231,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/deploy_jar.lockfile
+++ b/core/gradle/dependency-locks/deploy_jar.lockfile
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -102,22 +102,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -137,10 +139,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -230,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/nonprodCompile.lockfile
+++ b/core/gradle/dependency-locks/nonprodCompile.lockfile
@@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -97,22 +97,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -132,10 +134,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -220,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile
+++ b/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile
@@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -96,21 +96,23 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -130,10 +132,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -215,7 +217,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
--- a/core/gradle/dependency-locks/nonprodRuntime.lockfile
+++ b/core/gradle/dependency-locks/nonprodRuntime.lockfile
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -101,22 +101,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -136,10 +138,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -230,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile
+++ b/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -101,22 +101,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -136,10 +138,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -230,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/runtime.lockfile
+++ b/core/gradle/dependency-locks/runtime.lockfile
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -101,22 +101,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -136,10 +138,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -230,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/runtimeClasspath.lockfile
+++ b/core/gradle/dependency-locks/runtimeClasspath.lockfile
@@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
 com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
+com.google.api.grpc:proto-google-iam-v1:1.0.10
 com.google.api:api-common:1.10.1
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
+com.google.api:gax-httpjson:0.79.0
 com.google.api:gax:1.62.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -102,22 +102,24 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.94.1
+com.google.cloud:google-cloud-core:1.94.3
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.113.12
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotations:2.5.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
 com.google.http-client:google-http-client-apache-v2:1.39.0
@@ -137,10 +139,10 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
 com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.protobuf:protobuf-java-util:3.15.3
+com.google.protobuf:protobuf-java:3.15.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.googlecode.charts4j:charts4j:1.3
 com.googlecode.json-simple:json-simple:1.1.1
 com.ibm.icu:icu4j:68.2
@@ -230,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
 org.bouncycastle:bcpkix-jdk15on:1.61
 org.bouncycastle:bcprov-jdk15on:1.61
 org.checkerframework:checker-compat-qual:2.5.5
-org.checkerframework:checker-qual:3.7.0
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.jackson:jackson-core-asl:1.9.13
 org.codehaus.jackson:jackson-mapper-asl:1.9.13
 org.codehaus.mojo:animal-sniffer-annotations:1.20
--- a/core/gradle/dependency-locks/soy.lockfile
+++ b/core/gradle/dependency-locks/soy.lockfile
@@ -5,25 +5,25 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 com.google.code.findbugs:jsr305:3.0.2
 com.google.code.gson:gson:2.7
-com.google.common.html.types:types:1.0.4
+com.google.common.html.types:types:1.0.6
 com.google.errorprone:error_prone_annotations:2.3.4
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.guava:failureaccess:1.0.1
 com.google.guava:guava:30.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.google.gwt:gwt-user:2.8.0-beta1
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:5.0.1
 com.google.j2objc:j2objc-annotations:1.3
+com.google.jsinterop:jsinterop-annotations:1.0.1
 com.google.protobuf:protobuf-java:3.13.0
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.ibm.icu:icu4j:57.1
 javax.annotation:jsr250-api:1.0
 javax.inject:javax.inject:1
-javax.validation:validation-api:1.0.0.GA
 org.checkerframework:checker-qual:3.5.0
 org.json:json:20160212
-org.ow2.asm:asm-analysis:6.0
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-org.ow2.asm:asm-util:6.0
-org.ow2.asm:asm:6.0
+org.ow2.asm:asm-analysis:7.0
+org.ow2.asm:asm-commons:7.0
+org.ow2.asm:asm-tree:7.0
+org.ow2.asm:asm-util:7.0
+org.ow2.asm:asm:7.0
--- a/core/gradle/dependency-locks/testAnnotationProcessor.lockfile
+++ b/core/gradle/dependency-locks/testAnnotationProcessor.lockfile
@@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33
 com.google.dagger:dagger-spi:2.33
 com.google.dagger:dagger:2.33
 com.google.errorprone:error_prone_annotation:2.3.4
-com.google.errorprone:error_prone_annotations:2.3.4
+com.google.errorprone:error_prone_annotations:2.5.1
 com.google.errorprone:error_prone_check_api:2.3.4
 com.google.errorprone:error_prone_core:2.3.4
 com.google.errorprone:error_prone_type_annotations:2.3.4
 com.google.errorprone:javac-shaded:9-dev-r4023-3
 com.google.googlejavaformat:google-java-format:1.5
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava:30.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.j2objc:j2objc-annotations:1.3
 com.google.protobuf:protobuf-java:3.4.0
@@ -30,7 +30,7 @@ javax.inject:javax.inject:1
 javax.persistence:javax.persistence-api:2.2
 net.ltgt.gradle.incap:incap:0.2
 org.checkerframework:checker-compat-qual:2.5.3
-org.checkerframework:checker-qual:3.5.0
+org.checkerframework:checker-qual:3.8.0
 org.checkerframework:dataflow:3.0.0
 org.checkerframework:javacutil:3.0.0
 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
--- a/core/gradle/dependency-locks/testCompile.lockfile
+++ b/core/gradle/dependency-locks/testCompile.lockfile
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -54,12 +54,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -84,10 +84,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -98,31 +98,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -134,11 +137,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -160,7 +163,7 @@ io.github.classgraph:classgraph:4.8.102
 io.grpc:grpc-alts:1.36.0
 io.grpc:grpc-api:1.36.0
 io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
+io.grpc:grpc-context:1.39.0
 io.grpc:grpc-core:1.36.0
 io.grpc:grpc-grpclb:1.36.0
 io.grpc:grpc-netty-shaded:1.36.0
@@ -198,7 +201,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
@@ -306,7 +309,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/testCompileClasspath.lockfile
+++ b/core/gradle/dependency-locks/testCompileClasspath.lockfile
@@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
 args4j:args4j:2.0.23
 cglib:cglib-nodep:2.2
 com.beust:jcommander:1.60
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -53,12 +53,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -83,10 +83,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -97,30 +97,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -132,11 +135,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.14.0
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -158,7 +161,7 @@ io.github.classgraph:classgraph:4.8.102
 io.grpc:grpc-alts:1.36.0
 io.grpc:grpc-api:1.36.0
 io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
+io.grpc:grpc-context:1.39.0
 io.grpc:grpc-core:1.36.0
 io.grpc:grpc-grpclb:1.36.0
 io.grpc:grpc-netty-shaded:1.36.0
@@ -193,7 +196,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
 net.java.dev.jna:jna:5.5.0
@@ -300,7 +303,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.xerial.snappy:snappy-java:1.1.4
--- a/core/gradle/dependency-locks/testRuntime.lockfile
+++ b/core/gradle/dependency-locks/testRuntime.lockfile
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -103,31 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,11 +142,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -169,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
 io.grpc:grpc-alts:1.36.0
 io.grpc:grpc-api:1.36.0
 io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
+io.grpc:grpc-context:1.39.0
 io.grpc:grpc-core:1.36.0
 io.grpc:grpc-grpclb:1.36.0
 io.grpc:grpc-netty-shaded:1.36.0
@@ -207,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -319,7 +322,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/gradle/dependency-locks/testRuntimeClasspath.lockfile
+++ b/core/gradle/dependency-locks/testRuntimeClasspath.lockfile
@@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
-com.fasterxml.jackson.core:jackson-annotations:2.12.1
-com.fasterxml.jackson.core:jackson-core:2.12.1
-com.fasterxml.jackson.core:jackson-databind:2.12.1
-com.fasterxml.jackson:jackson-bom:2.12.1
+com.fasterxml.jackson.core:jackson-annotations:2.12.3
+com.fasterxml.jackson.core:jackson-core:2.12.3
+com.fasterxml.jackson.core:jackson-databind:2.12.3
+com.fasterxml.jackson:jackson-bom:2.12.3
 com.fasterxml:classmate:1.5.1
 com.github.docker-java:docker-java-api:3.2.7
 com.github.docker-java:docker-java-transport-zerodep:3.2.7
@@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
 com.google.api-client:google-api-client-jackson2:1.30.10
 com.google.api-client:google-api-client-java6:1.31.3
 com.google.api-client:google-api-client-servlet:1.31.3
-com.google.api-client:google-api-client:1.31.3
+com.google.api-client:google-api-client:1.32.1
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
@@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
-com.google.api.grpc:proto-google-common-protos:2.1.0
-com.google.api.grpc:proto-google-iam-v1:1.0.9
-com.google.api:api-common:1.10.1
+com.google.api.grpc:proto-google-common-protos:2.3.2
+com.google.api.grpc:proto-google-iam-v1:1.0.14
+com.google.api:api-common:1.10.4
 com.google.api:gax-grpc:1.62.0
-com.google.api:gax-httpjson:0.76.1
-com.google.api:gax:1.62.0
+com.google.api:gax-httpjson:0.83.0
+com.google.api:gax:1.66.0
 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
@@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
-com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
+com.google.apis:google-api-services-storage:v1-rev20210127-1.32.1
 com.google.appengine.tools:appengine-gcs-client:0.8.1
 com.google.appengine.tools:appengine-mapreduce:0.9
 com.google.appengine.tools:appengine-pipeline:0.2.13
@@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
 com.google.appengine:appengine-api-stubs:1.9.86
 com.google.appengine:appengine-remote-api:1.9.86
 com.google.appengine:appengine-testing:1.9.86
-com.google.auth:google-auth-library-credentials:0.24.1
-com.google.auth:google-auth-library-oauth2-http:0.24.1
+com.google.auth:google-auth-library-credentials:0.26.0
+com.google.auth:google-auth-library-oauth2-http:0.26.0
 com.google.auto.service:auto-service-annotations:1.0-rc7
-com.google.auto.value:auto-value-annotations:1.7.4
+com.google.auto.value:auto-value-annotations:1.8.1
 com.google.auto.value:auto-value:1.7.4
 com.google.cloud.bigdataoss:gcsio:2.1.6
 com.google.cloud.bigdataoss:util:2.1.6
@@ -103,31 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
 com.google.cloud:google-cloud-bigquerystorage:1.5.5
 com.google.cloud:google-cloud-bigtable:1.14.0
 com.google.cloud:google-cloud-core-grpc:1.93.9
-com.google.cloud:google-cloud-core-http:1.93.9
-com.google.cloud:google-cloud-core:1.93.9
+com.google.cloud:google-cloud-core-http:1.95.4
+com.google.cloud:google-cloud-core:1.95.4
+com.google.cloud:google-cloud-nio:0.123.4
 com.google.cloud:google-cloud-pubsub:1.110.0
 com.google.cloud:google-cloud-pubsublite:0.7.0
 com.google.cloud:google-cloud-secretmanager:1.4.0
 com.google.cloud:google-cloud-spanner:2.0.2
+com.google.cloud:google-cloud-storage:1.118.0
 com.google.code.findbugs:jsr305:3.0.2
-com.google.code.gson:gson:2.8.6
-com.google.common.html.types:types:1.0.4
+com.google.code.gson:gson:2.8.7
+com.google.common.html.types:types:1.0.6
 com.google.dagger:dagger:2.33
-com.google.errorprone:error_prone_annotations:2.5.1
+com.google.errorprone:error_prone_annotations:2.7.1
+com.google.escapevelocity:escapevelocity:0.9.1
 com.google.flogger:flogger-system-backend:0.5.1
 com.google.flogger:flogger:0.5.1
 com.google.flogger:google-extensions:0.5.1
 com.google.guava:failureaccess:1.0.1
-com.google.guava:guava-testlib:30.1-jre
-com.google.guava:guava:30.1-jre
+com.google.guava:guava-testlib:30.1.1-jre
+com.google.guava:guava:30.1.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.google.gwt:gwt-user:2.9.0
-com.google.http-client:google-http-client-apache-v2:1.39.0
-com.google.http-client:google-http-client-appengine:1.39.0
-com.google.http-client:google-http-client-gson:1.39.0
-com.google.http-client:google-http-client-jackson2:1.39.0
+com.google.http-client:google-http-client-apache-v2:1.39.2
+com.google.http-client:google-http-client-appengine:1.39.2
+com.google.http-client:google-http-client-gson:1.39.2
+com.google.http-client:google-http-client-jackson2:1.39.2
 com.google.http-client:google-http-client-protobuf:1.33.0
-com.google.http-client:google-http-client:1.39.0
+com.google.http-client:google-http-client:1.39.2
 com.google.inject.extensions:guice-multibindings:4.1.0
 com.google.inject:guice:4.1.0
 com.google.j2objc:j2objc-annotations:1.3
@@ -139,11 +142,11 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
 com.google.oauth-client:google-oauth-client-java6:1.31.4
 com.google.oauth-client:google-oauth-client-jetty:1.31.4
 com.google.oauth-client:google-oauth-client-servlet:1.31.4
-com.google.oauth-client:google-oauth-client:1.31.4
-com.google.protobuf:protobuf-java-util:3.15.2
-com.google.protobuf:protobuf-java:3.15.2
+com.google.oauth-client:google-oauth-client:1.31.5
+com.google.protobuf:protobuf-java-util:3.17.3
+com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
-com.google.template:soy:2018-03-14
+com.google.template:soy:2021-02-01
 com.google.truth.extensions:truth-java8-extension:1.1.2
 com.google.truth:truth:1.1.2
 com.googlecode.charts4j:charts4j:1.3
@@ -169,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
 io.grpc:grpc-alts:1.36.0
 io.grpc:grpc-api:1.36.0
 io.grpc:grpc-auth:1.36.0
-io.grpc:grpc-context:1.36.0
+io.grpc:grpc-context:1.39.0
 io.grpc:grpc-core:1.36.0
 io.grpc:grpc-grpclb:1.36.0
 io.grpc:grpc-netty-shaded:1.36.0
@@ -207,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
 javax.xml.bind:jaxb-api:2.3.1
 jline:jline:1.0
 joda-time:joda-time:2.10.5
-junit:junit:4.13.1
+junit:junit:4.13.2
 net.arnx:nashorn-promise:0.1.1
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
@@ -320,7 +323,7 @@ org.testcontainers:junit-jupiter:1.15.2
 org.testcontainers:postgresql:1.15.2
 org.testcontainers:selenium:1.15.2
 org.testcontainers:testcontainers:1.15.2
-org.threeten:threetenbp:1.5.0
+org.threeten:threetenbp:1.5.1
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3
 org.webjars.npm:viz.js-for-graphviz-java:2.1.3
--- a/core/karma.conf.js
+++ b/core/karma.conf.js
@@ -1,75 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-process.env.CHROME_BIN = require('puppeteer').executablePath()
-
-module.exports = function(config) {
-  config.set({
-    basePath: '..',
-    browsers: ['ChromeHeadlessNoSandbox'],
-    customLaunchers: {
-      ChromeHeadlessNoSandbox: {
-        base: 'ChromeHeadless',
-        flags: ['--no-sandbox']
-      }
-    },
-    frameworks: ['jasmine', 'closure'],
-    singleRun: true,
-    autoWatch: false,
-    files: [
-      'node_modules/google-closure-library/closure/goog/base.js',
-      'core/src/test/javascript/**/*_test.js',
-      {
-        pattern: 'core/src/test/javascript/**/!(*_test).js',
-        included: false
-      },
-      {
-        pattern: 'core/src/main/javascript/**/*.js',
-        included: false
-      },
-      {
-        pattern: 'core/build/generated/sources/custom/java/main/**/*.soy.js',
-        included: false
-      },
-      {
-        pattern: 'node_modules/google-closure-library/closure/goog/deps.js',
-        included: false,
-        served: false
-      },
-      {
-        pattern: 'node_modules/google-closure-library/closure/goog/**/*.js',
-        included: false
-      },
-      {
-        pattern: 'core/build/resources/main/google/registry/ui/assets/images/*.png',
-        included: false
-      },
-      {
-        pattern: 'core/build/resources/main/google/registry/ui/assets/images/icons/svg/*.svg',
-        included: false
-      }
-    ],
-    preprocessors: {
-      'node_modules/google-closure-library/closure/goog/deps.js': ['closure', 'closure-deps'],
-      'node_modules/google-closure-library/closure/goog/base.js': ['closure'],
-      'node_modules/google-closure-library/closure/**/*.js': ['closure'],
-      'core/src/*/javascript/**/*.js': ['closure'],
-      'core/build/generated/sources/custom/java/main/**/*.soy.js': ['closure'],
-    },
-    proxies: {
-      "/assets/": "/base/core/build/resources/main/google/registry/ui/assets/"
-    }
-  });
-};
-
--- a/core/src/main/java/google/registry/backup/BackupModule.java
+++ b/core/src/main/java/google/registry/backup/BackupModule.java
@@ -18,8 +18,10 @@ import static com.google.appengine.api.ThreadManager.currentRequestThreadFactory
 import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
+import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
+import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static java.util.concurrent.Executors.newFixedThreadPool;
@@ -32,6 +34,9 @@ import google.registry.cron.CommitLogFanoutAction;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
 import java.lang.annotation.Documented;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import javax.inject.Qualifier;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.DateTime;
@@ -75,6 +80,12 @@ public final class BackupModule {
    return extractRequiredDatetimeParameter(req, UPPER_CHECKPOINT_TIME_PARAM);
  }

+  @Provides
+  @Parameter(BUCKET_OVERRIDE_PARAM)
+  static Optional<String> provideBucketOverride(HttpServletRequest req) {
+    return extractOptionalParameter(req, BUCKET_OVERRIDE_PARAM);
+  }
+
  @Provides
  @Parameter(FROM_TIME_PARAM)
  static DateTime provideFromTime(HttpServletRequest req) {
@@ -92,4 +103,9 @@ public final class BackupModule {
  static ListeningExecutorService provideListeningExecutorService() {
    return listeningDecorator(newFixedThreadPool(NUM_THREADS, currentRequestThreadFactory()));
  }
+
+  @Provides
+  static ScheduledExecutorService provideScheduledExecutorService() {
+    return Executors.newSingleThreadScheduledExecutor();
+  }
 }
--- a/core/src/main/java/google/registry/backup/BackupUtils.java
+++ b/core/src/main/java/google/registry/backup/BackupUtils.java
@@ -56,12 +56,16 @@ public class BackupUtils {
   *
   * <p>The iterator reads from the stream on demand, and as such will fail if the stream is closed.
   */
-  public static Iterator<ImmutableObject> createDeserializingIterator(final InputStream input) {
+  public static Iterator<ImmutableObject> createDeserializingIterator(
+      final InputStream input, boolean withAppIdOverride) {
    return new AbstractIterator<ImmutableObject>() {
      @Override
      protected ImmutableObject computeNext() {
        EntityProto proto = new EntityProto();
        if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw.
+          if (withAppIdOverride) {
+            proto = EntityImports.fixEntity(proto);
+          }
          return auditedOfy().load().fromEntity(EntityTranslator.createFromPb(proto));
        }
        return endOfData();
@@ -70,6 +74,7 @@ public class BackupUtils {
  }

  public static ImmutableList<ImmutableObject> deserializeEntities(byte[] bytes) {
-    return ImmutableList.copyOf(createDeserializingIterator(new ByteArrayInputStream(bytes)));
+    return ImmutableList.copyOf(
+        createDeserializingIterator(new ByteArrayInputStream(bytes), false));
  }
 }
--- a/core/src/main/java/google/registry/backup/CommitLogImports.java
+++ b/core/src/main/java/google/registry/backup/CommitLogImports.java
@@ -59,7 +59,7 @@ public final class CommitLogImports {
      InputStream inputStream) {
    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
        InputStream input = new BufferedInputStream(inputStream)) {
-      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
      checkState(commitLogs.hasNext());
      checkState(commitLogs.next() instanceof CommitLogCheckpoint);

--- a/core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java
+++ b/core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java
@@ -66,6 +66,8 @@ import org.joda.time.Duration;
    service = Action.Service.BACKEND,
    path = "/_dr/task/deleteOldCommitLogs",
    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+// No longer needed in SQL. Subject to future removal.
+@Deprecated
 public final class DeleteOldCommitLogsAction implements Runnable {

  private static final int NUM_MAP_SHARDS = 20;
--- a/core/src/main/java/google/registry/backup/EntityImports.java
+++ b/core/src/main/java/google/registry/backup/EntityImports.java
@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import com.google.apphosting.api.ApiProxy;
+import com.google.storage.onestore.v3.OnestoreEntity;
+import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
+import com.google.storage.onestore.v3.OnestoreEntity.Path;
+import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
+import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+/** Utilities for handling imported Datastore entities. */
+public class EntityImports {
+
+  /**
+   * Transitively sets the {@code appId} of all keys in a foreign entity to that of the current
+   * system.
+   */
+  public static EntityProto fixEntity(EntityProto entityProto) {
+    String currentAappId = ApiProxy.getCurrentEnvironment().getAppId();
+    if (Objects.equals(currentAappId, entityProto.getKey().getApp())) {
+      return entityProto;
+    }
+    return fixEntity(entityProto, currentAappId);
+  }
+
+  private static EntityProto fixEntity(EntityProto entityProto, String appId) {
+    if (entityProto.hasKey()) {
+      fixKey(entityProto, appId);
+    }
+
+    for (OnestoreEntity.Property property : entityProto.mutablePropertys()) {
+      fixProperty(property, appId);
+    }
+
+    for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
+      fixProperty(property, appId);
+    }
+
+    // CommitLogMutation embeds an entity as bytes, which needs additional fixes.
+    if (isCommitLogMutation(entityProto)) {
+      fixMutationEntityProtoBytes(entityProto, appId);
+    }
+    return entityProto;
+  }
+
+  private static boolean isCommitLogMutation(EntityProto entityProto) {
+    if (!entityProto.hasKey()) {
+      return false;
+    }
+    Path path = entityProto.getKey().getPath();
+    if (path.elementSize() == 0) {
+      return false;
+    }
+    return Objects.equals(
+        path.getElement(path.elementSize() - 1).getType(StandardCharsets.UTF_8),
+        "CommitLogMutation");
+  }
+
+  private static void fixMutationEntityProtoBytes(EntityProto entityProto, String appId) {
+    for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) {
+      if (Objects.equals(property.getName(), "entityProtoBytes")) {
+        OnestoreEntity.PropertyValue value = property.getValue();
+        EntityProto fixedProto =
+            fixEntity(bytesToEntityProto(value.getStringValueAsBytes()), appId);
+        value.setStringValueAsBytes(fixedProto.toByteArray());
+        return;
+      }
+    }
+  }
+
+  private static void fixKey(EntityProto entityProto, String appId) {
+    entityProto.getMutableKey().setApp(appId);
+  }
+
+  private static void fixKey(ReferenceValue referenceValue, String appId) {
+    referenceValue.setApp(appId);
+  }
+
+  private static void fixProperty(OnestoreEntity.Property property, String appId) {
+    OnestoreEntity.PropertyValue value = property.getMutableValue();
+    if (value.hasReferenceValue()) {
+      fixKey(value.getMutableReferenceValue(), appId);
+      return;
+    }
+    if (property.getMeaningEnum().equals(Meaning.ENTITY_PROTO)) {
+      EntityProto embeddedProto = bytesToEntityProto(value.getStringValueAsBytes());
+      fixEntity(embeddedProto, appId);
+      value.setStringValueAsBytes(embeddedProto.toByteArray());
+    }
+  }
+
+  private static EntityProto bytesToEntityProto(byte[] bytes) {
+    EntityProto entityProto = new EntityProto();
+    boolean isParsed = entityProto.parseFrom(bytes);
+    if (!isParsed) {
+      throw new IllegalStateException("Failed to parse raw bytes as EntityProto.");
+    }
+    return entityProto;
+  }
+}
--- a/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java
+++ b/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java
@@ -28,18 +28,17 @@ import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
-import static java.nio.channels.Channels.newOutputStream;
 import static java.util.Comparator.comparingLong;

-import com.google.appengine.tools.cloudstorage.GcsFileOptions;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobId;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
@@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable {

  public static final String DIFF_FILE_PREFIX = "commit_diff_until_";

-  @Inject GcsService gcsService;
+  @Inject GcsUtils gcsUtils;
+
  @Inject @Config("commitLogGcsBucket") String gcsBucket;
  @Inject @Config("commitLogDiffExportBatchSize") int batchSize;
  @Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
@@ -102,13 +102,13 @@ public final class ExportCommitLogDiffAction implements Runnable {
    List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
    logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
    // Open an output channel to GCS, wrapped in a stream for convenience.
-    try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
-        new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
-        new GcsFileOptions.Builder()
-            .addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
-            .addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
-            .addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
-            .build()))) {
+    try (OutputStream gcsStream =
+        gcsUtils.openOutputStream(
+            BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
+            ImmutableMap.of(
+                LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
+                UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
+                NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
      // Export the upper checkpoint itself.
      serializeEntity(upperCheckpoint, gcsStream);
      // If there are no manifests to export, stop early, now that we've written out the file with
--- a/core/src/main/java/google/registry/backup/GcsDiffFileLister.java
+++ b/core/src/main/java/google/registry/backup/GcsDiffFileLister.java
@@ -15,30 +15,32 @@
 package google.registry.backup;

 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
 import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;

-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsFilename;
-import com.google.appengine.tools.cloudstorage.GcsService;
-import com.google.appengine.tools.cloudstorage.ListItem;
-import com.google.appengine.tools.cloudstorage.ListOptions;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.UncheckedExecutionException;
 import google.registry.backup.BackupModule.Backups;
-import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
 import java.io.IOException;
-import java.util.Iterator;
+import java.time.Duration;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.ScheduledExecutorService;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
+import javax.inject.Provider;
 import org.joda.time.DateTime;

 /** Utility class to list commit logs diff files stored on GCS. */
@@ -46,33 +48,49 @@ class GcsDiffFileLister {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

-  @Inject GcsService gcsService;
-  @Inject @Config("commitLogGcsBucket") String gcsBucket;
-  @Inject @Backups ListeningExecutorService executor;
-  @Inject GcsDiffFileLister() {}
+  /** Timeout for retrieving per-file information from GCS. */
+  private static final Duration FILE_INFO_TIMEOUT_DURATION = Duration.ofMinutes(1);
+
+  @Inject GcsUtils gcsUtils;
+
+  @Inject @Backups Provider<ListeningExecutorService> executorProvider;
+  @Inject ScheduledExecutorService scheduledExecutorService;
+
+  @Inject
+  GcsDiffFileLister() {}

  /**
   * Traverses the sequence of diff files backwards from checkpointTime and inserts the file
-   * metadata into "sequence".  Returns true if a complete sequence was discovered, false if one or
+   * metadata into "sequence". Returns true if a complete sequence was discovered, false if one or
   * more files are missing.
+   *
+   * @throws UncheckedExecutionException wrapping a {@link java.util.concurrent.TimeoutException} if
+   *     the GCS call fails to finish within one minute, or wrapping any other exception if
+   *     something else goes wrong.
   */
  private boolean constructDiffSequence(
-      Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
+      String gcsBucket,
+      Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
      DateTime fromTime,
      DateTime lastTime,
-      TreeMap<DateTime, GcsFileMetadata> sequence) {
+      TreeMap<DateTime, BlobInfo> sequence) {
    DateTime checkpointTime = lastTime;
    while (isBeforeOrAt(fromTime, checkpointTime)) {
-      GcsFileMetadata metadata;
-      if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
-        metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
+      BlobInfo blobInfo;
+      if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
+        blobInfo =
+            Futures.getUnchecked(
+                Futures.withTimeout(
+                    upperBoundTimesToBlobInfo.get(checkpointTime),
+                    FILE_INFO_TIMEOUT_DURATION,
+                    scheduledExecutorService));
      } else {
        String filename = DIFF_FILE_PREFIX + checkpointTime;
        logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
-        metadata = getMetadata(filename);
+        blobInfo = getBlobInfo(gcsBucket, filename);

        // If we hit a gap, quit.
-        if (metadata == null) {
+        if (blobInfo == null) {
          logger.atInfo().log(
              "Gap discovered in sequence terminating at %s, missing file: %s",
              sequence.lastKey(), filename);
@@ -80,14 +98,15 @@ class GcsDiffFileLister {
          return false;
        }
      }
-      sequence.put(checkpointTime, metadata);
-      checkpointTime = getLowerBoundTime(metadata);
+      sequence.put(checkpointTime, blobInfo);
+      checkpointTime = getLowerBoundTime(blobInfo);
    }
    logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
    return true;
  }

-  ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
+  ImmutableList<BlobInfo> listDiffFiles(
+      String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
    logger.atInfo().log("Requested restore from time: %s", fromTime);
    if (toTime != null) {
      logger.atInfo().log("  Until time: %s", toTime);
@@ -95,66 +114,74 @@ class GcsDiffFileLister {
    // List all of the diff files on GCS and build a map from each file's upper checkpoint time
    // (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
    // an upper checkpoint time > fromTime.
-    TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
-        = new TreeMap<>();
-    Iterator<ListItem> listItems;
+    TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
+    String commitLogDiffPrefix = getCommitLogDiffPrefix(fromTime, toTime);
+    ImmutableList<String> filenames;
    try {
-      // TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
-      listItems = gcsService.list(
-          gcsBucket,
-          new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
+      filenames =
+          gcsUtils.listFolderObjects(gcsBucket, commitLogDiffPrefix).stream()
+              .map(s -> commitLogDiffPrefix + s)
+              .collect(toImmutableList());
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    DateTime lastUpperBoundTime = START_OF_TIME;
-    while (listItems.hasNext()) {
-      final String filename = listItems.next().getName();
-      DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
-      if (isInRange(upperBoundTime, fromTime, toTime)) {
-        upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
-        lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
-      }
-    }
-    if (upperBoundTimesToMetadata.isEmpty()) {
-      logger.atInfo().log("No files found");
-      return ImmutableList.of();
-    }

-    // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
-    // last file that we found) and finding its previous file until we either run out of files or
-    // get to one that precedes "fromTime".
-    //
-    // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
-    // metadata of a file is sufficient to identify the preceding file, so if we start from the
-    // last file and work backwards we can verify that we have no holes in our chain (although we
-    // may be missing files at the end).
-    TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
-    logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
-    boolean inconsistentFileSet = !constructDiffSequence(
-        upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
-
-    // Verify that all of the elements in the original set are represented in the sequence.  If we
-    // find anything that's not represented, construct a sequence for it.
-    boolean checkForMoreExtraDiffs = true;  // Always loop at least once.
-    while (checkForMoreExtraDiffs) {
-      checkForMoreExtraDiffs = false;
-      for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
-        if (!isInRange(key, fromTime, toTime)) {
-          break;
-        }
-        if (!sequence.containsKey(key)) {
-          constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
-          checkForMoreExtraDiffs = true;
-          inconsistentFileSet = true;
-          break;
+    TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
+    ListeningExecutorService executor = executorProvider.get();
+    try {
+      for (String filename : filenames) {
+        String strippedFilename = filename.replaceFirst(DIFF_FILE_PREFIX, "");
+        DateTime upperBoundTime = DateTime.parse(strippedFilename);
+        if (isInRange(upperBoundTime, fromTime, toTime)) {
+          upperBoundTimesToBlobInfo.put(
+              upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
+          lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
        }
      }
-    }
+      if (upperBoundTimesToBlobInfo.isEmpty()) {
+        logger.atInfo().log("No files found");
+        return ImmutableList.of();
+      }

-    checkState(
-        !inconsistentFileSet,
-        "Unable to compute commit diff history, there are either gaps or forks in the history "
-        + "file set.  Check log for details.");
+      // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e.
+      // the last file that we found) and finding its previous file until we either run out of files
+      // or get to one that precedes "fromTime".
+      //
+      // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
+      // metadata of a file is sufficient to identify the preceding file, so if we start from the
+      // last file and work backwards we can verify that we have no holes in our chain (although we
+      // may be missing files at the end).
+      logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
+      boolean inconsistentFileSet =
+          !constructDiffSequence(
+              gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
+
+      // Verify that all of the elements in the original set are represented in the sequence.  If we
+      // find anything that's not represented, construct a sequence for it.
+      boolean checkForMoreExtraDiffs = true; // Always loop at least once.
+      while (checkForMoreExtraDiffs) {
+        checkForMoreExtraDiffs = false;
+        for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
+          if (!isInRange(key, fromTime, toTime)) {
+            break;
+          }
+          if (!sequence.containsKey(key)) {
+            constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
+            checkForMoreExtraDiffs = true;
+            inconsistentFileSet = true;
+            break;
+          }
+        }
+      }
+
+      checkState(
+          !inconsistentFileSet,
+          "Unable to compute commit diff history, there are either gaps or forks in the history "
+              + "file set.  Check log for details.");
+    } finally {
+      executor.shutdown();
+    }

    logger.atInfo().log(
        "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
@@ -171,15 +198,43 @@ class GcsDiffFileLister {
    return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
  }

-  private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
-    return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
+  private DateTime getLowerBoundTime(BlobInfo blobInfo) {
+    return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
  }

-  private GcsFileMetadata getMetadata(String filename) {
-    try {
-      return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
-    } catch (IOException e) {
-      throw new RuntimeException(e);
+  private BlobInfo getBlobInfo(String gcsBucket, String filename) {
+    return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
+  }
+
+  /**
+   * Returns a prefix guaranteed to cover all commit log diff files in the given range.
+   *
+   * <p>The listObjects call can be fairly slow if we search over many thousands or tens of
+   * thousands of files, so we restrict the search space. The commit logs have a file format of
+   * "commit_diff_until_2021-05-11T06:48:00.070Z" so we can often filter down as far as the hour.
+   *
+   * <p>Here, we get the longest prefix possible based on which fields (year, month, day, hour) the
+   * times in question have in common.
+   */
+  @VisibleForTesting
+  static String getCommitLogDiffPrefix(DateTime from, @Nullable DateTime to) {
+    StringBuilder result = new StringBuilder(DIFF_FILE_PREFIX);
+    if (to == null || from.getYear() != to.getYear()) {
+      return result.toString();
    }
+    result.append(from.getYear()).append('-');
+    if (from.getMonthOfYear() != to.getMonthOfYear()) {
+      return result.toString();
+    }
+    result.append(String.format("%02d-", from.getMonthOfYear()));
+    if (from.getDayOfMonth() != to.getDayOfMonth()) {
+      return result.toString();
+    }
+    result.append(String.format("%02dT", from.getDayOfMonth()));
+    if (from.getHourOfDay() != to.getHourOfDay()) {
+      return result.toString();
+    }
+    result.append(String.format("%02d:", from.getHourOfDay()));
+    return result.toString();
  }
 }
--- a/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java
+++ b/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java
@@ -14,70 +14,97 @@

 package google.registry.backup;

+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
+import static google.registry.backup.RestoreCommitLogsAction.DRY_RUN_PARAM;
 import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.DateTimeUtils.isAtOrAfter;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardHours;

 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.Key;
-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobInfo;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.replay.DatastoreEntity;
+import google.registry.model.replay.DatastoreOnlyEntity;
+import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.ReplaySpecializer;
+import google.registry.model.replay.SqlReplayCheckpoint;
 import google.registry.model.server.Lock;
 import google.registry.model.translators.VKeyTranslatorFactory;
 import google.registry.persistence.VKey;
 import google.registry.request.Action;
+import google.registry.request.Action.Method;
+import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.DatastoreOnlyEntity;
-import google.registry.schema.replay.NonReplicatedEntity;
-import google.registry.schema.replay.ReplaySpecializer;
-import google.registry.schema.replay.SqlReplayCheckpoint;
+import google.registry.util.Clock;
 import google.registry.util.RequestStatusChecker;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.channels.Channels;
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.servlet.http.HttpServletResponse;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
+import org.joda.time.Seconds;

 /** Action that replays commit logs to Cloud SQL to keep it up to date. */
@Action(
    service = Action.Service.BACKEND,
    path = ReplayCommitLogsToSqlAction.PATH,
-    method = Action.Method.POST,
+    method = Method.POST,
    automaticallyPrintOk = true,
    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public class ReplayCommitLogsToSqlAction implements Runnable {

  static final String PATH = "/_dr/task/replayCommitLogsToSql";

-  private static final int BLOCK_SIZE =
-      1024 * 1024; // Buffer 1mb at a time, for no particular reason.
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final Duration LEASE_LENGTH = standardHours(1);

-  @Inject GcsService gcsService;
+  private static final Duration LEASE_LENGTH = standardHours(1);
+  // Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
+  // request timeouts
+  private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
+
+  @Inject GcsUtils gcsUtils;
  @Inject Response response;
  @Inject RequestStatusChecker requestStatusChecker;
  @Inject GcsDiffFileLister diffLister;
+  @Inject Clock clock;
+
+  @Inject
+  @Config("commitLogGcsBucket")
+  String gcsBucket;
+
+  /** If true, will exit after logging the commit log files that would otherwise be replayed. */
+  @Inject
+  @Parameter(DRY_RUN_PARAM)
+  boolean dryRun;

  @Inject
  ReplayCommitLogsToSqlAction() {}

  @Override
  public void run() {
-    if (!RegistryConfig.getCloudSqlReplayCommitLogs()) {
-      String message = "ReplayCommitLogsToSqlAction was called but disabled in the config.";
-      logger.atWarning().log(message);
+    DateTime startTime = clock.nowUtc();
+    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(startTime);
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      String message =
+          String.format(
+              "Skipping ReplayCommitLogsToSqlAction because we are in migration phase %s.", state);
+      logger.atInfo().log(message);
      // App Engine will retry on any non-2xx status code, which we don't want in this case.
      response.setStatus(SC_NO_CONTENT);
      response.setPayload(message);
@@ -96,44 +123,117 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
      return;
    }
    try {
-      replayFiles();
-      response.setStatus(HttpServletResponse.SC_OK);
-      logger.atInfo().log("ReplayCommitLogsToSqlAction completed successfully.");
+      logger.atInfo().log("Beginning replay of commit logs.");
+      String resultMessage;
+      if (dryRun) {
+        resultMessage = executeDryRun();
+      } else {
+        resultMessage = replayFiles(startTime);
+      }
+      response.setStatus(SC_OK);
+      response.setPayload(resultMessage);
+      logger.atInfo().log(resultMessage);
+    } catch (Throwable t) {
+      String message = "Errored out replaying files.";
+      logger.atSevere().withCause(t).log(message);
+      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(message);
    } finally {
      lock.ifPresent(Lock::release);
    }
  }

-  private void replayFiles() {
+  private String executeDryRun() {
    // Start at the first millisecond we haven't seen yet
-    DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
-    // If there's an inconsistent file set, this will throw IllegalStateException and the job
-    // will try later -- this is likely because an export hasn't finished yet.
-    ImmutableList<GcsFileMetadata> commitLogFiles =
-        diffLister.listDiffFiles(fromTime, /* current time */ null);
-    for (GcsFileMetadata metadata : commitLogFiles) {
-      // One transaction per GCS file
-      jpaTm().transact(() -> processFile(metadata));
-    }
-    logger.atInfo().log("Replayed %d commit log files to SQL successfully.", commitLogFiles.size());
+    DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    // Search through the end of the hour
+    DateTime searchEndTime =
+        searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
+    ImmutableList<String> fileBatch =
+        diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime).stream()
+            .map(BlobInfo::getName)
+            .collect(toImmutableList());
+    return String.format(
+        "Running in dry-run mode, the first set of commit log files processed would be from "
+            + "searching from %s to %s and would contain %d file(s). They are (limit 10): \n%s",
+        searchStartTime,
+        searchEndTime,
+        fileBatch.size(),
+        fileBatch.stream().limit(10).collect(toImmutableList()));
  }

-  private void processFile(GcsFileMetadata metadata) {
-    try (InputStream input =
-        Channels.newInputStream(
-            gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+  private String replayFiles(DateTime startTime) {
+    DateTime replayTimeoutTime = startTime.plus(REPLAY_TIMEOUT_DURATION);
+    DateTime searchStartTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    int filesProcessed = 0;
+    int transactionsProcessed = 0;
+    // Starting from one millisecond after the last file we processed, search for and import files
+    // one hour at a time until we catch up to the current time or we hit the replay timeout (in
+    // which case the next run will pick up from where we leave off).
+    //
+    // We use hour-long batches because GCS supports filename prefix-based searches.
+    while (true) {
+      if (isAtOrAfter(clock.nowUtc(), replayTimeoutTime)) {
+        return createResponseString(
+            "Reached max execution time", startTime, filesProcessed, transactionsProcessed);
+      }
+      if (isBeforeOrAt(clock.nowUtc(), searchStartTime)) {
+        return createResponseString(
+            "Caught up to current time", startTime, filesProcessed, transactionsProcessed);
+      }
+      // Search through the end of the hour
+      DateTime searchEndTime =
+          searchStartTime.withMinuteOfHour(59).withSecondOfMinute(59).withMillisOfSecond(999);
+      ImmutableList<BlobInfo> fileBatch =
+          diffLister.listDiffFiles(gcsBucket, searchStartTime, searchEndTime);
+      if (fileBatch.isEmpty()) {
+        logger.atInfo().log(
+            "No remaining files found in hour %s, continuing search in the next hour.",
+            searchStartTime.toString("yyyy-MM-dd HH"));
+      }
+      for (BlobInfo file : fileBatch) {
+        transactionsProcessed += processFile(file);
+        filesProcessed++;
+        if (clock.nowUtc().isAfter(replayTimeoutTime)) {
+          return createResponseString(
+              "Reached max execution time", startTime, filesProcessed, transactionsProcessed);
+        }
+      }
+      searchStartTime = searchEndTime.plusMillis(1);
+    }
+  }
+
+  private String createResponseString(
+      String msg, DateTime startTime, int filesProcessed, int transactionsProcessed) {
+    double tps =
+        (double) transactionsProcessed
+            / (double) Seconds.secondsBetween(startTime, clock.nowUtc()).getSeconds();
+    return String.format(
+        "%s after replaying %d file(s) containing %d total transaction(s) (%.2f tx/s).",
+        msg, filesProcessed, transactionsProcessed, tps);
+  }
+
+  /**
+   * Replays the commit logs in the given commit log file and returns the number of transactions
+   * committed.
+   */
+  private int processFile(BlobInfo metadata) {
+    try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
      // Load and process the Datastore transactions one at a time
      ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
          CommitLogImports.loadEntitiesByTransaction(input);
-      allTransactions.forEach(this::replayTransaction);
+      allTransactions.forEach(
+          transaction -> jpaTm().transact(() -> replayTransaction(transaction)));
      // if we succeeded, set the last-seen time
-      DateTime checkpoint =
-          DateTime.parse(
-              metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
-      SqlReplayCheckpoint.set(checkpoint);
-      logger.atInfo().log("Replayed %d transactions from commit log file.", allTransactions.size());
+      DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
+      jpaTm().transact(() -> SqlReplayCheckpoint.set(checkpoint));
+      logger.atInfo().log(
+          "Replayed %d transactions from commit log file %s with size %d B.",
+          allTransactions.size(), metadata.getName(), metadata.getSize());
+      return allTransactions.size();
    } catch (IOException e) {
-      throw new RuntimeException(e);
+      throw new RuntimeException(
+          "Errored out while replaying commit log file " + metadata.getName(), e);
    }
  }

@@ -152,20 +252,25 @@ public class ReplayCommitLogsToSqlAction implements Runnable {

  private void handleEntityPut(Entity entity) {
    Object ofyPojo = auditedOfy().toPojo(entity);
-    if (ofyPojo instanceof DatastoreEntity) {
-      DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
-      datastoreEntity
-          .toSqlEntity()
-          .ifPresent(
-              sqlEntity -> {
-                ReplaySpecializer.beforeSqlSave(sqlEntity);
-                jpaTm().put(sqlEntity);
-              });
-    } else {
-      // this should never happen, but we shouldn't fail on it
-      logger.atSevere().log(
-          "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
-          ofyPojo.getClass());
+    try {
+      if (ofyPojo instanceof DatastoreEntity) {
+        DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
+        datastoreEntity
+            .toSqlEntity()
+            .ifPresent(
+                sqlEntity -> {
+                  sqlEntity.beforeSqlSaveOnReplay();
+                  jpaTm().put(sqlEntity);
+                });
+      } else {
+        // this should never happen, but we shouldn't fail on it
+        logger.atSevere().log(
+            "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
+            ofyPojo.getClass());
+      }
+    } catch (Throwable t) {
+      logger.atSevere().log("Error when replaying object %s", ofyPojo);
+      throw t;
    }
  }

@@ -181,13 +286,18 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
          "Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
      return;
    }
-    Class<?> entityClass = entityVKey.getKind();
-    // Delete the key iff the class represents a JPA entity that is replicated
-    if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
-        && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
-        && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
-      ReplaySpecializer.beforeSqlDelete(entityVKey);
-      jpaTm().delete(entityVKey);
+    try {
+      Class<?> entityClass = entityVKey.getKind();
+      // Delete the key iff the class represents a JPA entity that is replicated
+      if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
+          && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
+          && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
+        ReplaySpecializer.beforeSqlDelete(entityVKey);
+        jpaTm().delete(entityVKey);
+      }
+    } catch (Throwable t) {
+      logger.atSevere().log("Error when deleting key %s", entityVKey);
+      throw t;
    }
  }

--- a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java
+++ b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java
@@ -23,9 +23,9 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import com.google.appengine.api.datastore.DatastoreService;
 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.EntityTranslator;
-import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
-import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.cloud.storage.BlobInfo;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.PeekingIterator;
 import com.google.common.collect.Streams;
@@ -33,7 +33,9 @@ import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.Result;
 import com.googlecode.objectify.util.ResultNow;
+import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
+import google.registry.gcs.GcsUtils;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.CommitLogCheckpoint;
@@ -46,10 +48,10 @@ import google.registry.request.auth.Auth;
 import google.registry.util.Retrier;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.channels.Channels;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Stream;
 import javax.inject.Inject;
@@ -66,45 +68,57 @@ public class RestoreCommitLogsAction implements Runnable {

  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

-  static final int BLOCK_SIZE = 1024 * 1024;  // Buffer 1mb at a time, for no particular reason.
-
  public static final String PATH = "/_dr/task/restoreCommitLogs";
  static final String DRY_RUN_PARAM = "dryRun";
  static final String FROM_TIME_PARAM = "fromTime";
  static final String TO_TIME_PARAM = "toTime";
+  static final String BUCKET_OVERRIDE_PARAM = "gcsBucket";
+
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
+
+  @Inject GcsUtils gcsUtils;

-  @Inject GcsService gcsService;
  @Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
  @Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
  @Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
+
+  @Inject
+  @Parameter(BUCKET_OVERRIDE_PARAM)
+  Optional<String> gcsBucketOverride;
+
  @Inject DatastoreService datastoreService;
  @Inject GcsDiffFileLister diffLister;
+
+  @Inject
+  @Config("commitLogGcsBucket")
+  String defaultGcsBucket;
+
  @Inject Retrier retrier;
  @Inject RestoreCommitLogsAction() {}

  @Override
  public void run() {
    checkArgument(
-        RegistryEnvironment.get() == RegistryEnvironment.ALPHA
-            || RegistryEnvironment.get() == RegistryEnvironment.CRASH
-            || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
-        "DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
+        !FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get()),
+        "DO NOT RUN IN PRODUCTION OR SANDBOX.");
    if (dryRun) {
      logger.atInfo().log("Running in dryRun mode");
    }
-    List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
+    String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
+    logger.atInfo().log("Restoring from %s.", gcsBucket);
+    List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
    if (diffFiles.isEmpty()) {
      logger.atInfo().log("Nothing to restore");
      return;
    }
    Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
    CommitLogCheckpoint lastCheckpoint = null;
-    for (GcsFileMetadata metadata : diffFiles) {
-      logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
-      try (InputStream input = Channels.newInputStream(
-          gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+    for (BlobInfo metadata : diffFiles) {
+      logger.atInfo().log("Restoring: %s", metadata.getName());
+      try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
        PeekingIterator<ImmutableObject> commitLogs =
-            peekingIterator(createDeserializingIterator(input));
+            peekingIterator(createDeserializingIterator(input, true));
        lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
        saveOfy(ImmutableList.of(lastCheckpoint));  // Save the checkpoint itself.
        while (commitLogs.hasNext()) {
--- a/core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java
+++ b/core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java
@@ -28,10 +28,10 @@ import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
-import google.registry.schema.domain.RegistryLock;
 import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.Retrier;
 import javax.inject.Inject;
--- a/core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java
@@ -370,11 +370,10 @@ public class DeleteContactsAndHostsAction implements Runnable {
                      : "it was transferred prior to deletion");

      HistoryEntry historyEntry =
-          new HistoryEntry.Builder()
+          HistoryEntry.createBuilderForResource(resource)
              .setClientId(deletionRequest.requestingClientId())
              .setModificationTime(now)
              .setType(getHistoryEntryType(resource, deleteAllowed))
-              .setParent(deletionRequest.key())
              .build();

      PollMessage.OneTime pollMessage =
@@ -409,7 +408,9 @@ public class DeleteContactsAndHostsAction implements Runnable {
      } else {
        resourceToSave = resource.asBuilder().removeStatusValue(PENDING_DELETE).build();
      }
-      auditedOfy().save().<ImmutableObject>entities(resourceToSave, historyEntry, pollMessage);
+      auditedOfy()
+          .save()
+          .<ImmutableObject>entities(resourceToSave, historyEntry.asHistoryEntry(), pollMessage);
      return DeletionResult.create(
          deleteAllowed ? Type.DELETED : Type.NOT_DELETED, pollMessageText);
    }
--- a/core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java
@@ -17,8 +17,8 @@ package google.registry.batch;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.flows.FlowUtils.marshalWithLenientRetry;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -36,6 +36,7 @@ import google.registry.flows.StatelessRequestSessionMetadata;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.ProtocolDefinition;
 import google.registry.model.eppoutput.EppOutput;
+import google.registry.persistence.transaction.QueryComposer.Comparator;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
@@ -128,12 +129,15 @@ public class DeleteExpiredDomainsAction implements Runnable {
    logger.atInfo().log(
        "Deleting non-renewing domains with autorenew end times up through %s.", runTime);

-    // Note: This query is (and must be) non-transactional, and thus, is only eventually consistent.
+    // Note: in Datastore, this query is (and must be) non-transactional, and thus, is only
+    // eventually consistent.
    ImmutableList<DomainBase> domainsToDelete =
-        ofy().load().type(DomainBase.class).filter("autorenewEndTime <=", runTime).list().stream()
-            // Datastore can't do two inequalities in one query, so the second happens in-memory.
-            .filter(d -> d.getDeletionTime().isEqual(END_OF_TIME))
-            .collect(toImmutableList());
+        transactIfJpaTm(
+            () ->
+                tm().createQueryComposer(DomainBase.class)
+                    .where("autorenewEndTime", Comparator.LTE, runTime)
+                    .where("deletionTime", Comparator.EQ, END_OF_TIME)
+                    .list());
    if (domainsToDelete.isEmpty()) {
      logger.atInfo().log("Found 0 domains to delete.");
      response.setPayload("Found 0 domains to delete.");
--- a/core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteLoadTestDataAction.java
@@ -15,12 +15,14 @@
 package google.registry.batch;

 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryEnvironment.PRODUCTION;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;

 import com.google.appengine.tools.mapreduce.Mapper;
 import com.google.common.collect.ImmutableList;
@@ -28,16 +30,24 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryEnvironment;
+import google.registry.flows.poll.PollFlowUtils;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.model.EppResource;
+import google.registry.model.EppResourceUtils;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.ForeignKeyIndex;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
 import java.util.List;
 import javax.inject.Inject;

@@ -46,8 +56,8 @@ import javax.inject.Inject;
 * the associated ForeignKey and EppResourceIndex entities.
 *
 * <p>This only deletes contacts and hosts, NOT domains. To delete domains, use {@link
- * DeleteLoadTestDataAction} and pass it the TLD(s) that the load test domains were created on. Note
- * that DeleteLoadTestDataAction is safe enough to run in production whereas this mapreduce is not,
+ * DeleteProberDataAction} and pass it the TLD(s) that the load test domains were created on. Note
+ * that DeleteProberDataAction is safe enough to run in production whereas this mapreduce is not,
 * but this one does not need to be runnable in production because load testing isn't run against
 * production.
 */
@@ -68,15 +78,22 @@ public class DeleteLoadTestDataAction implements Runnable {
   */
  private static final ImmutableSet<String> LOAD_TEST_REGISTRARS = ImmutableSet.of("proxy");

-  @Inject
-  @Parameter(PARAM_DRY_RUN)
-  boolean isDryRun;
-
-  @Inject MapreduceRunner mrRunner;
-  @Inject Response response;
+  private final boolean isDryRun;
+  private final MapreduceRunner mrRunner;
+  private final Response response;
+  private final Clock clock;

  @Inject
-  DeleteLoadTestDataAction() {}
+  DeleteLoadTestDataAction(
+      @Parameter(PARAM_DRY_RUN) boolean isDryRun,
+      MapreduceRunner mrRunner,
+      Response response,
+      Clock clock) {
+    this.isDryRun = isDryRun;
+    this.mrRunner = mrRunner;
+    this.response = response;
+    this.clock = clock;
+  }

  @Override
  public void run() {
@@ -87,14 +104,85 @@ public class DeleteLoadTestDataAction implements Runnable {
        !RegistryEnvironment.get().equals(PRODUCTION),
        "This mapreduce is not safe to run on PRODUCTION.");

-    mrRunner
-        .setJobName("Delete load test data")
-        .setModuleName("backend")
-        .runMapOnly(
-            new DeleteLoadTestDataMapper(isDryRun),
-            ImmutableList.of(
-                createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
-        .sendLinkToMapreduceConsole(response);
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Delete load test data")
+          .setModuleName("backend")
+          .runMapOnly(
+              new DeleteLoadTestDataMapper(isDryRun),
+              ImmutableList.of(
+                  createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      tm().transact(
+              () -> {
+                LOAD_TEST_REGISTRARS.forEach(this::deletePollMessages);
+                tm().loadAllOfStream(ContactResource.class).forEach(this::deleteContact);
+                tm().loadAllOfStream(HostResource.class).forEach(this::deleteHost);
+              });
+    }
+  }
+
+  private void deletePollMessages(String registrarId) {
+    ImmutableList<PollMessage> pollMessages =
+        PollFlowUtils.createPollMessageQuery(registrarId, END_OF_TIME).list();
+    if (isDryRun) {
+      logger.atInfo().log(
+          "Would delete %d poll messages for registrar %s.", pollMessages.size(), registrarId);
+    } else {
+      pollMessages.forEach(tm()::delete);
+    }
+  }
+
+  private void deleteContact(ContactResource contact) {
+    if (!LOAD_TEST_REGISTRARS.contains(contact.getPersistedCurrentSponsorClientId())) {
+      return;
+    }
+    // We cannot remove contacts from domains in the general case, so we cannot delete contacts
+    // that are linked to domains (since it would break the foreign keys)
+    if (EppResourceUtils.isLinked(contact.createVKey(), clock.nowUtc())) {
+      logger.atWarning().log(
+          "Cannot delete contact with repo ID %s since it is referenced from a domain",
+          contact.getRepoId());
+      return;
+    }
+    deleteResource(contact);
+  }
+
+  private void deleteHost(HostResource host) {
+    if (!LOAD_TEST_REGISTRARS.contains(host.getPersistedCurrentSponsorClientId())) {
+      return;
+    }
+    VKey<HostResource> hostVKey = host.createVKey();
+    // We can remove hosts from linked domains, so we should do so then delete the hosts
+    ImmutableSet<VKey<DomainBase>> linkedDomains =
+        EppResourceUtils.getLinkedDomainKeys(hostVKey, clock.nowUtc(), null);
+    tm().loadByKeys(linkedDomains)
+        .values()
+        .forEach(
+            domain -> {
+              ImmutableSet<VKey<HostResource>> remainingHosts =
+                  domain.getNsHosts().stream()
+                      .filter(vkey -> !vkey.equals(hostVKey))
+                      .collect(toImmutableSet());
+              tm().put(domain.asBuilder().setNameservers(remainingHosts).build());
+            });
+    deleteResource(host);
+  }
+
+  private void deleteResource(EppResource eppResource) {
+    // In SQL, the only objects parented on the resource are poll messages (deleted above) and
+    // history objects.
+    ImmutableList<HistoryEntry> historyObjects =
+        HistoryEntryDao.loadHistoryObjectsForResource(eppResource.createVKey());
+    if (isDryRun) {
+      logger.atInfo().log(
+          "Would delete repo ID %s along with %d history objects",
+          eppResource.getRepoId(), historyObjects.size());
+    } else {
+      historyObjects.forEach(tm()::delete);
+      tm().delete(eppResource);
+    }
  }

  /** Provides the map method that runs for each existing contact and host entity. */
--- a/core/src/main/java/google/registry/batch/DeleteProberDataAction.java
+++ b/core/src/main/java/google/registry/batch/DeleteProberDataAction.java
@@ -21,8 +21,8 @@ import static google.registry.config.RegistryEnvironment.PRODUCTION;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.model.registry.Registries.getTldsOfType;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_DELETE;
+import static google.registry.model.tld.Registries.getTldsOfType;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLDS;
@@ -44,11 +44,11 @@ import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResourceUtils;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.index.ForeignKeyIndex;
-import google.registry.model.registry.Registry;
-import google.registry.model.registry.Registry.TldType;
-import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import google.registry.model.tld.Registry.TldType;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -253,9 +253,9 @@ public class DeleteProberDataAction implements Runnable {
                        .setDeletionTime(tm().getTransactionTime())
                        .setStatusValues(null)
                        .build();
-                HistoryEntry historyEntry =
-                    new HistoryEntry.Builder()
-                        .setParent(domain)
+                DomainHistory historyEntry =
+                    new DomainHistory.Builder()
+                        .setDomain(domain)
                        .setType(DOMAIN_DELETE)
                        .setModificationTime(tm().getTransactionTime())
                        .setBySuperuser(true)
@@ -263,11 +263,9 @@ public class DeleteProberDataAction implements Runnable {
                        .setClientId(registryAdminClientId)
                        .build();
                // Note that we don't bother handling grace periods, billing events, pending
-                // transfers,
-                // poll messages, or auto-renews because these will all be hard-deleted the next
-                // time the
-                // mapreduce runs anyway.
-                auditedOfy().save().entities(deletedDomain, historyEntry);
+                // transfers, poll messages, or auto-renews because these will all be hard-deleted
+                // the next time the mapreduce runs anyway.
+                tm().putAll(deletedDomain, historyEntry);
                updateForeignKeyIndexDeletionTime(deletedDomain);
                dnsQueue.addDomainRefreshTask(deletedDomain.getDomainName());
              });
--- a/core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java
+++ b/core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java
@@ -21,9 +21,12 @@ import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createChildEntityInput;
 import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING;
 import static google.registry.model.domain.Period.Unit.YEARS;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
+import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
 import static google.registry.util.CollectionUtils.union;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -38,10 +41,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Range;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.NullInput;
-import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
@@ -49,11 +50,12 @@ import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.common.Cursor;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.Period;
-import google.registry.model.registry.Registry;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
-import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -91,31 +93,88 @@ public class ExpandRecurringBillingEventsAction implements Runnable {

  @Override
  public void run() {
-    Cursor cursor =
-        tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING)).orElse(null);
    DateTime executeTime = clock.nowUtc();
-    DateTime persistedCursorTime = (cursor == null ? START_OF_TIME : cursor.getCursorTime());
+    DateTime persistedCursorTime =
+        transactIfJpaTm(
+            () ->
+                tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
+                    .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                    .getCursorTime());
    DateTime cursorTime = cursorTimeParam.orElse(persistedCursorTime);
    checkArgument(
-        cursorTime.isBefore(executeTime),
-        "Cursor time must be earlier than execution time.");
+        cursorTime.isBefore(executeTime), "Cursor time must be earlier than execution time.");
    logger.atInfo().log(
        "Running Recurring billing event expansion for billing time range [%s, %s).",
        cursorTime, executeTime);
-    mrRunner
-        .setJobName("Expand Recurring billing events into synthetic OneTime events.")
-        .setModuleName("backend")
-        .runMapreduce(
-            new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
-            new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
-            // Add an extra shard that maps over a null recurring event (see the mapper for why).
-            ImmutableList.of(
-                new NullInput<>(),
-                createChildEntityInput(
-                    ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
-        .sendLinkToMapreduceConsole(response);
-  }
+    if (tm().isOfy()) {
+      mrRunner
+          .setJobName("Expand Recurring billing events into synthetic OneTime events.")
+          .setModuleName("backend")
+          .runMapreduce(
+              new ExpandRecurringBillingEventsMapper(isDryRun, cursorTime, clock.nowUtc()),
+              new ExpandRecurringBillingEventsReducer(isDryRun, persistedCursorTime),
+              // Add an extra shard that maps over a null recurring event (see the mapper for why).
+              ImmutableList.of(
+                  new NullInput<>(),
+                  createChildEntityInput(
+                      ImmutableSet.of(DomainBase.class), ImmutableSet.of(Recurring.class))))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      int numBillingEventsSaved =
+          jpaTm()
+              .transact(
+                  () ->
+                      jpaTm()
+                          .query(
+                              "FROM BillingRecurrence "
+                                  + "WHERE eventTime <= :executeTime "
+                                  + "AND eventTime < recurrenceEndTime "
+                                  + "ORDER BY id ASC",
+                              Recurring.class)
+                          .setParameter("executeTime", executeTime)
+                          // Need to get a list from the transaction and then convert it to a stream
+                          // for further processing. If we get a stream directly, each elements gets
+                          // processed downstream eagerly but Hibernate returns a
+                          // ScrollableResultsIterator that cannot be advanced outside the
+                          // transaction, resulting in an exception.
+                          .getResultList())
+              .stream()
+              .map(
+                  recurring ->
+                      jpaTm()
+                          .transact(
+                              () ->
+                                  expandBillingEvent(recurring, executeTime, cursorTime, isDryRun)))
+              .reduce(0, Integer::sum);

+      if (!isDryRun) {
+        logger.atInfo().log("Saved OneTime billing events", numBillingEventsSaved);
+      } else {
+        logger.atInfo().log("Generated OneTime billing events (dry run)", numBillingEventsSaved);
+      }
+      logger.atInfo().log(
+          "Recurring event expansion %s complete for billing event range [%s, %s).",
+          isDryRun ? "(dry run) " : "", cursorTime, executeTime);
+      tm().transact(
+              () -> {
+                // Check for the unlikely scenario where the cursor has been altered during the
+                // expansion.
+                DateTime currentCursorTime =
+                    tm().loadByKeyIfPresent(Cursor.createGlobalVKey(RECURRING_BILLING))
+                        .orElse(Cursor.createGlobal(RECURRING_BILLING, START_OF_TIME))
+                        .getCursorTime();
+                if (!currentCursorTime.equals(persistedCursorTime)) {
+                  throw new IllegalStateException(
+                      String.format(
+                          "Current cursor position %s does not match persisted cursor position %s.",
+                          currentCursorTime, persistedCursorTime));
+                }
+                if (!isDryRun) {
+                  tm().put(Cursor.createGlobal(RECURRING_BILLING, executeTime));
+                }
+              });
+    }
+  }
  /** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */
  public static class ExpandRecurringBillingEventsMapper
      extends Mapper<Recurring, DateTime, DateTime> {
@@ -154,98 +213,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
      try {
        numBillingEventsSaved =
            tm().transactNew(
-                    () -> {
-                      ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder =
-                          new ImmutableSet.Builder<>();
-                      final Registry tld =
-                          Registry.get(getTldFromDomainName(recurring.getTargetId()));
-
-                      // Determine the complete set of times at which this recurring event should
-                      // occur (up to and including the runtime of the mapreduce).
-                      Iterable<DateTime> eventTimes =
-                          recurring
-                              .getRecurrenceTimeOfYear()
-                              .getInstancesInRange(
-                                  Range.closed(
-                                      recurring.getEventTime(),
-                                      earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
-
-                      // Convert these event times to billing times
-                      final ImmutableSet<DateTime> billingTimes =
-                          getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
-
-                      Key<? extends EppResource> domainKey = recurring.getParentKey().getParent();
-                      Iterable<OneTime> oneTimesForDomain =
-                          ofy().load().type(OneTime.class).ancestor(domainKey);
-
-                      // Determine the billing times that already have OneTime events persisted.
-                      ImmutableSet<DateTime> existingBillingTimes =
-                          getExistingBillingTimes(oneTimesForDomain, recurring);
-
-                      ImmutableSet.Builder<HistoryEntry> historyEntriesBuilder =
-                          new ImmutableSet.Builder<>();
-                      // Create synthetic OneTime events for all billing times that do not yet have
-                      // an event persisted.
-                      for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
-                        // Construct a new HistoryEntry that parents over the OneTime
-                        HistoryEntry historyEntry =
-                            new HistoryEntry.Builder()
-                                .setBySuperuser(false)
-                                .setClientId(recurring.getClientId())
-                                .setModificationTime(tm().getTransactionTime())
-                                .setParent(domainKey)
-                                .setPeriod(Period.create(1, YEARS))
-                                .setReason(
-                                    "Domain autorenewal by ExpandRecurringBillingEventsAction")
-                                .setRequestedByRegistrar(false)
-                                .setType(DOMAIN_AUTORENEW)
-                                // Don't write a domain transaction record if the recurrence was
-                                // ended prior to the billing time (i.e. a domain was deleted
-                                // during the autorenew grace period).
-                                .setDomainTransactionRecords(
-                                    recurring.getRecurrenceEndTime().isBefore(billingTime)
-                                        ? ImmutableSet.of()
-                                        : ImmutableSet.of(
-                                            DomainTransactionRecord.create(
-                                                tld.getTldStr(),
-                                                // We report this when the autorenew grace period
-                                                // ends
-                                                billingTime,
-                                                TransactionReportField.netRenewsFieldFromYears(1),
-                                                1)))
-                                .build();
-                        historyEntriesBuilder.add(historyEntry);
-
-                        DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
-                        // Determine the cost for a one-year renewal.
-                        Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
-                        syntheticOneTimesBuilder.add(
-                            new OneTime.Builder()
-                                .setBillingTime(billingTime)
-                                .setClientId(recurring.getClientId())
-                                .setCost(renewCost)
-                                .setEventTime(eventTime)
-                                .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
-                                .setParent(historyEntry)
-                                .setPeriodYears(1)
-                                .setReason(recurring.getReason())
-                                .setSyntheticCreationTime(executeTime)
-                                .setCancellationMatchingBillingEvent(recurring.createVKey())
-                                .setTargetId(recurring.getTargetId())
-                                .build());
-                      }
-                      Set<HistoryEntry> historyEntries = historyEntriesBuilder.build();
-                      Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
-                      if (!isDryRun) {
-                        ImmutableSet<ImmutableObject> entitiesToSave =
-                            new ImmutableSet.Builder<ImmutableObject>()
-                                .addAll(historyEntries)
-                                .addAll(syntheticOneTimes)
-                                .build();
-                        ofy().save().entities(entitiesToSave).now();
-                      }
-                      return syntheticOneTimes.size();
-                    });
+                    () -> expandBillingEvent(recurring, executeTime, cursorTime, isDryRun));
      } catch (Throwable t) {
        getContext().incrementCounter("error: " + t.getClass().getSimpleName());
        getContext().incrementCounter(ERROR_COUNTER);
@@ -257,45 +225,12 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
      if (!isDryRun) {
        getContext().incrementCounter("Saved OneTime billing events", numBillingEventsSaved);
      } else {
-        getContext().incrementCounter(
-            "Generated OneTime billing events (dry run)", numBillingEventsSaved);
+        getContext()
+            .incrementCounter("Generated OneTime billing events (dry run)", numBillingEventsSaved);
      }
    }
-
-    /**
-     * Filters a set of {@link DateTime}s down to event times that are in scope for a particular
-     * mapreduce run, given the cursor time and the mapreduce execution time.
-     */
-    private ImmutableSet<DateTime> getBillingTimesInScope(
-        Iterable<DateTime> eventTimes,
-        DateTime cursorTime,
-        DateTime executeTime,
-        final Registry tld) {
-      return Streams.stream(eventTimes)
-          .map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
-          .filter(Range.closedOpen(cursorTime, executeTime))
-          .collect(toImmutableSet());
-    }
-
-    /**
-     * Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted
-     * for a given recurring billing event.
-     */
-    private ImmutableSet<DateTime> getExistingBillingTimes(
-        Iterable<BillingEvent.OneTime> oneTimesForDomain,
-        final BillingEvent.Recurring recurringEvent) {
-      return Streams.stream(oneTimesForDomain)
-          .filter(
-              billingEvent ->
-                  recurringEvent
-                      .createVKey()
-                      .equals(billingEvent.getCancellationMatchingBillingEvent()))
-          .map(OneTime::getBillingTime)
-          .collect(toImmutableSet());
-    }
  }

-
  /**
   * "Reducer" to advance the cursor after all map jobs have been completed. The NullInput into the
   * mapper will cause the mapper to emit one timestamp pair (current cursor and execution time),
@@ -328,7 +263,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
          isDryRun ? "(dry run) " : "", cursorTime, executionTime);
      tm().transact(
              () -> {
-                Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
+                Cursor cursor =
+                    auditedOfy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
                DateTime currentCursorTime =
                    (cursor == null ? START_OF_TIME : cursor.getCursorTime());
                if (!currentCursorTime.equals(expectedPersistedCursorTime)) {
@@ -343,4 +279,135 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
              });
    }
  }
+
+  private static int expandBillingEvent(
+      Recurring recurring, DateTime executeTime, DateTime cursorTime, boolean isDryRun) {
+    ImmutableSet.Builder<OneTime> syntheticOneTimesBuilder = new ImmutableSet.Builder<>();
+    final Registry tld = Registry.get(getTldFromDomainName(recurring.getTargetId()));
+
+    // Determine the complete set of times at which this recurring event should
+    // occur (up to and including the runtime of the mapreduce).
+    Iterable<DateTime> eventTimes =
+        recurring
+            .getRecurrenceTimeOfYear()
+            .getInstancesInRange(
+                Range.closed(
+                    recurring.getEventTime(),
+                    earliestOf(recurring.getRecurrenceEndTime(), executeTime)));
+
+    // Convert these event times to billing times
+    final ImmutableSet<DateTime> billingTimes =
+        getBillingTimesInScope(eventTimes, cursorTime, executeTime, tld);
+
+    VKey<DomainBase> domainKey =
+        VKey.create(
+            DomainBase.class, recurring.getDomainRepoId(), recurring.getParentKey().getParent());
+    Iterable<OneTime> oneTimesForDomain;
+    if (tm().isOfy()) {
+      oneTimesForDomain = auditedOfy().load().type(OneTime.class).ancestor(domainKey.getOfyKey());
+    } else {
+      oneTimesForDomain =
+          tm().createQueryComposer(OneTime.class)
+              .where("domainRepoId", EQ, recurring.getDomainRepoId())
+              .list();
+    }
+
+    // Determine the billing times that already have OneTime events persisted.
+    ImmutableSet<DateTime> existingBillingTimes =
+        getExistingBillingTimes(oneTimesForDomain, recurring);
+
+    ImmutableSet.Builder<DomainHistory> historyEntriesBuilder = new ImmutableSet.Builder<>();
+    // Create synthetic OneTime events for all billing times that do not yet have
+    // an event persisted.
+    for (DateTime billingTime : difference(billingTimes, existingBillingTimes)) {
+      // Construct a new HistoryEntry that parents over the OneTime
+      DomainHistory historyEntry =
+          new DomainHistory.Builder()
+              .setBySuperuser(false)
+              .setClientId(recurring.getClientId())
+              .setModificationTime(tm().getTransactionTime())
+              .setDomain(tm().loadByKey(domainKey))
+              .setPeriod(Period.create(1, YEARS))
+              .setReason("Domain autorenewal by ExpandRecurringBillingEventsAction")
+              .setRequestedByRegistrar(false)
+              .setType(DOMAIN_AUTORENEW)
+              // Don't write a domain transaction record if the recurrence was
+              // ended prior to the billing time (i.e. a domain was deleted
+              // during the autorenew grace period).
+              .setDomainTransactionRecords(
+                  recurring.getRecurrenceEndTime().isBefore(billingTime)
+                      ? ImmutableSet.of()
+                      : ImmutableSet.of(
+                          DomainTransactionRecord.create(
+                              tld.getTldStr(),
+                              // We report this when the autorenew grace period
+                              // ends
+                              billingTime,
+                              TransactionReportField.netRenewsFieldFromYears(1),
+                              1)))
+              .build();
+      historyEntriesBuilder.add(historyEntry);
+
+      DateTime eventTime = billingTime.minus(tld.getAutoRenewGracePeriodLength());
+      // Determine the cost for a one-year renewal.
+      Money renewCost = getDomainRenewCost(recurring.getTargetId(), eventTime, 1);
+      syntheticOneTimesBuilder.add(
+          new OneTime.Builder()
+              .setBillingTime(billingTime)
+              .setClientId(recurring.getClientId())
+              .setCost(renewCost)
+              .setEventTime(eventTime)
+              .setFlags(union(recurring.getFlags(), Flag.SYNTHETIC))
+              .setParent(historyEntry)
+              .setPeriodYears(1)
+              .setReason(recurring.getReason())
+              .setSyntheticCreationTime(executeTime)
+              .setCancellationMatchingBillingEvent(recurring.createVKey())
+              .setTargetId(recurring.getTargetId())
+              .build());
+    }
+    Set<DomainHistory> historyEntries = historyEntriesBuilder.build();
+    Set<OneTime> syntheticOneTimes = syntheticOneTimesBuilder.build();
+    if (!isDryRun) {
+      ImmutableSet<ImmutableObject> entitiesToSave =
+          new ImmutableSet.Builder<ImmutableObject>()
+              .addAll(historyEntries)
+              .addAll(syntheticOneTimes)
+              .build();
+      tm().putAll(entitiesToSave);
+    }
+    return syntheticOneTimes.size();
+  }
+
+  /**
+   * Filters a set of {@link DateTime}s down to event times that are in scope for a particular
+   * mapreduce run, given the cursor time and the mapreduce execution time.
+   */
+  protected static ImmutableSet<DateTime> getBillingTimesInScope(
+      Iterable<DateTime> eventTimes,
+      DateTime cursorTime,
+      DateTime executeTime,
+      final Registry tld) {
+    return Streams.stream(eventTimes)
+        .map(eventTime -> eventTime.plus(tld.getAutoRenewGracePeriodLength()))
+        .filter(Range.closedOpen(cursorTime, executeTime))
+        .collect(toImmutableSet());
+  }
+
+  /**
+   * Determines an {@link ImmutableSet} of {@link DateTime}s that have already been persisted for a
+   * given recurring billing event.
+   */
+  private static ImmutableSet<DateTime> getExistingBillingTimes(
+      Iterable<BillingEvent.OneTime> oneTimesForDomain,
+      final BillingEvent.Recurring recurringEvent) {
+    return Streams.stream(oneTimesForDomain)
+        .filter(
+            billingEvent ->
+                recurringEvent
+                    .createVKey()
+                    .equals(billingEvent.getCancellationMatchingBillingEvent()))
+        .map(OneTime::getBillingTime)
+        .collect(toImmutableSet());
+  }
 }
--- a/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java
+++ b/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java
@@ -23,6 +23,7 @@ import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
+import static google.registry.model.EppResourceUtils.getLinkedDomainKeys;
 import static google.registry.model.EppResourceUtils.isActive;
 import static google.registry.model.EppResourceUtils.isDeleted;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -69,6 +70,7 @@ import java.util.logging.Level;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 import javax.inject.Named;
+import org.apache.http.HttpStatus;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;

@@ -86,6 +88,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
  @Inject Clock clock;
  @Inject MapreduceRunner mrRunner;
  @Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue;
+
+  @Inject DnsQueue dnsQueue;
  @Inject RequestStatusChecker requestStatusChecker;
  @Inject Response response;
  @Inject Retrier retrier;
@@ -153,7 +157,39 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
    } else {
      logger.atInfo().log(
          "Processing asynchronous DNS refresh for renamed hosts: %s", hostKeys.build());
-      runMapreduce(refreshRequests, lock);
+      if (tm().isOfy()) {
+        runMapreduce(refreshRequests, lock);
+      } else {
+        try {
+          refreshRequests.stream()
+              .flatMap(
+                  request ->
+                      getLinkedDomainKeys(request.hostKey(), request.lastUpdateTime(), null)
+                          .stream())
+              .distinct()
+              .map(domainKey -> tm().transact(() -> tm().loadByKey(domainKey).getDomainName()))
+              .forEach(
+                  domainName -> {
+                    retrier.callWithRetry(
+                        () -> dnsQueue.addDomainRefreshTask(domainName),
+                        TransientFailureException.class);
+                    logger.atInfo().log("Enqueued DNS refresh for domain %s.", domainName);
+                  });
+          deleteTasksWithRetry(
+              refreshRequests,
+              getQueue(QUEUE_ASYNC_HOST_RENAME),
+              asyncTaskMetrics,
+              retrier,
+              OperationResult.SUCCESS);
+        } catch (Throwable t) {
+          String message = "Error refreshing DNS on host rename.";
+          logger.atSevere().withCause(t).log(message);
+          response.setPayload(message);
+          response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR);
+        } finally {
+          lock.get().release();
+        }
+      }
    }
  }

--- a/core/src/main/java/google/registry/batch/RelockDomainAction.java
+++ b/core/src/main/java/google/registry/batch/RelockDomainAction.java
@@ -28,16 +28,16 @@ import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
-import google.registry.model.registry.RegistryLockDao;
+import google.registry.model.tld.RegistryLockDao;
 import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.schema.domain.RegistryLock;
 import google.registry.tools.DomainLockUtils;
 import google.registry.util.DateTimeUtils;
 import google.registry.util.EmailMessage;
--- a/core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java
+++ b/core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java
@@ -54,6 +54,8 @@ import javax.inject.Inject;
    service = Action.Service.BACKEND,
    path = "/_dr/task/resaveAllEppResources",
    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+// No longer needed in SQL. Subject to future removal.
+@Deprecated
 public class ResaveAllEppResourcesAction implements Runnable {

  @Inject MapreduceRunner mrRunner;
--- a/core/src/main/java/google/registry/batch/WipeOutCloudSqlAction.java
+++ b/core/src/main/java/google/registry/batch/WipeOutCloudSqlAction.java
@@ -22,7 +22,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
 import google.registry.request.Action;
 import google.registry.request.Response;
@@ -48,22 +48,18 @@ import javax.inject.Inject;
 public class WipeOutCloudSqlAction implements Runnable {
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();

-  // As a short-lived class, hardcode allowed projects here instead of using config files.
-  private static final ImmutableSet<String> ALLOWED_PROJECTS =
-      ImmutableSet.of("domain-registry-qa");
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);

-  private final String projectId;
  private final Supplier<Connection> connectionSupplier;
  private final Response response;
  private final Retrier retrier;

  @Inject
  WipeOutCloudSqlAction(
-      @Config("projectId") String projectId,
      @SchemaManagerConnection Supplier<Connection> connectionSupplier,
      Response response,
      Retrier retrier) {
-    this.projectId = projectId;
    this.connectionSupplier = connectionSupplier;
    this.response = response;
    this.retrier = retrier;
@@ -73,9 +69,9 @@ public class WipeOutCloudSqlAction implements Runnable {
  public void run() {
    response.setContentType(PLAIN_TEXT_UTF_8);

-    if (!ALLOWED_PROJECTS.contains(projectId)) {
+    if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
      response.setStatus(SC_FORBIDDEN);
-      response.setPayload("Wipeout is not allowed in " + projectId);
+      response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
      return;
    }

@@ -90,11 +86,11 @@ public class WipeOutCloudSqlAction implements Runnable {
          },
          e -> !(e instanceof SQLException));
      response.setStatus(SC_OK);
-      response.setPayload("Wiped out Cloud SQL in " + projectId);
+      response.setPayload("Wiped out Cloud SQL in " + RegistryEnvironment.get());
    } catch (RuntimeException e) {
      logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
      response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
+      response.setPayload("Failed to wipe out Cloud SQL in " + RegistryEnvironment.get());
    }
  }

--- a/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java
+++ b/core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.request.Action;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
@@ -49,9 +50,8 @@ public class WipeoutDatastoreAction implements Runnable {

  private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";

-  // As a short-lived class, hardcode allowed projects here instead of using config files.
-  private static final ImmutableSet<String> ALLOWED_PROJECTS =
-      ImmutableSet.of("domain-registry-qa");
+  private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
+      ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);

  private final String projectId;
  private final String jobRegion;
@@ -80,9 +80,9 @@ public class WipeoutDatastoreAction implements Runnable {
  public void run() {
    response.setContentType(PLAIN_TEXT_UTF_8);

-    if (!ALLOWED_PROJECTS.contains(projectId)) {
+    if (FORBIDDEN_ENVIRONMENTS.contains(RegistryEnvironment.get())) {
      response.setStatus(SC_FORBIDDEN);
-      response.setPayload("Wipeout is not allowed in " + projectId);
+      response.setPayload("Wipeout is not allowed in " + RegistryEnvironment.get());
      return;
    }

--- a/core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java
+++ b/core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java
@@ -18,17 +18,23 @@ import static com.google.common.base.Verify.verify;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;

 import google.registry.backup.AppEngineEnvironment;
+import google.registry.model.contact.ContactResource;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import java.io.Serializable;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;

-/** Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines. */
+/**
+ * Toy pipeline that demonstrates how to use {@link JpaTransactionManager} in BEAM pipelines.
+ *
+ * <p>This pipeline may also be used as an integration test for {@link RegistryJpaIO.Read} in a
+ * project with realistic data.
+ */
 public class JpaDemoPipeline implements Serializable {

  public static void main(String[] args) {
@@ -38,23 +44,16 @@ public class JpaDemoPipeline implements Serializable {

    Pipeline pipeline = Pipeline.create(options);
    pipeline
-        .apply("Start", Create.of((Void) null))
        .apply(
-            "Generate Elements",
-            ParDo.of(
-                new DoFn<Void, Void>() {
-                  @ProcessElement
-                  public void processElement(OutputReceiver<Void> output) {
-                    for (int i = 0; i < 500; i++) {
-                      output.output(null);
-                    }
-                  }
-                }))
+            "Read contacts",
+            RegistryJpaIO.read(
+                () -> CriteriaQueryBuilder.create(ContactResource.class).build(),
+                ContactResource::getRepoId))
        .apply(
-            "Make Query",
+            "Count Contacts",
            ParDo.of(
-                new DoFn<Void, Void>() {
-                  private Counter counter = Metrics.counter("Demo", "Read");
+                new DoFn<String, Void>() {
+                  private Counter counter = Metrics.counter("Contacts", "Read");

                  @ProcessElement
                  public void processElement() {
--- a/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java
+++ b/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java
@@ -21,13 +21,16 @@ import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
 import google.registry.backup.AppEngineEnvironment;
+import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
 import google.registry.model.ofy.ObjectifyService;
+import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
-import google.registry.persistence.transaction.QueryComposer;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import java.io.Serializable;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ThreadLocalRandom;
+import javax.annotation.Nullable;
 import javax.persistence.criteria.CriteriaQuery;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.SerializableCoder;
@@ -59,30 +62,68 @@ public final class RegistryJpaIO {

  private RegistryJpaIO() {}

-  public static <R> Read<R, R> read(QueryComposerFactory<R> queryFactory) {
-    return Read.<R, R>builder().queryFactory(queryFactory).build();
+  public static <R> Read<R, R> read(CriteriaQuerySupplier<R> query) {
+    return read(query, x -> x);
  }

  public static <R, T> Read<R, T> read(
-      QueryComposerFactory<R> queryFactory, SerializableFunction<R, T> resultMapper) {
-    return Read.<R, T>builder().queryFactory(queryFactory).resultMapper(resultMapper).build();
+      CriteriaQuerySupplier<R> query, SerializableFunction<R, T> resultMapper) {
+    return Read.<R, T>builder().criteriaQuery(query).resultMapper(resultMapper).build();
+  }
+
+  public static <R, T> Read<R, T> read(
+      String sql, boolean nativeQuery, SerializableFunction<R, T> resultMapper) {
+    return read(sql, null, nativeQuery, resultMapper);
+  }
+
+  /**
+   * Returns a {@link Read} connector based on the given {@code jpql} query string.
+   *
+   * <p>User should take care to prevent sql-injection attacks.
+   */
+  public static <R, T> Read<R, T> read(
+      String sql,
+      @Nullable Map<String, Object> parameter,
+      boolean nativeQuery,
+      SerializableFunction<R, T> resultMapper) {
+    Read.Builder<R, T> builder = Read.builder();
+    if (nativeQuery) {
+      builder.nativeQuery(sql, parameter);
+    } else {
+      builder.jpqlQuery(sql, parameter);
+    }
+    return builder.resultMapper(resultMapper).build();
+  }
+
+  public static <R, T> Read<R, T> read(
+      String jpql, Class<R> clazz, SerializableFunction<R, T> resultMapper) {
+    return read(jpql, null, clazz, resultMapper);
+  }
+
+  /**
+   * Returns a {@link Read} connector based on the given {@code jpql} typed query string.
+   *
+   * <p>User should take care to prevent sql-injection attacks.
+   */
+  public static <R, T> Read<R, T> read(
+      String jpql,
+      @Nullable Map<String, Object> parameter,
+      Class<R> clazz,
+      SerializableFunction<R, T> resultMapper) {
+    return Read.<R, T>builder()
+        .jpqlQuery(jpql, clazz, parameter)
+        .resultMapper(resultMapper)
+        .build();
  }

  public static <T> Write<T> write() {
    return Write.<T>builder().build();
  }

-  // TODO(mmuller): Consider detached JpaQueryComposer that works with any JpaTransactionManager
-  // instance, i.e., change composer.buildQuery() to composer.buildQuery(JpaTransactionManager).
-  // This way QueryComposer becomes reusable and serializable (at least with Hibernate), and this
-  // interface would no longer be necessary.
-  public interface QueryComposerFactory<T>
-      extends SerializableFunction<JpaTransactionManager, QueryComposer<T>> {}
-
  /**
-   * A {@link PTransform transform} that executes a JPA {@link CriteriaQuery} and adds the results
-   * to the BEAM pipeline. Users have the option to transform the results before sending them to the
-   * next stages.
+   * A {@link PTransform transform} that transactionally executes a JPA {@link CriteriaQuery} and
+   * adds the results to the BEAM pipeline. Users have the option to transform the results before
+   * sending them to the next stages.
   */
  @AutoValue
  public abstract static class Read<R, T> extends PTransform<PBegin, PCollection<T>> {
@@ -91,12 +132,10 @@ public final class RegistryJpaIO {

    abstract String name();

-    abstract RegistryJpaIO.QueryComposerFactory<R> queryFactory();
+    abstract RegistryQuery<R> query();

    abstract SerializableFunction<R, T> resultMapper();

-    abstract TransactionMode transactionMode();
-
    abstract Coder<T> coder();

    abstract Builder<R, T> toBuilder();
@@ -106,9 +145,7 @@ public final class RegistryJpaIO {
    public PCollection<T> expand(PBegin input) {
      return input
          .apply("Starting " + name(), Create.of((Void) null))
-          .apply(
-              "Run query for " + name(),
-              ParDo.of(new QueryRunner<>(queryFactory(), resultMapper())))
+          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
          .setCoder(coder())
          .apply("Reshuffle", Reshuffle.viaRandomKey());
    }
@@ -121,19 +158,13 @@ public final class RegistryJpaIO {
      return toBuilder().resultMapper(mapper).build();
    }

-    public Read<R, T> withTransactionMode(TransactionMode transactionMode) {
-      return toBuilder().transactionMode(transactionMode).build();
-    }
-
    public Read<R, T> withCoder(Coder<T> coder) {
      return toBuilder().coder(coder).build();
    }

    static <R, T> Builder<R, T> builder() {
-      return new AutoValue_RegistryJpaIO_Read.Builder()
+      return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
          .name(DEFAULT_NAME)
-          .resultMapper(x -> x)
-          .transactionMode(TransactionMode.TRANSACTIONAL)
          .coder(SerializableCoder.of(Serializable.class));
    }

@@ -142,23 +173,37 @@ public final class RegistryJpaIO {

      abstract Builder<R, T> name(String name);

-      abstract Builder<R, T> queryFactory(RegistryJpaIO.QueryComposerFactory<R> queryFactory);
+      abstract Builder<R, T> query(RegistryQuery<R> query);

      abstract Builder<R, T> resultMapper(SerializableFunction<R, T> mapper);

-      abstract Builder<R, T> transactionMode(TransactionMode transactionMode);
-
      abstract Builder<R, T> coder(Coder coder);

      abstract Read<R, T> build();
+
+      Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
+        return query(RegistryQuery.createQuery(criteriaQuery));
+      }
+
+      Builder<R, T> nativeQuery(String sql, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(sql, parameters, true));
+      }
+
+      Builder<R, T> jpqlQuery(String jpql, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(jpql, parameters, false));
+      }
+
+      Builder<R, T> jpqlQuery(String jpql, Class<R> clazz, Map<String, Object> parameters) {
+        return query(RegistryQuery.createQuery(jpql, parameters, clazz));
+      }
    }

    static class QueryRunner<R, T> extends DoFn<Void, T> {
-      private final QueryComposerFactory<R> querySupplier;
+      private final RegistryQuery<R> query;
      private final SerializableFunction<R, T> resultMapper;

-      QueryRunner(QueryComposerFactory<R> querySupplier, SerializableFunction<R, T> resultMapper) {
-        this.querySupplier = querySupplier;
+      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+        this.query = query;
        this.resultMapper = resultMapper;
      }

@@ -170,18 +215,10 @@ public final class RegistryJpaIO {
          // TODO(b/187210388): JpaTransactionManager should support non-transactional query.
          jpaTm()
              .transactNoRetry(
-                  () ->
-                      querySupplier.apply(jpaTm()).stream()
-                          .map(resultMapper::apply)
-                          .forEach(outputReceiver::output));
+                  () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
        }
      }
    }
-
-    public enum TransactionMode {
-      NOT_TRANSACTIONAL,
-      TRANSACTIONAL;
-    }
  }

  /**
@@ -322,17 +359,17 @@ public final class RegistryJpaIO {
    @ProcessElement
    public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ImmutableList<Object> ofyEntities =
+        ImmutableList<Object> entities =
            Streams.stream(kv.getValue())
                .map(this.jpaConverter::apply)
                // TODO(b/177340730): post migration delete the line below.
                .filter(Objects::nonNull)
                .collect(ImmutableList.toImmutableList());
        try {
-          jpaTm().transact(() -> jpaTm().putAll(ofyEntities));
-          counter.inc(ofyEntities.size());
+          jpaTm().transact(() -> jpaTm().putAll(entities));
+          counter.inc(entities.size());
        } catch (RuntimeException e) {
-          processSingly(ofyEntities);
+          processSingly(entities);
        }
      }
    }
@@ -341,19 +378,22 @@ public final class RegistryJpaIO {
     * Writes entities in a failed batch one by one to identify the first bad entity and throws a
     * {@link RuntimeException} on it.
     */
-    private void processSingly(ImmutableList<Object> ofyEntities) {
-      for (Object ofyEntity : ofyEntities) {
+    private void processSingly(ImmutableList<Object> entities) {
+      for (Object entity : entities) {
        try {
-          jpaTm().transact(() -> jpaTm().put(ofyEntity));
+          jpaTm().transact(() -> jpaTm().put(entity));
          counter.inc();
        } catch (RuntimeException e) {
-          throw new RuntimeException(toOfyKey(ofyEntity).toString(), e);
+          throw new RuntimeException(toEntityKeyString(entity), e);
        }
      }
    }

-    private com.googlecode.objectify.Key<?> toOfyKey(Object ofyEntity) {
-      return com.googlecode.objectify.Key.create(ofyEntity);
+    private String toEntityKeyString(Object entity) {
+      if (entity instanceof SqlEntity) {
+        return ((SqlEntity) entity).getPrimaryKeyString();
+      }
+      return "Non-SqlEntity: " + String.valueOf(entity);
    }
  }
 }
--- a/core/src/main/java/google/registry/beam/common/RegistryQuery.java
+++ b/core/src/main/java/google/registry/beam/common/RegistryQuery.java
@@ -0,0 +1,116 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaQuery;
+
+/** Interface for query instances used by {@link RegistryJpaIO.Read}. */
+public interface RegistryQuery<T> extends Serializable {
+  Stream<T> stream();
+
+  interface CriteriaQuerySupplier<T> extends Supplier<CriteriaQuery<T>>, Serializable {}
+
+  /**
+   * Returns a {@link RegistryQuery} that creates a string query from constant text.
+   *
+   * @param nativeQuery whether the given string is to be interpreted as a native query or JPQL.
+   * @param parameters parameters to be substituted in the query.
+   * @param <T> Type of each row in the result set, {@link Object} in single-select queries, and
+   *     {@code Object[]} in multi-select queries.
+   */
+  static <T> RegistryQuery<T> createQuery(
+      String sql, @Nullable Map<String, Object> parameters, boolean nativeQuery) {
+    return () -> {
+      EntityManager entityManager = jpaTm().getEntityManager();
+      Query query =
+          nativeQuery ? entityManager.createNativeQuery(sql) : entityManager.createQuery(sql);
+      if (parameters != null) {
+        parameters.forEach(query::setParameter);
+      }
+      @SuppressWarnings("unchecked")
+      Stream<T> resultStream = query.getResultStream();
+      return nativeQuery ? resultStream : resultStream.map(e -> detach(entityManager, e));
+    };
+  }
+
+  /**
+   * Returns a {@link RegistryQuery} that creates a typed JPQL query from constant text.
+   *
+   * @param parameters parameters to be substituted in the query.
+   * @param <T> Type of each row in the result set.
+   */
+  static <T> RegistryQuery<T> createQuery(
+      String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
+    return () -> {
+      TypedQuery<T> query = jpaTm().query(jpql, clazz);
+      if (parameters != null) {
+        parameters.forEach(query::setParameter);
+      }
+      return query.getResultStream();
+    };
+  }
+
+  /**
+   * Returns a {@link RegistryQuery} from a {@link CriteriaQuery} supplier.
+   *
+   * <p>A serializable supplier is needed in because {@link CriteriaQuery} itself must be created
+   * within a transaction, and we are not in a transaction yet when this function is called to set
+   * up the pipeline.
+   *
+   * @param <T> Type of each row in the result set.
+   */
+  static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
+    return () -> jpaTm().query(criteriaQuery.get()).getResultStream();
+  }
+
+  /**
+   * Removes an object from the JPA session cache if applicable.
+   *
+   * @param object An object that represents a row in the result set. It may be a JPA entity, a
+   *     non-entity object, or an array that holds JPA entities and/or non-entities.
+   */
+  static <T> T detach(EntityManager entityManager, T object) {
+    if (object.getClass().isArray()) {
+      for (Object arrayElement : (Object[]) object) {
+        detachObject(entityManager, arrayElement);
+      }
+    } else {
+      detachObject(entityManager, object);
+    }
+    return object;
+  }
+
+  static void detachObject(EntityManager entityManager, Object object) {
+    Class<?> objectClass = object.getClass();
+    if (objectClass.isPrimitive() || objectClass == String.class) {
+      return;
+    }
+    try {
+      entityManager.detach(object);
+    } catch (IllegalArgumentException e) {
+      // Not an entity. Do nothing.
+    }
+  }
+}
--- a/core/src/main/java/google/registry/beam/initsql/CloudSqlCredentialDecryptor.java
+++ b/core/src/main/java/google/registry/beam/initsql/CloudSqlCredentialDecryptor.java
@@ -1,50 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.initsql;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import com.google.api.services.cloudkms.v1.model.DecryptRequest;
-import com.google.common.base.Strings;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.keyring.kms.KmsConnection;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import javax.inject.Inject;
-
-/**
- * Decrypts data using Cloud KMS, with the same crypto key with which Cloud SQL credential files on
- * GCS was encrypted. See {@link BackupPaths#getCloudSQLCredentialFilePatterns} for more
- * information.
- */
-public class CloudSqlCredentialDecryptor {
-
-  private static final String CRYPTO_KEY_NAME = "nomulus-tool-key";
-  private final KmsConnection kmsConnection;
-
-  @Inject
-  CloudSqlCredentialDecryptor(@Config("beamKmsConnection") KmsConnection kmsConnection) {
-    this.kmsConnection = kmsConnection;
-  }
-
-  public String decrypt(String data) {
-    checkArgument(!Strings.isNullOrEmpty(data), "Null or empty data.");
-    byte[] ciphertext = Base64.getDecoder().decode(data);
-    // Re-encode for Cloud KMS JSON REST API, invoked through kmsConnection.
-    String urlSafeCipherText = new DecryptRequest().encodeCiphertext(ciphertext).getCiphertext();
-    return new String(
-        kmsConnection.decrypt(CRYPTO_KEY_NAME, urlSafeCipherText), StandardCharsets.UTF_8);
-  }
-}
--- a/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java
+++ b/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java
@@ -32,8 +32,8 @@ import google.registry.model.host.HostResource;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
-import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.io.Serializable;
 import java.util.Collection;
--- a/core/src/main/java/google/registry/beam/initsql/SqlAccessInfo.java
+++ b/core/src/main/java/google/registry/beam/initsql/SqlAccessInfo.java
@@ -1,45 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.initsql;
-
-import com.google.auto.value.AutoValue;
-import java.util.Optional;
-
-/**
- * Information needed to connect to a database, including JDBC URL, user name, password, and in the
- * case of Cloud SQL, the database instance's name.
- */
-@AutoValue
-abstract class SqlAccessInfo {
-
-  abstract String jdbcUrl();
-
-  abstract String user();
-
-  abstract String password();
-
-  abstract Optional<String> cloudSqlInstanceName();
-
-  public static SqlAccessInfo createCloudSqlAccessInfo(
-      String sqlInstanceName, String username, String password) {
-    return new AutoValue_SqlAccessInfo(
-        "jdbc:postgresql://google/postgres", username, password, Optional.of(sqlInstanceName));
-  }
-
-  public static SqlAccessInfo createLocalSqlAccessInfo(
-      String jdbcUrl, String username, String password) {
-    return new AutoValue_SqlAccessInfo(jdbcUrl, username, password, Optional.empty());
-  }
-}
--- a/core/src/main/java/google/registry/beam/initsql/Transforms.java
+++ b/core/src/main/java/google/registry/beam/initsql/Transforms.java
@@ -39,9 +39,9 @@ import google.registry.backup.VersionedEntity;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.replay.DatastoreAndSqlEntity;
+import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.schema.replay.DatastoreAndSqlEntity;
-import google.registry.schema.replay.SqlEntity;
 import google.registry.tools.LevelDbLogReader;
 import java.io.Serializable;
 import java.util.Collection;
--- a/core/src/main/java/google/registry/beam/invoicing/BillingEvent.java
+++ b/core/src/main/java/google/registry/beam/invoicing/BillingEvent.java
@@ -251,7 +251,14 @@ public abstract class BillingEvent implements Serializable {
  InvoiceGroupingKey getInvoiceGroupingKey() {
    return new AutoValue_BillingEvent_InvoiceGroupingKey(
        billingTime().toLocalDate().withDayOfMonth(1).toString(),
-        billingTime().toLocalDate().withDayOfMonth(1).plusYears(years()).minusDays(1).toString(),
+        years() == 0
+            ? ""
+            : billingTime()
+                .toLocalDate()
+                .withDayOfMonth(1)
+                .plusYears(years())
+                .minusDays(1)
+                .toString(),
        billingId(),
        String.format("%s - %s", registrarId(), tld()),
        String.format("%s | TLD: %s | TERM: %d-year", action(), tld(), years()),
--- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java
+++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java
@@ -14,18 +14,29 @@

 package google.registry.beam.invoicing;

+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;

+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryJpaIO.Read;
 import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey;
 import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
+import google.registry.model.billing.BillingEvent.Flag;
+import google.registry.model.registrar.Registrar;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.reporting.billing.BillingModule;
+import google.registry.util.DateTimeUtils;
+import google.registry.util.DomainNameUtils;
 import google.registry.util.SqlTemplate;
 import java.io.Serializable;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.YearMonth;
+import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Optional;
+import java.util.regex.Pattern;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.SerializableCoder;
@@ -58,6 +69,9 @@ public class InvoicingPipeline implements Serializable {
  private static final DateTimeFormatter TIMESTAMP_FORMATTER =
      DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");

+  private static final Pattern SQL_COMMENT_REGEX =
+      Pattern.compile("^\\s*--.*\\n", Pattern.MULTILINE);
+
  private final InvoicingPipelineOptions options;

  InvoicingPipeline(InvoicingPipelineOptions options) {
@@ -71,21 +85,60 @@ public class InvoicingPipeline implements Serializable {
  }

  void setupPipeline(Pipeline pipeline) {
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
    PCollection<BillingEvent> billingEvents =
-        pipeline.apply(
-            "Read BillingEvents from Bigquery",
-            BigQueryIO.read(BillingEvent::parseFromRecord)
-                .fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
-                .withCoder(SerializableCoder.of(BillingEvent.class))
-                .usingStandardSql()
-                .withoutValidation()
-                .withTemplateCompatibility());
+        options.getDatabase().equals("DATASTORE")
+            ? readFromBigQuery(options, pipeline)
+            : readFromCloudSql(options, pipeline);

    saveInvoiceCsv(billingEvents, options);

    saveDetailedCsv(billingEvents, options);
  }

+  static PCollection<BillingEvent> readFromBigQuery(
+      InvoicingPipelineOptions options, Pipeline pipeline) {
+    return pipeline.apply(
+        "Read BillingEvents from Bigquery",
+        BigQueryIO.read(BillingEvent::parseFromRecord)
+            .fromQuery(makeQuery(options.getYearMonth(), options.getProject()))
+            .withCoder(SerializableCoder.of(BillingEvent.class))
+            .usingStandardSql()
+            .withoutValidation()
+            .withTemplateCompatibility());
+  }
+
+  static PCollection<BillingEvent> readFromCloudSql(
+      InvoicingPipelineOptions options, Pipeline pipeline) {
+    Read<Object[], BillingEvent> read =
+        RegistryJpaIO.read(
+            makeCloudSqlQuery(options.getYearMonth()), false, InvoicingPipeline::parseRow);
+
+    return pipeline.apply("Read BillingEvents from Cloud SQL", read);
+  }
+
+  private static BillingEvent parseRow(Object[] row) {
+    google.registry.model.billing.BillingEvent.OneTime oneTime =
+        (google.registry.model.billing.BillingEvent.OneTime) row[0];
+    Registrar registrar = (Registrar) row[1];
+    return BillingEvent.create(
+        oneTime.getId(),
+        DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
+        DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
+        registrar.getClientId(),
+        registrar.getBillingIdentifier().toString(),
+        registrar.getPoNumber().orElse(""),
+        DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
+        oneTime.getReason().toString(),
+        oneTime.getTargetId(),
+        oneTime.getDomainRepoId(),
+        Optional.ofNullable(oneTime.getPeriodYears()).orElse(0),
+        oneTime.getCost().getCurrencyUnit().toString(),
+        oneTime.getCost().getAmount().doubleValue(),
+        String.join(
+            " ", oneTime.getFlags().stream().map(Flag::toString).collect(toImmutableSet())));
+  }
+
  /** Transform that converts a {@code BillingEvent} into an invoice CSV row. */
  private static class GenerateInvoiceRows
      extends PTransform<PCollection<BillingEvent>, PCollection<String>> {
@@ -171,6 +224,21 @@ public class InvoicingPipeline implements Serializable {
        .build();
  }

+  /** Create the Cloud SQL query for a given yearMonth at runtime. */
+  static String makeCloudSqlQuery(String yearMonth) {
+    YearMonth endMonth = YearMonth.parse(yearMonth).plusMonths(1);
+    String queryWithComments =
+        SqlTemplate.create(
+                getQueryFromFile(InvoicingPipeline.class, "cloud_sql_billing_events.sql"))
+            .put("FIRST_TIMESTAMP_OF_MONTH", yearMonth.concat("-01"))
+            .put(
+                "LAST_TIMESTAMP_OF_MONTH",
+                String.format("%d-%d-01", endMonth.getYear(), endMonth.getMonthValue()))
+            .build();
+    // Remove the comments from the query string
+    return SQL_COMMENT_REGEX.matcher(queryWithComments).replaceAll("");
+  }
+
  public static void main(String[] args) {
    PipelineOptionsFactory.register(InvoicingPipelineOptions.class);
    InvoicingPipelineOptions options =
--- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java
+++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java
@@ -30,6 +30,11 @@ public interface InvoicingPipelineOptions extends RegistryPipelineOptions {

  void setInvoiceFilePrefix(String value);

+  @Description("The database to read data from.")
+  String getDatabase();
+
+  void setDatabase(String value);
+
  @Description("The GCS bucket URL for invoices and detailed  reports to be uploaded.")
  String getBillingBucketUrl();

--- a/core/src/main/java/google/registry/beam/rde/RdeIO.java
+++ b/core/src/main/java/google/registry/beam/rde/RdeIO.java
@@ -0,0 +1,273 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.rde;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Verify.verify;
+import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.common.flogger.FluentLogger;
+import google.registry.gcs.GcsUtils;
+import google.registry.keyring.api.PgpHelper;
+import google.registry.model.common.Cursor;
+import google.registry.model.rde.RdeMode;
+import google.registry.model.rde.RdeNamingUtils;
+import google.registry.model.rde.RdeRevision;
+import google.registry.model.tld.Registry;
+import google.registry.rde.DepositFragment;
+import google.registry.rde.Ghostryde;
+import google.registry.rde.PendingDeposit;
+import google.registry.rde.RdeCounter;
+import google.registry.rde.RdeMarshaller;
+import google.registry.rde.RdeResourceType;
+import google.registry.rde.RdeUtil;
+import google.registry.tldconfig.idn.IdnTableEnum;
+import google.registry.xjc.rdeheader.XjcRdeHeader;
+import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
+import google.registry.xml.ValidationMode;
+import google.registry.xml.XmlException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.security.Security;
+import java.util.Optional;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.openpgp.PGPPublicKey;
+import org.joda.time.DateTime;
+
+public class RdeIO {
+
+  @AutoValue
+  abstract static class Write
+      extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
+
+    abstract GcsUtils gcsUtils();
+
+    abstract String rdeBucket();
+
+    // It's OK to return a primitive array because we are only using it to construct the
+    // PGPPublicKey, which is not serializable.
+    @SuppressWarnings("mutable")
+    abstract byte[] stagingKeyBytes();
+
+    abstract ValidationMode validationMode();
+
+    static Builder builder() {
+      return new AutoValue_RdeIO_Write.Builder();
+    }
+
+    @AutoValue.Builder
+    abstract static class Builder {
+      abstract Builder setGcsUtils(GcsUtils gcsUtils);
+
+      abstract Builder setRdeBucket(String value);
+
+      abstract Builder setStagingKeyBytes(byte[] value);
+
+      abstract Builder setValidationMode(ValidationMode value);
+
+      abstract Write build();
+    }
+
+    @Override
+    public PDone expand(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
+      input
+          .apply(
+              "Write to GCS",
+              ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
+          .apply("Update cursors", ParDo.of(new CursorUpdater()));
+      return PDone.in(input.getPipeline());
+    }
+  }
+
+  private static class RdeWriter
+      extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
+
+    private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+    private final GcsUtils gcsUtils;
+    private final String rdeBucket;
+    private final byte[] stagingKeyBytes;
+    private final RdeMarshaller marshaller;
+
+    protected RdeWriter(
+        GcsUtils gcsUtils,
+        String rdeBucket,
+        byte[] stagingKeyBytes,
+        ValidationMode validationMode) {
+      this.gcsUtils = gcsUtils;
+      this.rdeBucket = rdeBucket;
+      this.stagingKeyBytes = stagingKeyBytes;
+      this.marshaller = new RdeMarshaller(validationMode);
+    }
+
+    @Setup
+    public void setup() {
+      Security.addProvider(new BouncyCastleProvider());
+    }
+
+    @ProcessElement
+    public void processElement(
+        @Element KV<PendingDeposit, Iterable<DepositFragment>> kv,
+        PipelineOptions options,
+        OutputReceiver<KV<PendingDeposit, Integer>> outputReceiver) {
+      PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
+      PendingDeposit key = kv.getKey();
+      Iterable<DepositFragment> fragments = kv.getValue();
+      RdeCounter counter = new RdeCounter();
+
+      // Determine some basic things about the deposit.
+      final RdeMode mode = key.mode();
+      final String tld = key.tld();
+      final DateTime watermark = key.watermark();
+      final int revision =
+          Optional.ofNullable(key.revision())
+              .orElseGet(() -> RdeRevision.getNextRevision(tld, watermark, mode));
+      String id = RdeUtil.timestampToId(watermark);
+      String prefix = options.getJobName();
+      String basename = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
+      if (key.manual()) {
+        checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
+        prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
+      } else {
+        prefix = prefix + "/" + basename;
+      }
+      BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
+      // This file will contain the byte length (ASCII) of the raw unencrypted XML.
+      //
+      // This is necessary because RdeUploadAction creates a tar file which requires that the length
+      // be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
+      // length, so we just save it separately.
+      BlobId xmlLengthFilename = BlobId.of(rdeBucket, prefix + ".xml.length");
+      BlobId reportFilename = BlobId.of(rdeBucket, prefix + "-report.xml.ghostryde");
+
+      // These variables will be populated as we write the deposit XML and used for other files.
+      boolean failed = false;
+      XjcRdeHeader header;
+
+      // Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
+
+      logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
+      try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
+          OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
+          OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
+          Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
+
+        // Output the top portion of the XML document.
+        output.write(marshaller.makeHeader(id, watermark, RdeResourceType.getUris(mode), revision));
+
+        // Output XML fragments while counting them.
+        for (DepositFragment fragment : fragments) {
+          if (!fragment.xml().isEmpty()) {
+            output.write(fragment.xml());
+            counter.increment(fragment.type());
+          }
+          if (!fragment.error().isEmpty()) {
+            failed = true;
+            logger.atSevere().log("Fragment error: %s", fragment.error());
+          }
+        }
+
+        // Don't write the IDN elements for BRDA.
+        if (mode == RdeMode.FULL) {
+          for (IdnTableEnum idn : IdnTableEnum.values()) {
+            output.write(marshaller.marshalIdn(idn.getTable()));
+            counter.increment(RdeResourceType.IDN);
+          }
+        }
+
+        // Output XML that says how many resources were emitted.
+        header = counter.makeHeader(tld, mode);
+        output.write(marshaller.marshalOrDie(new XjcRdeHeaderElement(header)));
+
+        // Output the bottom of the XML document.
+        output.write(marshaller.makeFooter());
+
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+
+      // If an entity was broken, abort after writing as much logs/deposit data as possible.
+      verify(!failed, "RDE staging failed for TLD %s", tld);
+
+      // Write a tiny XML file to GCS containing some information about the deposit.
+      //
+      // This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
+      if (mode == RdeMode.FULL) {
+        logger.atInfo().log("Writing %s", reportFilename);
+        try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
+            OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
+          counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
+        } catch (IOException | XmlException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      // Now that we're done, output roll the cursor forward.
+      if (key.manual()) {
+        logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task");
+      } else {
+        outputReceiver.output(KV.of(key, revision));
+      }
+    }
+  }
+
+  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+    private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+    @ProcessElement
+    public void processElement(@Element KV<PendingDeposit, Integer> input) {
+      tm().transact(
+              () -> {
+                PendingDeposit key = input.getKey();
+                int revision = input.getValue();
+                Registry registry = Registry.get(key.tld());
+                Optional<Cursor> cursor =
+                    transactIfJpaTm(
+                        () ->
+                            tm().loadByKeyIfPresent(
+                                    Cursor.createVKey(key.cursor(), registry.getTldStr())));
+                DateTime position = getCursorTimeOrStartOfTime(cursor);
+                checkState(key.interval() != null, "Interval must be present");
+                DateTime newPosition = key.watermark().plus(key.interval());
+                if (!position.isBefore(newPosition)) {
+                  logger.atWarning().log("Cursor has already been rolled forward.");
+                  return;
+                }
+                verify(
+                    position.equals(key.watermark()),
+                    "Partial ordering of RDE deposits broken: %s %s",
+                    position,
+                    key);
+                tm().put(Cursor.create(key.cursor(), newPosition, registry));
+                logger.atInfo().log(
+                    "Rolled forward %s on %s cursor to %s", key.cursor(), key.tld(), newPosition);
+                RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+              });
+    }
+  }
+}
--- a/core/src/main/java/google/registry/beam/rde/RdePipeline.java
+++ b/core/src/main/java/google/registry/beam/rde/RdePipeline.java
@@ -0,0 +1,303 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.rde;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.io.BaseEncoding;
+import dagger.BindsInstance;
+import dagger.Component;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.model.EppResource;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.host.HostResource;
+import google.registry.model.rde.RdeMode;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.Registrar.Type;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
+import google.registry.rde.DepositFragment;
+import google.registry.rde.PendingDeposit;
+import google.registry.rde.PendingDeposit.PendingDepositCoder;
+import google.registry.rde.RdeFragmenter;
+import google.registry.rde.RdeMarshaller;
+import google.registry.xml.ValidationMode;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.persistence.Entity;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.FlatMapElements;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.joda.time.DateTime;
+
+/**
+ * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
+ *
+ * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ *
+ * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
+ *
+ * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
+ *     Flex Templates</a>
+ */
+@Singleton
+public class RdePipeline implements Serializable {
+
+  private final transient RdePipelineOptions options;
+  private final ValidationMode mode;
+  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final String rdeBucket;
+  private final byte[] stagingKeyBytes;
+  private final GcsUtils gcsUtils;
+
+  // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
+  // if sneaks into production we would get an extra signal.
+  private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
+      Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
+
+  private static final String EPP_RESOURCE_QUERY =
+      "SELECT id FROM %entity% "
+          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
+          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
+          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
+
+  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
+    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
+        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
+  }
+
+  @Inject
+  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
+    this.options = options;
+    this.mode = ValidationMode.valueOf(options.getValidationMode());
+    this.pendings = decodePendings(options.getPendings());
+    this.rdeBucket = options.getGcsBucket();
+    this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
+    this.gcsUtils = gcsUtils;
+  }
+
+  PipelineResult run() {
+    Pipeline pipeline = Pipeline.create(options);
+    PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> fragments =
+        createFragments(pipeline);
+    persistData(fragments);
+    return pipeline.run();
+  }
+
+  PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
+    return PCollectionList.of(processRegistrars(pipeline))
+        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
+        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
+        .and(processNonRegistrarEntities(pipeline, HostResource.class))
+        .apply(Flatten.pCollections())
+        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
+        .apply("Group by PendingDeposit", GroupByKey.create());
+  }
+
+  void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
+    input.apply(
+        "Write to GCS and update cursors",
+        RdeIO.Write.builder()
+            .setRdeBucket(rdeBucket)
+            .setGcsUtils(gcsUtils)
+            .setValidationMode(mode)
+            .setStagingKeyBytes(stagingKeyBytes)
+            .build());
+  }
+
+  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    return pipeline
+        .apply(
+            "Read all production Registrar entities",
+            RegistryJpaIO.read(
+                "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
+                ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
+                String.class,
+                // TODO: consider adding coders for entities and pass them directly instead of using
+                // VKeys.
+                id -> VKey.createSql(Registrar.class, id)))
+        .apply(
+            "Marshal Registrar into DepositFragment",
+            FlatMapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (VKey<Registrar> key) -> {
+                      Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
+                      DepositFragment fragment =
+                          new RdeMarshaller(mode).marshalRegistrar(registrar);
+                      return pendings.values().stream()
+                          .map(pending -> KV.of(pending, fragment))
+                          .collect(toImmutableSet());
+                    }));
+  }
+
+  @SuppressWarnings("deprecation") // Reshuffle is still recommended by Dataflow.
+  <T extends EppResource>
+      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
+          Pipeline pipeline, Class<T> clazz) {
+    return createInputs(pipeline, clazz)
+        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
+        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
+        .apply(
+            "Reshuffle KV<PendingDeposit, DepositFragment> of "
+                + clazz.getSimpleName()
+                + " to prevent fusion",
+            Reshuffle.of());
+  }
+
+  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
+    return pipeline.apply(
+        "Read all production " + clazz.getSimpleName() + " entities",
+        RegistryJpaIO.read(
+            createEppResourceQuery(clazz),
+            clazz.equals(DomainBase.class)
+                ? ImmutableMap.of("tlds", pendings.keySet())
+                : ImmutableMap.of(),
+            String.class,
+            // TODO: consider adding coders for entities and pass them directly instead of using
+            // VKeys.
+            x -> VKey.create(clazz, x)));
+  }
+
+  <T extends EppResource>
+      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
+    return FlatMapElements.into(
+            TypeDescriptors.kvs(
+                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
+        .via(
+            (VKey<T> key) -> {
+              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
+              // The set of all TLDs to which this resource should be emitted.
+              ImmutableSet<String> tlds =
+                  clazz.equals(DomainBase.class)
+                      ? ImmutableSet.of(((DomainBase) resource).getTld())
+                      : pendings.keySet();
+              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
+              ImmutableSet<DateTime> dates =
+                  tlds.stream()
+                      .map(pendings::get)
+                      .flatMap(ImmutableSet::stream)
+                      .map(PendingDeposit::watermark)
+                      .collect(toImmutableSet());
+              // Launch asynchronous fetches of point-in-time representations of resource.
+              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
+                  ImmutableMap.copyOf(
+                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
+              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
+              // the result.
+              RdeFragmenter fragmenter =
+                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
+              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
+              for (String tld : tlds) {
+                for (PendingDeposit pending : pendings.get(tld)) {
+                  // Hosts and contacts don't get included in BRDA deposits.
+                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
+                    continue;
+                  }
+                  Optional<DepositFragment> fragment =
+                      fragmenter.marshal(pending.watermark(), pending.mode());
+                  fragment.ifPresent(
+                      depositFragment -> results.add(KV.of(pending, depositFragment)));
+                }
+              }
+              return results;
+            });
+  }
+
+  /**
+   * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
+   * the original TLD to pending deposit map.
+   */
+  @SuppressWarnings("unchecked")
+  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+    try (ObjectInputStream ois =
+        new ObjectInputStream(
+            new ByteArrayInputStream(
+                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
+      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+    } catch (IOException | ClassNotFoundException e) {
+      throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
+    }
+  }
+
+  /**
+   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
+   * worker by the pipeline launcher as a pipeline option.
+   */
+  static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+      throws IOException {
+    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+      ObjectOutputStream oos = new ObjectOutputStream(baos);
+      oos.writeObject(pendings);
+      oos.flush();
+      return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
+    }
+  }
+
+  public static void main(String[] args) throws IOException, ClassNotFoundException {
+    PipelineOptionsFactory.register(RdePipelineOptions.class);
+    RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
+    DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
+  }
+
+  @Singleton
+  @Component(modules = {CredentialModule.class, ConfigModule.class})
+  interface RdePipelineComponent {
+    RdePipeline rdePipeline();
+
+    @Component.Builder
+    interface Builder {
+      @BindsInstance
+      Builder options(RdePipelineOptions options);
+
+      RdePipelineComponent build();
+    }
+  }
+}
--- a/core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java
+++ b/core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java
@@ -0,0 +1,42 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.rde;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import org.apache.beam.sdk.options.Description;
+
+/** Custom options for running the spec11 pipeline. */
+public interface RdePipelineOptions extends RegistryPipelineOptions {
+
+  @Description("The Base64-encoded serialized map of TLDs to PendingDeposit.")
+  String getPendings();
+
+  void setPendings(String value);
+
+  @Description("The validation mode (LENIENT|STRICT) that the RDE marshaller uses.")
+  String getValidationMode();
+
+  void setValidationMode(String value);
+
+  @Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
+  String getGcsBucket();
+
+  void setGcsBucket(String value);
+
+  @Description("The Base64-encoded PGP public key to encrypt the deposits.")
+  String getStagingKey();
+
+  void setStagingKey(String value);
+}
--- a/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java
+++ b/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java
@@ -23,10 +23,13 @@ import dagger.Component;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryJpaIO.Read;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
 import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.reporting.Spec11ThreatMatch;
 import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
 import google.registry.util.UtilsModule;
@@ -96,21 +99,11 @@ public class Spec11Pipeline implements Serializable {
  }

  void setupPipeline(Pipeline pipeline) {
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
    PCollection<Subdomain> domains =
-        pipeline.apply(
-            "Read active domains from BigQuery",
-            BigQueryIO.read(Subdomain::parseFromRecord)
-                .fromQuery(
-                    SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
-                        .put("PROJECT_ID", options.getProject())
-                        .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
-                        .put("REGISTRAR_TABLE", "Registrar")
-                        .put("DOMAIN_BASE_TABLE", "DomainBase")
-                        .build())
-                .withCoder(SerializableCoder.of(Subdomain.class))
-                .usingStandardSql()
-                .withoutValidation()
-                .withTemplateCompatibility());
+        options.getDatabase().equals("DATASTORE")
+            ? readFromBigQuery(options, pipeline)
+            : readFromCloudSql(pipeline);

    PCollection<KV<Subdomain, ThreatMatch>> threatMatches =
        domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn));
@@ -119,6 +112,48 @@ public class Spec11Pipeline implements Serializable {
    saveToGcs(threatMatches, options);
  }

+  static PCollection<Subdomain> readFromCloudSql(Pipeline pipeline) {
+    Read<Object[], Subdomain> read =
+        RegistryJpaIO.read(
+            "select d, r.emailAddress from Domain d join Registrar r on"
+                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
+                + " and d.deletionTime > now()",
+            false,
+            Spec11Pipeline::parseRow);
+
+    return pipeline.apply("Read active domains from Cloud SQL", read);
+  }
+
+  static PCollection<Subdomain> readFromBigQuery(Spec11PipelineOptions options, Pipeline pipeline) {
+    return pipeline.apply(
+        "Read active domains from BigQuery",
+        BigQueryIO.read(Subdomain::parseFromRecord)
+            .fromQuery(
+                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
+                    .put("PROJECT_ID", options.getProject())
+                    .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
+                    .put("REGISTRAR_TABLE", "Registrar")
+                    .put("DOMAIN_BASE_TABLE", "DomainBase")
+                    .build())
+            .withCoder(SerializableCoder.of(Subdomain.class))
+            .usingStandardSql()
+            .withoutValidation()
+            .withTemplateCompatibility());
+  }
+
+  private static Subdomain parseRow(Object[] row) {
+    DomainBase domainBase = (DomainBase) row[0];
+    String emailAddress = (String) row[1];
+    if (emailAddress == null) {
+      emailAddress = "";
+    }
+    return Subdomain.create(
+        domainBase.getDomainName(),
+        domainBase.getRepoId(),
+        domainBase.getCurrentSponsorClientId(),
+        emailAddress);
+  }
+
  static void saveToSql(
      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
    String transformId = "Spec11 Threat Matches";
--- a/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java
+++ b/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java
@@ -34,4 +34,9 @@ public interface Spec11PipelineOptions extends RegistryPipelineOptions {
  String getReportingBucketUrl();

  void setReportingBucketUrl(String value);
+
+  @Description("The database to read data from.")
+  String getDatabase();
+
+  void setDatabase(String value);
 }
--- a/core/src/main/java/google/registry/config/RegistryConfig.java
+++ b/core/src/main/java/google/registry/config/RegistryConfig.java
@@ -60,7 +60,7 @@ import org.joda.time.Duration;
 * <p>This class does not represent the total configuration of the Nomulus service. It's <b>only
 * meant for settings that need to be configured <i>once</i></b>. Settings which may be subject to
 * change in the future, should instead be retrieved from Datastore. The {@link
- * google.registry.model.registry.Registry Registry} class is one such example of this.
+ * google.registry.model.tld.Registry Registry} class is one such example of this.
 *
 * <p>Note: Only settings that are actually configurable belong in this file. It's not a catch-all
 * for anything widely used throughout the code base.
@@ -203,7 +203,7 @@ public final class RegistryConfig {
     * Configuration for analytics services installed in the web console.
     *
     * @see google.registry.ui.server.registrar.ConsoleUiAction
-     * @see google.registry.ui.soy.AnalyticsSoyInfo
+     * @see google.registry.ui.soy.registrar.AnalyticsSoyInfo
     */
    @Provides
    @Config("analyticsConfig")
@@ -416,17 +416,6 @@ public final class RegistryConfig {
      return Optional.ofNullable(config.cloudDns.servicePath);
    }

-    /**
-     * Returns size of Google Cloud Storage client connection buffer in bytes.
-     *
-     * @see google.registry.gcs.GcsUtils
-     */
-    @Provides
-    @Config("gcsBufferSize")
-    public static int provideGcsBufferSize() {
-      return 1024 * 1024;
-    }
-
    /**
     * Returns the email address of the admin account on the G Suite app used to perform
     * administrative actions.
@@ -1278,16 +1267,34 @@ public final class RegistryConfig {

    @Provides
    @Config("expirationWarningDays")
-    public static int provideDaysToExpiration(RegistryConfigSettings config) {
+    public static int provideExpirationWarningDays(RegistryConfigSettings config) {
      return config.sslCertificateValidation.expirationWarningDays;
    }

+    @Provides
+    @Config("expirationWarningIntervalDays")
+    public static int provideExpirationWarningIntervalDays(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningIntervalDays;
+    }
+
    @Provides
    @Config("minimumRsaKeyLength")
    public static int provideMinimumRsaKeyLength(RegistryConfigSettings config) {
      return config.sslCertificateValidation.minimumRsaKeyLength;
    }

+    @Provides
+    @Config("expirationWarningEmailBodyText")
+    public static String provideExpirationWarningEmailBodyText(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningEmailBodyText;
+    }
+
+    @Provides
+    @Config("expirationWarningEmailSubjectText")
+    public static String provideExpirationWarningEmailSubjectText(RegistryConfigSettings config) {
+      return config.sslCertificateValidation.expirationWarningEmailSubjectText;
+    }
+
    @Provides
    @Config("allowedEcdsaCurves")
    public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
@@ -1387,8 +1394,8 @@ public final class RegistryConfig {
  /**
   * Returns the amount of time a domain label list should be cached in memory before expiring.
   *
-   * @see google.registry.model.registry.label.ReservedList
-   * @see google.registry.model.registry.label.PremiumList
+   * @see google.registry.model.tld.label.ReservedList
+   * @see google.registry.model.tld.label.PremiumList
   */
  public static Duration getDomainLabelListCacheDuration() {
    return Duration.standardSeconds(CONFIG_SETTINGS.get().caching.domainLabelCachingSeconds);
@@ -1487,37 +1494,6 @@ public final class RegistryConfig {
    return CONFIG_SETTINGS.get().hibernate.hikariIdleTimeout;
  }

-  /**
-   * Returns whether to replicate cloud SQL transactions to datastore.
-   *
-   * <p>If true, all cloud SQL transactions will be persisted as TransactionEntity objects in the
-   * Transaction table and replayed against datastore in a cron job.
-   */
-  public static boolean getCloudSqlReplicateTransactions() {
-    return CONFIG_SETTINGS.get().cloudSql.replicateTransactions;
-  }
-
-  @VisibleForTesting
-  public static void overrideCloudSqlReplicateTransactions(boolean replicateTransactions) {
-    CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
-  }
-
-  /**
-   * Returns whether or not to replay commit logs to the SQL database after export to GCS.
-   *
-   * <p>If true, we will trigger the {@link google.registry.backup.ReplayCommitLogsToSqlAction}
-   * after the {@link google.registry.backup.ExportCommitLogDiffAction} to load the commit logs and
-   * replay them to SQL.
-   */
-  public static boolean getCloudSqlReplayCommitLogs() {
-    return CONFIG_SETTINGS.get().cloudSql.replayCommitLogs;
-  }
-
-  @VisibleForTesting
-  public static void overrideCloudSqlReplayCommitLogs(boolean replayCommitLogs) {
-    CONFIG_SETTINGS.get().cloudSql.replayCommitLogs = replayCommitLogs;
-  }
-
  /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
  public static String getContactAndHostRoidSuffix() {
    return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;
--- a/core/src/main/java/google/registry/config/RegistryConfigSettings.java
+++ b/core/src/main/java/google/registry/config/RegistryConfigSettings.java
@@ -126,8 +126,6 @@ public class RegistryConfigSettings {
    // TODO(05012021): remove username field after it is removed from all yaml files.
    public String username;
    public String instanceConnectionName;
-    public boolean replicateTransactions;
-    public boolean replayCommitLogs;
  }

  /** Configuration for Apache Beam (Cloud Dataflow). */
@@ -229,7 +227,10 @@ public class RegistryConfigSettings {
  public static class SslCertificateValidation {
    public Map<String, Integer> maxValidityDaysSchedule;
    public int expirationWarningDays;
+    public int expirationWarningIntervalDays;
    public int minimumRsaKeyLength;
    public Set<String> allowedEcdsaCurves;
+    public String expirationWarningEmailBodyText;
+    public String expirationWarningEmailSubjectText;
  }
 }
--- a/core/src/main/java/google/registry/config/files/default-config.yaml
+++ b/core/src/main/java/google/registry/config/files/default-config.yaml
@@ -227,11 +227,6 @@ cloudSql:
  jdbcUrl: jdbc:postgresql://localhost
  # This name is used by Cloud SQL when connecting to the database.
  instanceConnectionName: project-id:region:instance-id
-  # Set this to true to replicate cloud SQL transactions to datastore in the
-  # background.
-  replicateTransactions: false
-  # Set this to true to enable replay of commit logs to SQL
-  replayCommitLogs: false

 cloudDns:
  # Set both properties to null in Production.
@@ -454,6 +449,15 @@ sslCertificateValidation:
  # The number of days before a certificate expires that indicates the
  # certificate is nearing expiration and warnings should be sent.
  expirationWarningDays: 30
+  # The minimum number of days between two successive expiring notification emails.
+  expirationWarningIntervalDays: 15
+  # Text for expiring certificate notification email subject.
+  expirationWarningEmailSubjectText: Certificate Expring Within 30 Days.
+  # Text for expiring certificate notification email body that accepts 3 parameters:
+  # registrar name, certificate type, and expiration date, respectively.
+  expirationWarningEmailBodyText: |
+    Hello Registrar %s,
+       The %s certificate is expiring on %s.
  # The minimum number of bits an RSA key must contain.
  minimumRsaKeyLength: 2048
  # The ECDSA curves that are allowed for public keys.
--- a/core/src/main/java/google/registry/cron/TldFanoutAction.java
+++ b/core/src/main/java/google/registry/cron/TldFanoutAction.java
@@ -31,9 +31,9 @@ import static google.registry.cron.CronModule.FOR_EACH_TEST_TLD_PARAM;
 import static google.registry.cron.CronModule.JITTER_SECONDS_PARAM;
 import static google.registry.cron.CronModule.QUEUE_PARAM;
 import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
-import static google.registry.model.registry.Registries.getTldsOfType;
-import static google.registry.model.registry.Registry.TldType.REAL;
-import static google.registry.model.registry.Registry.TldType.TEST;
+import static google.registry.model.tld.Registries.getTldsOfType;
+import static google.registry.model.tld.Registry.TldType.REAL;
+import static google.registry.model.tld.Registry.TldType.TEST;
 import static java.util.concurrent.TimeUnit.SECONDS;

 import com.google.appengine.api.taskqueue.Queue;
--- a/core/src/main/java/google/registry/dns/DnsQueue.java
+++ b/core/src/main/java/google/registry/dns/DnsQueue.java
@@ -20,7 +20,7 @@ import static google.registry.dns.DnsConstants.DNS_PULL_QUEUE_NAME;
 import static google.registry.dns.DnsConstants.DNS_TARGET_CREATE_TIME_PARAM;
 import static google.registry.dns.DnsConstants.DNS_TARGET_NAME_PARAM;
 import static google.registry.dns.DnsConstants.DNS_TARGET_TYPE_PARAM;
-import static google.registry.model.registry.Registries.assertTldExists;
+import static google.registry.model.tld.Registries.assertTldExists;
 import static google.registry.request.RequestParameters.PARAM_TLD;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -38,7 +38,7 @@ import com.google.common.flogger.FluentLogger;
 import com.google.common.net.InternetDomainName;
 import com.google.common.util.concurrent.RateLimiter;
 import google.registry.dns.DnsConstants.TargetType;
-import google.registry.model.registry.Registries;
+import google.registry.model.tld.Registries;
 import google.registry.util.Clock;
 import google.registry.util.NonFinalForTesting;
 import google.registry.util.SystemClock;
--- a/core/src/main/java/google/registry/dns/DnsWriterProxy.java
+++ b/core/src/main/java/google/registry/dns/DnsWriterProxy.java
@@ -19,7 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.dns.writer.DnsWriter;
-import google.registry.model.registry.Registry;
+import google.registry.model.tld.Registry;
 import java.util.Map;
 import javax.inject.Inject;

--- a/core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java
+++ b/core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java
@@ -32,7 +32,7 @@ import google.registry.dns.DnsMetrics.ActionStatus;
 import google.registry.dns.DnsMetrics.CommitStatus;
 import google.registry.dns.DnsMetrics.PublishStatus;
 import google.registry.dns.writer.DnsWriter;
-import google.registry.model.registry.Registry;
+import google.registry.model.tld.Registry;
 import google.registry.request.Action;
 import google.registry.request.HttpException.ServiceUnavailableException;
 import google.registry.request.Parameter;
--- a/core/src/main/java/google/registry/dns/ReadDnsQueueAction.java
+++ b/core/src/main/java/google/registry/dns/ReadDnsQueueAction.java
@@ -47,8 +47,8 @@ import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.dns.DnsConstants.TargetType;
-import google.registry.model.registry.Registries;
-import google.registry.model.registry.Registry;
+import google.registry.model.tld.Registries;
+import google.registry.model.tld.Registry;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
--- a/core/src/main/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java
+++ b/core/src/main/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java
@@ -39,7 +39,7 @@ import google.registry.dns.writer.DnsWriterZone;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.host.HostResource;
-import google.registry.model.registry.Registries;
+import google.registry.model.tld.Registries;
 import google.registry.util.Clock;
 import google.registry.util.Concurrent;
 import google.registry.util.Retrier;
@@ -284,9 +284,10 @@ public class CloudDnsWriter extends BaseDnsWriter {
        .filter(hostName -> hostName.endsWith("." + domainName) && !hostName.equals(domainName));
  }

-  /** Mutate the zone with the provided {@code desiredRecords}. */
+  /** Mutate the zone with the provided map of hostnames to desired DNS records. */
  @VisibleForTesting
  void mutateZone(ImmutableMap<String, ImmutableSet<ResourceRecordSet>> desiredRecords) {
+    logger.atInfo().log("Updating DNS records for hostname(s) %s.", desiredRecords.keySet());
    // Fetch all existing records for names that this writer is trying to modify
    ImmutableSet.Builder<ResourceRecordSet> flattenedExistingRecords = new ImmutableSet.Builder<>();

@@ -313,7 +314,12 @@ public class CloudDnsWriter extends BaseDnsWriter {
    desiredRecords.values().forEach(flattenedDesiredRecords::addAll);

    // Delete all existing records and add back the desired records
-    updateResourceRecords(flattenedDesiredRecords.build(), flattenedExistingRecords.build());
+    try {
+      updateResourceRecords(flattenedDesiredRecords.build(), flattenedExistingRecords.build());
+    } catch (IOException e) {
+      throw new RuntimeException(
+          "Error updating DNS records for hostname(s) " + desiredRecords.keySet(), e);
+    }
  }

  /**
@@ -363,10 +369,12 @@ public class CloudDnsWriter extends BaseDnsWriter {
   *
   * @throws ZoneStateException if the operation could not be completely successfully because the
   *     records to delete do not exist, already exist or have been modified with different
-   *     attributes since being queried.
+   *     attributes since being queried. These errors will be retried.
+   * @throws IOException on non-retryable API errors, e.g. invalid request.
   */
  private void updateResourceRecords(
-      ImmutableSet<ResourceRecordSet> additions, ImmutableSet<ResourceRecordSet> deletions) {
+      ImmutableSet<ResourceRecordSet> additions, ImmutableSet<ResourceRecordSet> deletions)
+      throws IOException {
    // Find records that are both in additions and deletions, so we can remove them from both before
    // requesting the change. This is mostly for optimization reasons - not doing so doesn't affect
    // the result.
@@ -392,17 +400,15 @@ public class CloudDnsWriter extends BaseDnsWriter {
      GoogleJsonError err = e.getDetails();
      // We did something really wrong here, just give up and re-throw
      if (err == null || err.getErrors().size() > 1) {
-        throw new RuntimeException(e);
+        throw e;
      }
      String errorReason = err.getErrors().get(0).getReason();

      if (RETRYABLE_EXCEPTION_REASONS.contains(errorReason)) {
        throw new ZoneStateException(errorReason);
      } else {
-        throw new RuntimeException(e);
+        throw e;
      }
-    } catch (IOException e) {
-      throw new RuntimeException(e);
    }
  }

--- a/core/src/main/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriter.java
+++ b/core/src/main/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriter.java
@@ -30,7 +30,7 @@ import google.registry.dns.writer.DnsWriterZone;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.host.HostResource;
-import google.registry.model.registry.Registries;
+import google.registry.model.tld.Registries;
 import google.registry.util.Clock;
 import java.io.IOException;
 import java.net.Inet4Address;
--- a/core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml
+++ b/core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml
@@ -199,6 +199,15 @@
    <target>backend</target>
  </cron>

+  <cron>
+    <url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
+    <description>
+      Replays recent commit logs from Datastore to the SQL secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
  <cron>
    <url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
    <description>
--- a/core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml
+++ b/core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml
@@ -237,6 +237,12 @@
    <url-pattern>/_dr/task/killCommitLogs</url-pattern>
  </servlet-mapping>

+  <!-- Replays Datastore commit logs to SQL. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/replayCommitLogsToSql</url-pattern>
+  </servlet-mapping>
+
  <!-- MapReduce servlet. -->
  <servlet>
    <servlet-name>mapreduce</servlet-name>
--- a/core/src/main/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml
+++ b/core/src/main/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml
@@ -41,6 +41,11 @@
    <property name="nsHosts" direction="asc"/>
    <property name="deletionTime" direction="asc"/>
  </datastore-index>
+  <!-- For deleting expired not-previously-deleted domains. -->
+  <datastore-index kind="DomainBase" ancestor="false" source="manual">
+    <property name="deletionTime" direction="asc"/>
+    <property name="autorenewEndTime" direction="asc"/>
+  </datastore-index>
  <!-- For RDAP searches by linked nameserver. -->
  <datastore-index kind="DomainBase" ancestor="false" source="manual">
    <property name="nsHosts" direction="asc"/>
--- a/Show More
+++ b/Show More